Search in sources :

Example 1 with CoordinatorClient

use of org.apache.druid.client.coordinator.CoordinatorClient in project druid by druid-io.

the class CoordinatorBasedSegmentHandoffNotifierTest method testHandoffCallbackCalled.

@Test
public void testHandoffCallbackCalled() {
    Interval interval = Intervals.of("2011-04-01/2011-04-02");
    SegmentDescriptor descriptor = new SegmentDescriptor(interval, "v1", 2);
    final AtomicBoolean callbackCalled = new AtomicBoolean(false);
    CoordinatorClient coordinatorClient = EasyMock.createMock(CoordinatorClient.class);
    EasyMock.expect(coordinatorClient.isHandOffComplete("test_ds", descriptor)).andReturn(true).anyTimes();
    EasyMock.replay(coordinatorClient);
    CoordinatorBasedSegmentHandoffNotifier notifier = new CoordinatorBasedSegmentHandoffNotifier("test_ds", coordinatorClient, notifierConfig);
    notifier.registerSegmentHandoffCallback(descriptor, Execs.directExecutor(), () -> callbackCalled.set(true));
    Assert.assertEquals(1, notifier.getHandOffCallbacks().size());
    Assert.assertTrue(notifier.getHandOffCallbacks().containsKey(descriptor));
    notifier.checkForSegmentHandoffs();
    // callback should have been removed
    Assert.assertTrue(notifier.getHandOffCallbacks().isEmpty());
    Assert.assertTrue(callbackCalled.get());
    EasyMock.verify(coordinatorClient);
}
Also used : AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) SegmentDescriptor(org.apache.druid.query.SegmentDescriptor) CoordinatorClient(org.apache.druid.client.coordinator.CoordinatorClient) Interval(org.joda.time.Interval) Test(org.junit.Test)

Example 2 with CoordinatorClient

use of org.apache.druid.client.coordinator.CoordinatorClient in project druid by druid-io.

the class IngestSegmentFirehoseFactoryTimelineTest method constructorFeeder.

@Parameterized.Parameters(name = "{0}")
public static Collection<Object[]> constructorFeeder() {
    final List<TestCase> testCases = ImmutableList.of(tc("2000/2000T02", 3, 7, ds("2000/2000T01", "v1", 0, ir("2000", 1), ir("2000T00:01", 2)), ds("2000T01/2000T02", "v1", 0, ir("2000T01", 4))), /* Adjacent segments */
    tc("2000/2000T02", 3, 7, ds("2000/2000T02", "v1", 0, ir("2000", 1), ir("2000T00:01", 2), ir("2000T01", 8)), ds("2000T01/2000T02", "v2", 0, ir("2000T01:01", 4))), /* 1H segment overlaid on top of 2H segment */
    tc("2000/2000-01-02", 4, 23, ds("2000/2000-01-02", "v1", 0, ir("2000", 1), ir("2000T00:01", 2), ir("2000T01", 8), ir("2000T02", 16)), ds("2000T01/2000T02", "v2", 0, ir("2000T01:01", 4))), /* 1H segment overlaid on top of 1D segment */
    tc("2000/2000T02", 4, 15, ds("2000/2000T02", "v1", 0, ir("2000", 1), ir("2000T00:01", 2), ir("2000T01", 8)), ds("2000/2000T02", "v1", 1, ir("2000T01:01", 4))), /* Segment set with two segments for the same interval */
    tc("2000T01/2000T02", 1, 2, ds("2000/2000T03", "v1", 0, ir("2000", 1), ir("2000T01", 2), ir("2000T02", 4))), /* Segment wider than desired interval */
    tc("2000T02/2000T04", 2, 12, ds("2000/2000T03", "v1", 0, ir("2000", 1), ir("2000T01", 2), ir("2000T02", 4)), ds("2000T03/2000T04", "v1", 0, ir("2000T03", 8))));
    final List<Object[]> constructors = new ArrayList<>();
    for (final TestCase testCase : testCases) {
        SegmentHandoffNotifierFactory notifierFactory = EasyMock.createNiceMock(SegmentHandoffNotifierFactory.class);
        EasyMock.replay(notifierFactory);
        final SegmentCacheManagerFactory slf = new SegmentCacheManagerFactory(MAPPER);
        final RetryPolicyFactory retryPolicyFactory = new RetryPolicyFactory(new RetryPolicyConfig());
        final CoordinatorClient cc = new CoordinatorClient(null, null) {

            @Override
            public Collection<DataSegment> fetchUsedSegmentsInDataSourceForIntervals(String dataSource, List<Interval> intervals) {
                // Expect the interval we asked for
                if (intervals.equals(ImmutableList.of(testCase.interval))) {
                    return ImmutableSet.copyOf(testCase.segments);
                } else {
                    throw new IllegalArgumentException("BAD");
                }
            }

            @Override
            public DataSegment fetchUsedSegment(String dataSource, String segmentId) {
                return testCase.segments.stream().filter(s -> s.getId().toString().equals(segmentId)).findAny().get();
            }
        };
        final IngestSegmentFirehoseFactory factory = new IngestSegmentFirehoseFactory(DATA_SOURCE, testCase.interval, null, TrueDimFilter.instance(), Arrays.asList(DIMENSIONS), Arrays.asList(METRICS), // Split as much as possible
        1L, INDEX_IO, cc, slf, retryPolicyFactory);
        constructors.add(new Object[] { testCase.toString(), factory, testCase.tmpDir, testCase.expectedCount, testCase.expectedSum, testCase.segments.size() });
    }
    return constructors;
}
Also used : RetryPolicyConfig(org.apache.druid.indexing.common.RetryPolicyConfig) ArrayList(java.util.ArrayList) SegmentCacheManagerFactory(org.apache.druid.indexing.common.SegmentCacheManagerFactory) RetryPolicyFactory(org.apache.druid.indexing.common.RetryPolicyFactory) DataSegment(org.apache.druid.timeline.DataSegment) SegmentHandoffNotifierFactory(org.apache.druid.segment.handoff.SegmentHandoffNotifierFactory) CoordinatorClient(org.apache.druid.client.coordinator.CoordinatorClient) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList)

Example 3 with CoordinatorClient

use of org.apache.druid.client.coordinator.CoordinatorClient in project druid by druid-io.

the class IngestSegmentFirehoseFactoryTest method constructorFeeder.

@Parameterized.Parameters(name = "{0}")
public static Collection<Object[]> constructorFeeder() throws IOException {
    final IndexSpec indexSpec = new IndexSpec();
    final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder().withMinTimestamp(JodaUtils.MIN_INSTANT).withDimensionsSpec(ROW_PARSER).withMetrics(new LongSumAggregatorFactory(METRIC_LONG_NAME, DIM_LONG_NAME), new DoubleSumAggregatorFactory(METRIC_FLOAT_NAME, DIM_FLOAT_NAME)).build();
    final IncrementalIndex index = new OnheapIncrementalIndex.Builder().setIndexSchema(schema).setMaxRowCount(MAX_ROWS * MAX_SHARD_NUMBER).build();
    for (Integer i = 0; i < MAX_ROWS; ++i) {
        index.add(ROW_PARSER.parseBatch(buildRow(i.longValue())).get(0));
    }
    FileUtils.mkdirp(PERSIST_DIR);
    INDEX_MERGER_V9.persist(index, PERSIST_DIR, indexSpec, null);
    final CoordinatorClient cc = new CoordinatorClient(null, null) {

        @Override
        public Collection<DataSegment> fetchUsedSegmentsInDataSourceForIntervals(String dataSource, List<Interval> intervals) {
            return ImmutableSet.copyOf(SEGMENT_SET);
        }
    };
    SegmentHandoffNotifierFactory notifierFactory = EasyMock.createNiceMock(SegmentHandoffNotifierFactory.class);
    EasyMock.replay(notifierFactory);
    final SegmentCacheManagerFactory slf = new SegmentCacheManagerFactory(MAPPER);
    final RetryPolicyFactory retryPolicyFactory = new RetryPolicyFactory(new RetryPolicyConfig());
    Collection<Object[]> values = new ArrayList<>();
    for (InputRowParser parser : Arrays.<InputRowParser>asList(ROW_PARSER, new MapInputRowParser(new JSONParseSpec(new TimestampSpec(TIME_COLUMN, "auto", null), DimensionsSpec.builder().setDimensionExclusions(ImmutableList.of(DIM_FLOAT_NAME, DIM_LONG_NAME)).build(), null, null, null)))) {
        for (List<String> dim_names : Arrays.<List<String>>asList(null, ImmutableList.of(DIM_NAME))) {
            for (List<String> metric_names : Arrays.<List<String>>asList(null, ImmutableList.of(METRIC_LONG_NAME, METRIC_FLOAT_NAME))) {
                for (Boolean wrapInCombining : Arrays.asList(false, true)) {
                    final IngestSegmentFirehoseFactory isfFactory = new IngestSegmentFirehoseFactory(TASK.getDataSource(), Intervals.ETERNITY, null, new SelectorDimFilter(DIM_NAME, DIM_VALUE, null), dim_names, metric_names, null, INDEX_IO, cc, slf, retryPolicyFactory);
                    final FirehoseFactory factory = wrapInCombining ? new CombiningFirehoseFactory(ImmutableList.of(isfFactory)) : isfFactory;
                    values.add(new Object[] { StringUtils.format("DimNames[%s]MetricNames[%s]ParserDimNames[%s]WrapInCombining[%s]", dim_names == null ? "null" : "dims", metric_names == null ? "null" : "metrics", parser == ROW_PARSER ? "dims" : "null", wrapInCombining), factory, parser });
                }
            }
        }
    }
    return values;
}
Also used : IndexSpec(org.apache.druid.segment.IndexSpec) MapInputRowParser(org.apache.druid.data.input.impl.MapInputRowParser) CombiningFirehoseFactory(org.apache.druid.segment.realtime.firehose.CombiningFirehoseFactory) FirehoseFactory(org.apache.druid.data.input.FirehoseFactory) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) ArrayList(java.util.ArrayList) DataSegment(org.apache.druid.timeline.DataSegment) SelectorDimFilter(org.apache.druid.query.filter.SelectorDimFilter) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) CoordinatorClient(org.apache.druid.client.coordinator.CoordinatorClient) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) JSONParseSpec(org.apache.druid.data.input.impl.JSONParseSpec) IncrementalIndexSchema(org.apache.druid.segment.incremental.IncrementalIndexSchema) CombiningFirehoseFactory(org.apache.druid.segment.realtime.firehose.CombiningFirehoseFactory) DoubleSumAggregatorFactory(org.apache.druid.query.aggregation.DoubleSumAggregatorFactory) RetryPolicyConfig(org.apache.druid.indexing.common.RetryPolicyConfig) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) SegmentCacheManagerFactory(org.apache.druid.indexing.common.SegmentCacheManagerFactory) RetryPolicyFactory(org.apache.druid.indexing.common.RetryPolicyFactory) SegmentHandoffNotifierFactory(org.apache.druid.segment.handoff.SegmentHandoffNotifierFactory) InputRowParser(org.apache.druid.data.input.impl.InputRowParser) MapInputRowParser(org.apache.druid.data.input.impl.MapInputRowParser)

Example 4 with CoordinatorClient

use of org.apache.druid.client.coordinator.CoordinatorClient in project druid by druid-io.

the class DruidInputSource method createSplits.

public static Iterator<InputSplit<List<WindowedSegmentId>>> createSplits(CoordinatorClient coordinatorClient, RetryPolicyFactory retryPolicyFactory, String dataSource, Interval interval, SplitHintSpec splitHintSpec) {
    final SplitHintSpec convertedSplitHintSpec;
    if (splitHintSpec instanceof SegmentsSplitHintSpec) {
        final SegmentsSplitHintSpec segmentsSplitHintSpec = (SegmentsSplitHintSpec) splitHintSpec;
        convertedSplitHintSpec = new MaxSizeSplitHintSpec(segmentsSplitHintSpec.getMaxInputSegmentBytesPerTask(), segmentsSplitHintSpec.getMaxNumSegments());
    } else {
        convertedSplitHintSpec = splitHintSpec;
    }
    final List<TimelineObjectHolder<String, DataSegment>> timelineSegments = getTimelineForInterval(coordinatorClient, retryPolicyFactory, dataSource, interval);
    final Map<WindowedSegmentId, Long> segmentIdToSize = createWindowedSegmentIdFromTimeline(timelineSegments);
    // noinspection ConstantConditions
    return Iterators.transform(convertedSplitHintSpec.split(// the same input split.
    segmentIdToSize.keySet().iterator(), segmentId -> new InputFileAttribute(Preconditions.checkNotNull(segmentIdToSize.get(segmentId), "segment size for [%s]", segmentId))), InputSplit::new);
}
Also used : JsonProperty(com.fasterxml.jackson.annotation.JsonProperty) SegmentCacheManagerFactory(org.apache.druid.indexing.common.SegmentCacheManagerFactory) TaskConfig(org.apache.druid.indexing.common.config.TaskConfig) Comparators(org.apache.druid.java.util.common.guava.Comparators) AbstractInputSource(org.apache.druid.data.input.AbstractInputSource) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) FluentIterable(com.google.common.collect.FluentIterable) Map(java.util.Map) InputSourceReader(org.apache.druid.data.input.InputSourceReader) IAE(org.apache.druid.java.util.common.IAE) JacksonInject(com.fasterxml.jackson.annotation.JacksonInject) RetryPolicyFactory(org.apache.druid.indexing.common.RetryPolicyFactory) InputFormat(org.apache.druid.data.input.InputFormat) Collection(java.util.Collection) SplitHintSpec(org.apache.druid.data.input.SplitHintSpec) SplittableInputSource(org.apache.druid.data.input.impl.SplittableInputSource) ISE(org.apache.druid.java.util.common.ISE) Objects(java.util.Objects) MaxSizeSplitHintSpec(org.apache.druid.data.input.MaxSizeSplitHintSpec) PartitionHolder(org.apache.druid.timeline.partition.PartitionHolder) List(java.util.List) Stream(java.util.stream.Stream) DimFilter(org.apache.druid.query.filter.DimFilter) DataSegment(org.apache.druid.timeline.DataSegment) SortedMap(java.util.SortedMap) Logger(org.apache.druid.java.util.common.logger.Logger) Streams(org.apache.druid.utils.Streams) InputSplit(org.apache.druid.data.input.InputSplit) Duration(org.joda.time.Duration) SegmentsSplitHintSpec(org.apache.druid.data.input.SegmentsSplitHintSpec) HashMap(java.util.HashMap) InputRowSchema(org.apache.druid.data.input.InputRowSchema) Iterators(com.google.common.collect.Iterators) ArrayList(java.util.ArrayList) PartitionChunk(org.apache.druid.timeline.partition.PartitionChunk) Interval(org.joda.time.Interval) ColumnHolder(org.apache.druid.segment.column.ColumnHolder) ImmutableList(com.google.common.collect.ImmutableList) WindowedSegmentId(org.apache.druid.indexing.firehose.WindowedSegmentId) CoordinatorClient(org.apache.druid.client.coordinator.CoordinatorClient) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) InputFileAttribute(org.apache.druid.data.input.InputFileAttribute) Nullable(javax.annotation.Nullable) RetryPolicy(org.apache.druid.indexing.common.RetryPolicy) VersionedIntervalTimeline(org.apache.druid.timeline.VersionedIntervalTimeline) Iterator(java.util.Iterator) TimelineObjectHolder(org.apache.druid.timeline.TimelineObjectHolder) File(java.io.File) InputEntityIteratingReader(org.apache.druid.data.input.impl.InputEntityIteratingReader) TreeMap(java.util.TreeMap) JsonCreator(com.fasterxml.jackson.annotation.JsonCreator) JsonInclude(com.fasterxml.jackson.annotation.JsonInclude) Preconditions(com.google.common.base.Preconditions) SegmentCacheManager(org.apache.druid.segment.loading.SegmentCacheManager) Comparator(java.util.Comparator) IndexIO(org.apache.druid.segment.IndexIO) Collections(java.util.Collections) SegmentsSplitHintSpec(org.apache.druid.data.input.SegmentsSplitHintSpec) TimelineObjectHolder(org.apache.druid.timeline.TimelineObjectHolder) InputFileAttribute(org.apache.druid.data.input.InputFileAttribute) WindowedSegmentId(org.apache.druid.indexing.firehose.WindowedSegmentId) InputSplit(org.apache.druid.data.input.InputSplit) SplitHintSpec(org.apache.druid.data.input.SplitHintSpec) MaxSizeSplitHintSpec(org.apache.druid.data.input.MaxSizeSplitHintSpec) SegmentsSplitHintSpec(org.apache.druid.data.input.SegmentsSplitHintSpec) MaxSizeSplitHintSpec(org.apache.druid.data.input.MaxSizeSplitHintSpec)

Example 5 with CoordinatorClient

use of org.apache.druid.client.coordinator.CoordinatorClient in project druid by druid-io.

the class CoordinatorBasedSegmentHandoffNotifierTest method testHandoffCallbackNotCalled.

@Test
public void testHandoffCallbackNotCalled() {
    Interval interval = Intervals.of("2011-04-01/2011-04-02");
    SegmentDescriptor descriptor = new SegmentDescriptor(interval, "v1", 2);
    CoordinatorClient coordinatorClient = EasyMock.createMock(CoordinatorClient.class);
    EasyMock.expect(coordinatorClient.isHandOffComplete("test_ds", descriptor)).andReturn(false).anyTimes();
    EasyMock.replay(coordinatorClient);
    CoordinatorBasedSegmentHandoffNotifier notifier = new CoordinatorBasedSegmentHandoffNotifier("test_ds", coordinatorClient, notifierConfig);
    final AtomicBoolean callbackCalled = new AtomicBoolean(false);
    notifier.registerSegmentHandoffCallback(descriptor, Execs.directExecutor(), () -> callbackCalled.set(true));
    notifier.checkForSegmentHandoffs();
    // callback should have registered
    Assert.assertEquals(1, notifier.getHandOffCallbacks().size());
    Assert.assertTrue(notifier.getHandOffCallbacks().containsKey(descriptor));
    Assert.assertFalse(callbackCalled.get());
    EasyMock.verify(coordinatorClient);
}
Also used : AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) SegmentDescriptor(org.apache.druid.query.SegmentDescriptor) CoordinatorClient(org.apache.druid.client.coordinator.CoordinatorClient) Interval(org.joda.time.Interval) Test(org.junit.Test)

Aggregations

CoordinatorClient (org.apache.druid.client.coordinator.CoordinatorClient)5 ImmutableList (com.google.common.collect.ImmutableList)3 ArrayList (java.util.ArrayList)3 List (java.util.List)3 RetryPolicyFactory (org.apache.druid.indexing.common.RetryPolicyFactory)3 Interval (org.joda.time.Interval)3 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 TimestampSpec (org.apache.druid.data.input.impl.TimestampSpec)2 RetryPolicyConfig (org.apache.druid.indexing.common.RetryPolicyConfig)2 SegmentCacheManagerFactory (org.apache.druid.indexing.common.SegmentCacheManagerFactory)2 SegmentHandoffNotifierFactory (org.apache.druid.segment.handoff.SegmentHandoffNotifierFactory)2 DataSegment (org.apache.druid.timeline.DataSegment)2 JacksonInject (com.fasterxml.jackson.annotation.JacksonInject)1 JsonCreator (com.fasterxml.jackson.annotation.JsonCreator)1 JsonInclude (com.fasterxml.jackson.annotation.JsonInclude)1 JsonProperty (com.fasterxml.jackson.annotation.JsonProperty)1 Preconditions (com.google.common.base.Preconditions)1 FluentIterable (com.google.common.collect.FluentIterable)1 Iterators (com.google.common.collect.Iterators)1 File (java.io.File)1