Examples with SegmentDescriptor io.druid.query.SegmentDescriptor used on opensource projects

Example 6 with SegmentDescriptor

use of io.druid.query.SegmentDescriptor in project druid by druid-io.

the class CachingClusteredClientTest method toFilteredQueryableTimeseriesResults.

private Sequence<Result<TimeseriesResultValue>> toFilteredQueryableTimeseriesResults(TimeseriesQuery query, List<String> segmentIds, List<Interval> queryIntervals, List<Iterable<Result<TimeseriesResultValue>>> results) {
    MultipleSpecificSegmentSpec spec = (MultipleSpecificSegmentSpec) query.getQuerySegmentSpec();
    List<Result<TimeseriesResultValue>> ret = Lists.newArrayList();
    for (SegmentDescriptor descriptor : spec.getDescriptors()) {
        String id = String.format("%s_%s", queryIntervals.indexOf(descriptor.getInterval()), descriptor.getPartitionNumber());
        int index = segmentIds.indexOf(id);
        if (index != -1) {
            ret.add(new Result(results.get(index).iterator().next().getTimestamp(), new BySegmentResultValueClass(Lists.newArrayList(results.get(index)), id, descriptor.getInterval())));
        } else {
            throw new ISE("Descriptor %s not found in server", id);
        }
    }
    return Sequences.simple(ret);
}

Example 7 with SegmentDescriptor

use of io.druid.query.SegmentDescriptor in project druid by druid-io.

the class RealtimeManagerTest method testQueryWithSegmentSpec.

@Test(timeout = 10_000L)
public void testQueryWithSegmentSpec() throws IOException, InterruptedException {
    List<Row> expectedResults = Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L));
    chiefStartedLatch.await();
    for (QueryRunner runner : QueryRunnerTestHelper.makeQueryRunners((GroupByQueryRunnerFactory) factory)) {
        GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource).setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird).setDimensions(Lists.<DimensionSpec>newArrayList(new DefaultDimensionSpec("quality", "alias"))).setAggregatorSpecs(Arrays.asList(QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index"))).setGranularity(QueryRunnerTestHelper.dayGran).build();
        plumber.setRunners(ImmutableMap.of(query.getIntervals().get(0), runner));
        plumber2.setRunners(ImmutableMap.of(query.getIntervals().get(0), runner));
        Iterable<Row> results = GroupByQueryRunnerTestHelper.runQuery(factory, realtimeManager3.getQueryRunnerForSegments(query, ImmutableList.<SegmentDescriptor>of(new SegmentDescriptor(new Interval("2011-04-01T00:00:00.000Z/2011-04-03T00:00:00.000Z"), "ver", 0))), query);
        TestHelper.assertExpectedObjects(expectedResults, results, "");
        results = GroupByQueryRunnerTestHelper.runQuery(factory, realtimeManager3.getQueryRunnerForSegments(query, ImmutableList.<SegmentDescriptor>of(new SegmentDescriptor(new Interval("2011-04-01T00:00:00.000Z/2011-04-03T00:00:00.000Z"), "ver", 1))), query);
        TestHelper.assertExpectedObjects(expectedResults, results, "");
    }
}

Example 8 with SegmentDescriptor

use of io.druid.query.SegmentDescriptor in project druid by druid-io.

the class AppenderatorTest method testQueryBySegments.

@Test
public void testQueryBySegments() throws Exception {
    try (final AppenderatorTester tester = new AppenderatorTester(2)) {
        final Appenderator appenderator = tester.getAppenderator();
        appenderator.startJob();
        appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 2), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(1), IR("2000", "foo", 4), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(2), IR("2001", "foo", 8), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(2), IR("2001T01", "foo", 16), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(2), IR("2001T02", "foo", 32), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(2), IR("2001T03", "foo", 64), Suppliers.ofInstance(Committers.nil()));
        // Query1: segment #2
        final TimeseriesQuery query1 = Druids.newTimeseriesQueryBuilder().dataSource(AppenderatorTester.DATASOURCE).aggregators(Arrays.<AggregatorFactory>asList(new LongSumAggregatorFactory("count", "count"), new LongSumAggregatorFactory("met", "met"))).granularity(Granularities.DAY).intervals(new MultipleSpecificSegmentSpec(ImmutableList.of(new SegmentDescriptor(IDENTIFIERS.get(2).getInterval(), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum())))).build();
        final List<Result<TimeseriesResultValue>> results1 = Lists.newArrayList();
        Sequences.toList(query1.run(appenderator, ImmutableMap.<String, Object>of()), results1);
        Assert.assertEquals("query1", ImmutableList.of(new Result<>(new DateTime("2001"), new TimeseriesResultValue(ImmutableMap.<String, Object>of("count", 4L, "met", 120L)))), results1);
        // Query2: segment #2, partial
        final TimeseriesQuery query2 = Druids.newTimeseriesQueryBuilder().dataSource(AppenderatorTester.DATASOURCE).aggregators(Arrays.<AggregatorFactory>asList(new LongSumAggregatorFactory("count", "count"), new LongSumAggregatorFactory("met", "met"))).granularity(Granularities.DAY).intervals(new MultipleSpecificSegmentSpec(ImmutableList.of(new SegmentDescriptor(new Interval("2001/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum())))).build();
        final List<Result<TimeseriesResultValue>> results2 = Lists.newArrayList();
        Sequences.toList(query2.run(appenderator, ImmutableMap.<String, Object>of()), results2);
        Assert.assertEquals("query2", ImmutableList.of(new Result<>(new DateTime("2001"), new TimeseriesResultValue(ImmutableMap.<String, Object>of("count", 1L, "met", 8L)))), results2);
        // Query3: segment #2, two disjoint intervals
        final TimeseriesQuery query3 = Druids.newTimeseriesQueryBuilder().dataSource(AppenderatorTester.DATASOURCE).aggregators(Arrays.<AggregatorFactory>asList(new LongSumAggregatorFactory("count", "count"), new LongSumAggregatorFactory("met", "met"))).granularity(Granularities.DAY).intervals(new MultipleSpecificSegmentSpec(ImmutableList.of(new SegmentDescriptor(new Interval("2001/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum()), new SegmentDescriptor(new Interval("2001T03/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum())))).build();
        final List<Result<TimeseriesResultValue>> results3 = Lists.newArrayList();
        Sequences.toList(query3.run(appenderator, ImmutableMap.<String, Object>of()), results3);
        Assert.assertEquals("query2", ImmutableList.of(new Result<>(new DateTime("2001"), new TimeseriesResultValue(ImmutableMap.<String, Object>of("count", 2L, "met", 72L)))), results3);
    }
}

Example 9 with SegmentDescriptor

use of io.druid.query.SegmentDescriptor in project druid by druid-io.

the class CoordinatorBasedSegmentHandoffNotifierTest method testHandoffCallbackCalled.

@Test
public void testHandoffCallbackCalled() throws IOException, InterruptedException {
    Interval interval = new Interval("2011-04-01/2011-04-02");
    SegmentDescriptor descriptor = new SegmentDescriptor(interval, "v1", 2);
    DataSegment segment = new DataSegment("test_ds", interval, "v1", null, null, null, new NumberedShardSpec(2, 3), 0, 0);
    final AtomicBoolean callbackCalled = new AtomicBoolean(false);
    CoordinatorClient coordinatorClient = EasyMock.createMock(CoordinatorClient.class);
    EasyMock.expect(coordinatorClient.fetchServerView("test_ds", interval, true)).andReturn(Lists.newArrayList(new ImmutableSegmentLoadInfo(segment, Sets.newHashSet(createHistoricalServerMetadata("a1"))))).anyTimes();
    EasyMock.replay(coordinatorClient);
    CoordinatorBasedSegmentHandoffNotifier notifier = new CoordinatorBasedSegmentHandoffNotifier("test_ds", coordinatorClient, notifierConfig);
    notifier.registerSegmentHandoffCallback(descriptor, MoreExecutors.sameThreadExecutor(), new Runnable() {

        @Override
        public void run() {
            callbackCalled.set(true);
        }
    });
    Assert.assertEquals(1, notifier.getHandOffCallbacks().size());
    Assert.assertTrue(notifier.getHandOffCallbacks().containsKey(descriptor));
    notifier.checkForSegmentHandoffs();
    // callback should have been removed
    Assert.assertTrue(notifier.getHandOffCallbacks().isEmpty());
    Assert.assertTrue(callbackCalled.get());
    EasyMock.verify(coordinatorClient);
}

Example 10 with SegmentDescriptor

use of io.druid.query.SegmentDescriptor in project druid by druid-io.

the class CoordinatorBasedSegmentHandoffNotifierTest method testHandoffChecksForInterval.

@Test
public void testHandoffChecksForInterval() {
    Assert.assertFalse(CoordinatorBasedSegmentHandoffNotifier.isHandOffComplete(Lists.newArrayList(new ImmutableSegmentLoadInfo(createSegment(new Interval("2011-04-01/2011-04-02"), "v1", 1), Sets.newHashSet(createHistoricalServerMetadata("a")))), new SegmentDescriptor(new Interval("2011-04-01/2011-04-03"), "v1", 1)));
    Assert.assertTrue(CoordinatorBasedSegmentHandoffNotifier.isHandOffComplete(Lists.newArrayList(new ImmutableSegmentLoadInfo(createSegment(new Interval("2011-04-01/2011-04-04"), "v1", 1), Sets.newHashSet(createHistoricalServerMetadata("a")))), new SegmentDescriptor(new Interval("2011-04-02/2011-04-03"), "v1", 1)));
}