Examples with SegmentDescriptor org.apache.druid.query.SegmentDescriptor used on opensource projects

Example 16 with SegmentDescriptor

use of org.apache.druid.query.SegmentDescriptor in project druid by druid-io.

the class StreamAppenderatorTest method testQueryBySegments.

@Test
public void testQueryBySegments() throws Exception {
    try (final StreamAppenderatorTester tester = new StreamAppenderatorTester(2, true)) {
        final Appenderator appenderator = tester.getAppenderator();
        appenderator.startJob();
        appenderator.add(IDENTIFIERS.get(0), ir("2000", "foo", 1), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(0), ir("2000", "foo", 2), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(1), ir("2000", "foo", 4), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(2), ir("2001", "foo", 8), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(2), ir("2001T01", "foo", 16), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(2), ir("2001T02", "foo", 32), Suppliers.ofInstance(Committers.nil()));
        appenderator.add(IDENTIFIERS.get(2), ir("2001T03", "foo", 64), Suppliers.ofInstance(Committers.nil()));
        // Query1: segment #2
        final TimeseriesQuery query1 = Druids.newTimeseriesQueryBuilder().dataSource(StreamAppenderatorTester.DATASOURCE).aggregators(Arrays.asList(new LongSumAggregatorFactory("count", "count"), new LongSumAggregatorFactory("met", "met"))).granularity(Granularities.DAY).intervals(new MultipleSpecificSegmentSpec(ImmutableList.of(new SegmentDescriptor(IDENTIFIERS.get(2).getInterval(), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum())))).build();
        final List<Result<TimeseriesResultValue>> results1 = QueryPlus.wrap(query1).run(appenderator, ResponseContext.createEmpty()).toList();
        Assert.assertEquals("query1", ImmutableList.of(new Result<>(DateTimes.of("2001"), new TimeseriesResultValue(ImmutableMap.of("count", 4L, "met", 120L)))), results1);
        // Query2: segment #2, partial
        final TimeseriesQuery query2 = Druids.newTimeseriesQueryBuilder().dataSource(StreamAppenderatorTester.DATASOURCE).aggregators(Arrays.asList(new LongSumAggregatorFactory("count", "count"), new LongSumAggregatorFactory("met", "met"))).granularity(Granularities.DAY).intervals(new MultipleSpecificSegmentSpec(ImmutableList.of(new SegmentDescriptor(Intervals.of("2001/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum())))).build();
        final List<Result<TimeseriesResultValue>> results2 = QueryPlus.wrap(query2).run(appenderator, ResponseContext.createEmpty()).toList();
        Assert.assertEquals("query2", ImmutableList.of(new Result<>(DateTimes.of("2001"), new TimeseriesResultValue(ImmutableMap.of("count", 1L, "met", 8L)))), results2);
        // Query3: segment #2, two disjoint intervals
        final TimeseriesQuery query3 = Druids.newTimeseriesQueryBuilder().dataSource(StreamAppenderatorTester.DATASOURCE).aggregators(Arrays.asList(new LongSumAggregatorFactory("count", "count"), new LongSumAggregatorFactory("met", "met"))).granularity(Granularities.DAY).intervals(new MultipleSpecificSegmentSpec(ImmutableList.of(new SegmentDescriptor(Intervals.of("2001/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum()), new SegmentDescriptor(Intervals.of("2001T03/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum())))).build();
        final List<Result<TimeseriesResultValue>> results3 = QueryPlus.wrap(query3).run(appenderator, ResponseContext.createEmpty()).toList();
        Assert.assertEquals("query3", ImmutableList.of(new Result<>(DateTimes.of("2001"), new TimeseriesResultValue(ImmutableMap.of("count", 2L, "met", 72L)))), results3);
        final ScanQuery query4 = Druids.newScanQueryBuilder().dataSource(StreamAppenderatorTester.DATASOURCE).intervals(new MultipleSpecificSegmentSpec(ImmutableList.of(new SegmentDescriptor(Intervals.of("2001/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum()), new SegmentDescriptor(Intervals.of("2001T03/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum())))).order(ScanQuery.Order.ASCENDING).batchSize(10).resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST).build();
        final List<ScanResultValue> results4 = QueryPlus.wrap(query4).run(appenderator, ResponseContext.createEmpty()).toList();
        // 2 segments, 1 row per segment
        Assert.assertEquals(2, results4.size());
        Assert.assertArrayEquals(new String[] { "__time", "dim", "count", "met" }, results4.get(0).getColumns().toArray());
        Assert.assertArrayEquals(new Object[] { DateTimes.of("2001").getMillis(), "foo", 1L, 8L }, ((List<Object>) ((List<Object>) results4.get(0).getEvents()).get(0)).toArray());
        Assert.assertArrayEquals(new String[] { "__time", "dim", "count", "met" }, results4.get(0).getColumns().toArray());
        Assert.assertArrayEquals(new Object[] { DateTimes.of("2001T03").getMillis(), "foo", 1L, 64L }, ((List<Object>) ((List<Object>) results4.get(1).getEvents()).get(0)).toArray());
    }
}

Example 17 with SegmentDescriptor

use of org.apache.druid.query.SegmentDescriptor in project druid by druid-io.

the class ServerManagerTest method testGetQueryRunnerForSegmentsWhenTimelineEntryIsMissingReportingMissingSegments.

@Test
public void testGetQueryRunnerForSegmentsWhenTimelineEntryIsMissingReportingMissingSegments() {
    final Interval interval = Intervals.of("P1d/2011-04-01");
    final SearchQuery query = searchQuery("test", interval, Granularities.ALL);
    final List<SegmentDescriptor> unknownSegments = Collections.singletonList(new SegmentDescriptor(interval, "unknown_version", 0));
    final QueryRunner<Result<SearchResultValue>> queryRunner = serverManager.getQueryRunnerForSegments(query, unknownSegments);
    final ResponseContext responseContext = DefaultResponseContext.createEmpty();
    final List<Result<SearchResultValue>> results = queryRunner.run(QueryPlus.wrap(query), responseContext).toList();
    Assert.assertTrue(results.isEmpty());
    Assert.assertNotNull(responseContext.getMissingSegments());
    Assert.assertEquals(unknownSegments, responseContext.getMissingSegments());
}

Example 18 with SegmentDescriptor

use of org.apache.druid.query.SegmentDescriptor in project druid by druid-io.

the class ServerManagerTest method testGetQueryRunnerForSegmentsForUnknownQueryThrowingException.

@Test
public void testGetQueryRunnerForSegmentsForUnknownQueryThrowingException() {
    final Interval interval = Intervals.of("P1d/2011-04-01");
    final List<SegmentDescriptor> descriptors = Collections.singletonList(new SegmentDescriptor(interval, "1", 0));
    expectedException.expect(QueryUnsupportedException.class);
    expectedException.expectMessage("Unknown query type");
    serverManager.getQueryRunnerForSegments(new BaseQuery<Object>(new TableDataSource("test"), new MultipleSpecificSegmentSpec(descriptors), false, new HashMap<>()) {

        @Override
        public boolean hasFilters() {
            return false;
        }

        @Override
        public DimFilter getFilter() {
            return null;
        }

        @Override
        public String getType() {
            return null;
        }

        @Override
        public Query<Object> withOverriddenContext(Map<String, Object> contextOverride) {
            return null;
        }

        @Override
        public Query<Object> withQuerySegmentSpec(QuerySegmentSpec spec) {
            return null;
        }

        @Override
        public Query<Object> withDataSource(DataSource dataSource) {
            return null;
        }
    }, descriptors);
}

Example 19 with SegmentDescriptor

use of org.apache.druid.query.SegmentDescriptor in project druid by druid-io.

the class ServerManager method buildAndDecorateQueryRunner.

private <T> QueryRunner<T> buildAndDecorateQueryRunner(final QueryRunnerFactory<T, Query<T>> factory, final QueryToolChest<T, Query<T>> toolChest, final SegmentReference segment, final Optional<byte[]> cacheKeyPrefix, final SegmentDescriptor segmentDescriptor, final AtomicLong cpuTimeAccumulator) {
    final SpecificSegmentSpec segmentSpec = new SpecificSegmentSpec(segmentDescriptor);
    final SegmentId segmentId = segment.getId();
    final Interval segmentInterval = segment.getDataInterval();
    // If the segment is closed after this line, ReferenceCountingSegmentQueryRunner will handle and do the right thing.
    if (segmentId == null || segmentInterval == null) {
        return new ReportTimelineMissingSegmentQueryRunner<>(segmentDescriptor);
    }
    String segmentIdString = segmentId.toString();
    MetricsEmittingQueryRunner<T> metricsEmittingQueryRunnerInner = new MetricsEmittingQueryRunner<>(emitter, toolChest, new ReferenceCountingSegmentQueryRunner<>(factory, segment, segmentDescriptor), QueryMetrics::reportSegmentTime, queryMetrics -> queryMetrics.segment(segmentIdString));
    StorageAdapter storageAdapter = segment.asStorageAdapter();
    long segmentMaxTime = storageAdapter.getMaxTime().getMillis();
    long segmentMinTime = storageAdapter.getMinTime().getMillis();
    Interval actualDataInterval = Intervals.utc(segmentMinTime, segmentMaxTime + 1);
    CachingQueryRunner<T> cachingQueryRunner = new CachingQueryRunner<>(segmentIdString, cacheKeyPrefix, segmentDescriptor, actualDataInterval, objectMapper, cache, toolChest, metricsEmittingQueryRunnerInner, cachePopulator, cacheConfig);
    BySegmentQueryRunner<T> bySegmentQueryRunner = new BySegmentQueryRunner<>(segmentId, segmentInterval.getStart(), cachingQueryRunner);
    MetricsEmittingQueryRunner<T> metricsEmittingQueryRunnerOuter = new MetricsEmittingQueryRunner<>(emitter, toolChest, bySegmentQueryRunner, QueryMetrics::reportSegmentAndCacheTime, queryMetrics -> queryMetrics.segment(segmentIdString)).withWaitMeasuredFromNow();
    SpecificSegmentQueryRunner<T> specificSegmentQueryRunner = new SpecificSegmentQueryRunner<>(metricsEmittingQueryRunnerOuter, segmentSpec);
    PerSegmentOptimizingQueryRunner<T> perSegmentOptimizingQueryRunner = new PerSegmentOptimizingQueryRunner<>(specificSegmentQueryRunner, new PerSegmentQueryOptimizationContext(segmentDescriptor));
    return new SetAndVerifyContextQueryRunner<>(serverConfig, CPUTimeMetricQueryRunner.safeBuild(perSegmentOptimizingQueryRunner, toolChest, emitter, cpuTimeAccumulator, false));
}

Example 20 with SegmentDescriptor

use of org.apache.druid.query.SegmentDescriptor in project druid by druid-io.

the class ServerManager method getQueryRunnerForIntervals.

@Override
public <T> QueryRunner<T> getQueryRunnerForIntervals(Query<T> query, Iterable<Interval> intervals) {
    final DataSourceAnalysis analysis = DataSourceAnalysis.forDataSource(query.getDataSource());
    final VersionedIntervalTimeline<String, ReferenceCountingSegment> timeline;
    final Optional<VersionedIntervalTimeline<String, ReferenceCountingSegment>> maybeTimeline = segmentManager.getTimeline(analysis);
    if (maybeTimeline.isPresent()) {
        timeline = maybeTimeline.get();
    } else {
        // we must find.
        return new NoopQueryRunner<>();
    }
    FunctionalIterable<SegmentDescriptor> segmentDescriptors = FunctionalIterable.create(intervals).transformCat(timeline::lookup).transformCat(holder -> {
        if (holder == null) {
            return null;
        }
        return FunctionalIterable.create(holder.getObject()).transform(partitionChunk -> new SegmentDescriptor(holder.getInterval(), holder.getVersion(), partitionChunk.getChunkNumber()));
    });
    return getQueryRunnerForSegments(query, segmentDescriptors);
}