use of org.apache.druid.query.spec.MultipleSpecificSegmentSpec in project druid by druid-io.
the class Queries method withSpecificSegments.
/**
* Rewrite "query" to refer to some specific segment descriptors.
*
* The dataSource for "query" must be based on a single table for this operation to be valid. Otherwise, this
* function will throw an exception.
*
* Unlike the seemingly-similar {@code query.withQuerySegmentSpec(new MultipleSpecificSegmentSpec(descriptors))},
* this this method will walk down subqueries found within the query datasource, if any, and modify the lowest-level
* subquery. The effect is that
* {@code DataSourceAnalysis.forDataSource(query.getDataSource()).getBaseQuerySegmentSpec()} is guaranteed to return
* either {@code new MultipleSpecificSegmentSpec(descriptors)} or empty.
*
* Because {@link BaseQuery#getRunner} is implemented using {@link DataSourceAnalysis#getBaseQuerySegmentSpec}, this
* method will cause the runner to be a specific-segments runner.
*/
public static <T> Query<T> withSpecificSegments(final Query<T> query, final List<SegmentDescriptor> descriptors) {
final Query<T> retVal;
if (query.getDataSource() instanceof QueryDataSource) {
final Query<?> subQuery = ((QueryDataSource) query.getDataSource()).getQuery();
retVal = query.withDataSource(new QueryDataSource(withSpecificSegments(subQuery, descriptors)));
} else {
retVal = query.withQuerySegmentSpec(new MultipleSpecificSegmentSpec(descriptors));
}
// Verify preconditions and invariants, just in case.
final DataSourceAnalysis analysis = DataSourceAnalysis.forDataSource(retVal.getDataSource());
// Sanity check: query must be based on a single table.
if (!analysis.getBaseTableDataSource().isPresent()) {
throw new ISE("Unable to apply specific segments to non-table-based dataSource[%s]", query.getDataSource());
}
if (analysis.getBaseQuerySegmentSpec().isPresent() && !analysis.getBaseQuerySegmentSpec().get().equals(new MultipleSpecificSegmentSpec(descriptors))) {
// If you see the error message below, it's a bug in either this function or in DataSourceAnalysis.
throw new ISE("Unable to apply specific segments to query with dataSource[%s]", query.getDataSource());
}
return retVal;
}
use of org.apache.druid.query.spec.MultipleSpecificSegmentSpec in project druid by druid-io.
the class CachingClusteredClientTest method testHashBasedPruningQueryContextEnabledWithPartitionFunctionAndPartitionDimensionsDoSegmentPruning.
@Test
public void testHashBasedPruningQueryContextEnabledWithPartitionFunctionAndPartitionDimensionsDoSegmentPruning() {
DimFilter filter = new AndDimFilter(new SelectorDimFilter("dim1", "a", null), new BoundDimFilter("dim2", "e", "zzz", true, true, false, null, StringComparators.LEXICOGRAPHIC), // Equivalent filter of dim3 below is InDimFilter("dim3", Arrays.asList("c"), null)
new AndDimFilter(new InDimFilter("dim3", Arrays.asList("a", "c", "e", "g"), null), new BoundDimFilter("dim3", "aaa", "ddd", false, false, false, null, StringComparators.LEXICOGRAPHIC)));
final Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder().dataSource(DATA_SOURCE).filters(filter).granularity(GRANULARITY).intervals(SEG_SPEC).context(CONTEXT).intervals("2011-01-05/2011-01-10").aggregators(RENAMED_AGGS).postAggregators(RENAMED_POST_AGGS).randomQueryId();
TimeseriesQuery query = builder.build();
QueryRunner runner = new FinalizeResultsQueryRunner(getDefaultQueryRunner(), new TimeseriesQueryQueryToolChest());
final Interval interval1 = Intervals.of("2011-01-06/2011-01-07");
final Interval interval2 = Intervals.of("2011-01-07/2011-01-08");
final Interval interval3 = Intervals.of("2011-01-08/2011-01-09");
final DruidServer lastServer = servers[random.nextInt(servers.length)];
List<String> partitionDimensions1 = ImmutableList.of("dim1");
ServerSelector selector1 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 0, 6);
ServerSelector selector2 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 1, 6);
ServerSelector selector3 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 2, 6);
ServerSelector selector4 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 3, 6);
ServerSelector selector5 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 4, 6);
ServerSelector selector6 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 5, 6);
List<String> partitionDimensions2 = ImmutableList.of("dim2");
ServerSelector selector7 = makeMockHashBasedSelector(lastServer, partitionDimensions2, HashPartitionFunction.MURMUR3_32_ABS, 0, 3);
ServerSelector selector8 = makeMockHashBasedSelector(lastServer, partitionDimensions2, HashPartitionFunction.MURMUR3_32_ABS, 1, 3);
ServerSelector selector9 = makeMockHashBasedSelector(lastServer, partitionDimensions2, HashPartitionFunction.MURMUR3_32_ABS, 2, 3);
List<String> partitionDimensions3 = ImmutableList.of("dim1", "dim3");
ServerSelector selector10 = makeMockHashBasedSelector(lastServer, partitionDimensions3, HashPartitionFunction.MURMUR3_32_ABS, 0, 4);
ServerSelector selector11 = makeMockHashBasedSelector(lastServer, partitionDimensions3, HashPartitionFunction.MURMUR3_32_ABS, 1, 4);
ServerSelector selector12 = makeMockHashBasedSelector(lastServer, partitionDimensions3, HashPartitionFunction.MURMUR3_32_ABS, 2, 4);
ServerSelector selector13 = makeMockHashBasedSelector(lastServer, partitionDimensions3, HashPartitionFunction.MURMUR3_32_ABS, 3, 4);
timeline.add(interval1, "v", new NumberedPartitionChunk<>(0, 6, selector1));
timeline.add(interval1, "v", new NumberedPartitionChunk<>(1, 6, selector2));
timeline.add(interval1, "v", new NumberedPartitionChunk<>(2, 6, selector3));
timeline.add(interval1, "v", new NumberedPartitionChunk<>(3, 6, selector4));
timeline.add(interval1, "v", new NumberedPartitionChunk<>(4, 6, selector5));
timeline.add(interval1, "v", new NumberedPartitionChunk<>(5, 6, selector6));
timeline.add(interval2, "v", new NumberedPartitionChunk<>(0, 3, selector7));
timeline.add(interval2, "v", new NumberedPartitionChunk<>(1, 3, selector8));
timeline.add(interval2, "v", new NumberedPartitionChunk<>(2, 3, selector9));
timeline.add(interval3, "v", new NumberedPartitionChunk<>(0, 3, selector10));
timeline.add(interval3, "v", new NumberedPartitionChunk<>(1, 3, selector11));
timeline.add(interval3, "v", new NumberedPartitionChunk<>(2, 3, selector12));
timeline.add(interval3, "v", new NumberedPartitionChunk<>(2, 3, selector13));
final Capture<QueryPlus> capture = Capture.newInstance();
final Capture<ResponseContext> contextCap = Capture.newInstance();
QueryRunner mockRunner = EasyMock.createNiceMock(QueryRunner.class);
EasyMock.expect(mockRunner.run(EasyMock.capture(capture), EasyMock.capture(contextCap))).andReturn(Sequences.empty()).anyTimes();
EasyMock.expect(serverView.getQueryRunner(lastServer)).andReturn(mockRunner).anyTimes();
EasyMock.replay(serverView);
EasyMock.replay(mockRunner);
List<SegmentDescriptor> expcetedDescriptors = new ArrayList<>();
// Narrow down to 1 chunk
expcetedDescriptors.add(new SegmentDescriptor(interval1, "v", 3));
// Can't filter out any chunks
expcetedDescriptors.add(new SegmentDescriptor(interval2, "v", 0));
expcetedDescriptors.add(new SegmentDescriptor(interval2, "v", 1));
expcetedDescriptors.add(new SegmentDescriptor(interval2, "v", 2));
// Narrow down to 1 chunk
expcetedDescriptors.add(new SegmentDescriptor(interval3, "v", 2));
MultipleSpecificSegmentSpec expected = new MultipleSpecificSegmentSpec(expcetedDescriptors);
runner.run(QueryPlus.wrap(query)).toList();
Assert.assertEquals(expected, ((TimeseriesQuery) capture.getValue().getQuery()).getQuerySegmentSpec());
}
use of org.apache.druid.query.spec.MultipleSpecificSegmentSpec in project druid by druid-io.
the class DruidSchema method runSegmentMetadataQuery.
/**
* Execute a SegmentMetadata query and return a {@link Sequence} of {@link SegmentAnalysis}.
*
* @param segments Iterable of {@link SegmentId} objects that are subject of the SegmentMetadata query.
* @return {@link Sequence} of {@link SegmentAnalysis} objects
*/
@VisibleForTesting
protected Sequence<SegmentAnalysis> runSegmentMetadataQuery(final Iterable<SegmentId> segments) {
// Sanity check: getOnlyElement of a set, to ensure all segments have the same dataSource.
final String dataSource = Iterables.getOnlyElement(StreamSupport.stream(segments.spliterator(), false).map(SegmentId::getDataSource).collect(Collectors.toSet()));
final MultipleSpecificSegmentSpec querySegmentSpec = new MultipleSpecificSegmentSpec(StreamSupport.stream(segments.spliterator(), false).map(SegmentId::toDescriptor).collect(Collectors.toList()));
final SegmentMetadataQuery segmentMetadataQuery = new SegmentMetadataQuery(new TableDataSource(dataSource), querySegmentSpec, new AllColumnIncluderator(), false, brokerInternalQueryConfig.getContext(), EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), false, false);
return queryLifecycleFactory.factorize().runSimple(segmentMetadataQuery, escalator.createEscalatedAuthenticationResult(), Access.OK);
}
Aggregations