Examples with DataSource org.apache.druid.query.DataSource used on opensource projects

Example 26 with DataSource

use of org.apache.druid.query.DataSource in project druid by druid-io.

the class SegmentManagerBroadcastJoinIndexedTableTest method testLoadIndexedTable.

@Test
public void testLoadIndexedTable() throws IOException, SegmentLoadingException {
    final DataSource dataSource = new GlobalTableDataSource(TABLE_NAME);
    Assert.assertFalse(joinableFactory.isDirectlyJoinable(dataSource));
    final String version = DateTimes.nowUtc().toString();
    IncrementalIndex data = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv");
    final String interval = "2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z";
    DataSegment segment = createSegment(data, interval, version);
    Assert.assertTrue(segmentManager.loadSegment(segment, false, SegmentLazyLoadFailCallback.NOOP));
    Assert.assertTrue(joinableFactory.isDirectlyJoinable(dataSource));
    Optional<Joinable> maybeJoinable = makeJoinable(dataSource);
    Assert.assertTrue(maybeJoinable.isPresent());
    Joinable joinable = maybeJoinable.get();
    // cardinality currently tied to number of rows,
    Assert.assertEquals(1210, joinable.getCardinality("market"));
    Assert.assertEquals(1210, joinable.getCardinality("placement"));
    Assert.assertEquals(Optional.of(ImmutableSet.of("preferred")), joinable.getCorrelatedColumnValues("market", "spot", "placement", Long.MAX_VALUE, false));
    Optional<byte[]> bytes = joinableFactory.computeJoinCacheKey(dataSource, JOIN_CONDITION_ANALYSIS);
    Assert.assertTrue(bytes.isPresent());
    assertSegmentIdEquals(segment.getId(), bytes.get());
    // dropping the segment should make the table no longer available
    segmentManager.dropSegment(segment);
    maybeJoinable = makeJoinable(dataSource);
    Assert.assertFalse(maybeJoinable.isPresent());
    bytes = joinableFactory.computeJoinCacheKey(dataSource, JOIN_CONDITION_ANALYSIS);
    Assert.assertFalse(bytes.isPresent());
}

Example 27 with DataSource

use of org.apache.druid.query.DataSource in project druid by druid-io.

the class DruidJoinQueryRel method toDruidQuery.

@Override
public DruidQuery toDruidQuery(final boolean finalizeAggregations) {
    final DruidRel<?> leftDruidRel = (DruidRel<?>) left;
    final DruidQuery leftQuery = Preconditions.checkNotNull((leftDruidRel).toDruidQuery(false), "leftQuery");
    final RowSignature leftSignature = leftQuery.getOutputRowSignature();
    final DataSource leftDataSource;
    final DruidRel<?> rightDruidRel = (DruidRel<?>) right;
    final DruidQuery rightQuery = Preconditions.checkNotNull(rightDruidRel.toDruidQuery(false), "rightQuery");
    final RowSignature rightSignature = rightQuery.getOutputRowSignature();
    final DataSource rightDataSource;
    if (computeLeftRequiresSubquery(leftDruidRel)) {
        leftDataSource = new QueryDataSource(leftQuery.getQuery());
        if (leftFilter != null) {
            throw new ISE("Filter on left table is supposed to be null if left child is a query source");
        }
    } else {
        leftDataSource = leftQuery.getDataSource();
    }
    if (computeRightRequiresSubquery(rightDruidRel)) {
        rightDataSource = new QueryDataSource(rightQuery.getQuery());
    } else {
        rightDataSource = rightQuery.getDataSource();
    }
    final Pair<String, RowSignature> prefixSignaturePair = computeJoinRowSignature(leftSignature, rightSignature);
    VirtualColumnRegistry virtualColumnRegistry = VirtualColumnRegistry.create(prefixSignaturePair.rhs, getPlannerContext().getExprMacroTable());
    getPlannerContext().setJoinExpressionVirtualColumnRegistry(virtualColumnRegistry);
    // Generate the condition for this join as a Druid expression.
    final DruidExpression condition = Expressions.toDruidExpression(getPlannerContext(), prefixSignaturePair.rhs, joinRel.getCondition());
    // Unsetting it to avoid any VC Registry leaks incase there are multiple druid quries for the SQL
    // It should be fixed soon with changes in interface for SqlOperatorConversion and Expressions bridge class
    getPlannerContext().setJoinExpressionVirtualColumnRegistry(null);
    // quiets static code analysis.
    if (condition == null) {
        throw new CannotBuildQueryException(joinRel, joinRel.getCondition());
    }
    return partialQuery.build(JoinDataSource.create(leftDataSource, rightDataSource, prefixSignaturePair.lhs, condition.getExpression(), toDruidJoinType(joinRel.getJoinType()), getDimFilter(getPlannerContext(), leftSignature, leftFilter), getPlannerContext().getExprMacroTable()), prefixSignaturePair.rhs, getPlannerContext(), getCluster().getRexBuilder(), finalizeAggregations, virtualColumnRegistry);
}

Example 28 with DataSource

use of org.apache.druid.query.DataSource in project druid by druid-io.

the class DruidQuery method toScanQuery.

/**
 * Return this query as a Scan query, or null if this query is not compatible with Scan.
 *
 * @return query or null
 */
@Nullable
private ScanQuery toScanQuery(final QueryFeatureInspector queryFeatureInspector) {
    if (grouping != null) {
        // Scan cannot GROUP BY.
        return null;
    }
    if (outputRowSignature.size() == 0) {
        // Should never do a scan query without any columns that we're interested in. This is probably a planner bug.
        throw new ISE("Cannot convert to Scan query without any columns.");
    }
    final Pair<DataSource, Filtration> dataSourceFiltrationPair = getFiltration(dataSource, filter, virtualColumnRegistry);
    final DataSource newDataSource = dataSourceFiltrationPair.lhs;
    final Filtration filtration = dataSourceFiltrationPair.rhs;
    final List<ScanQuery.OrderBy> orderByColumns;
    long scanOffset = 0L;
    long scanLimit = 0L;
    if (sorting != null) {
        scanOffset = sorting.getOffsetLimit().getOffset();
        if (sorting.getOffsetLimit().hasLimit()) {
            final long limit = sorting.getOffsetLimit().getLimit();
            if (limit == 0) {
                // Can't handle zero limit (the Scan query engine would treat it as unlimited).
                return null;
            }
            scanLimit = limit;
        }
        orderByColumns = sorting.getOrderBys().stream().map(orderBy -> new ScanQuery.OrderBy(orderBy.getDimension(), orderBy.getDirection() == OrderByColumnSpec.Direction.DESCENDING ? ScanQuery.Order.DESCENDING : ScanQuery.Order.ASCENDING)).collect(Collectors.toList());
    } else {
        orderByColumns = Collections.emptyList();
    }
    if (!queryFeatureInspector.feature(QueryFeature.SCAN_CAN_ORDER_BY_NON_TIME) && (orderByColumns.size() > 1 || orderByColumns.stream().anyMatch(orderBy -> !orderBy.getColumnName().equals(ColumnHolder.TIME_COLUMN_NAME)))) {
        // Cannot handle this ordering.
        // Scan cannot ORDER BY non-time columns.
        plannerContext.setPlanningError("SQL query requires order by non-time column %s that is not supported.", orderByColumns);
        return null;
    }
    // Compute the list of columns to select, sorted and deduped.
    final SortedSet<String> scanColumns = new TreeSet<>(outputRowSignature.getColumnNames());
    orderByColumns.forEach(column -> scanColumns.add(column.getColumnName()));
    return new ScanQuery(newDataSource, filtration.getQuerySegmentSpec(), getVirtualColumns(true), ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST, 0, scanOffset, scanLimit, null, orderByColumns, filtration.getDimFilter(), ImmutableList.copyOf(scanColumns), false, ImmutableSortedMap.copyOf(plannerContext.getQueryContext()));
}

Example 29 with DataSource

use of org.apache.druid.query.DataSource in project druid by druid-io.

the class DruidQuery method toTimeseriesQuery.

/**
 * Return this query as a Timeseries query, or null if this query is not compatible with Timeseries.
 *
 * @return query
 */
@Nullable
private TimeseriesQuery toTimeseriesQuery(final QueryFeatureInspector queryFeatureInspector) {
    if (!queryFeatureInspector.feature(QueryFeature.CAN_RUN_TIMESERIES) || grouping == null || grouping.getSubtotals().hasEffect(grouping.getDimensionSpecs()) || grouping.getHavingFilter() != null) {
        return null;
    }
    if (sorting != null && sorting.getOffsetLimit().hasOffset()) {
        // Timeseries cannot handle offsets.
        return null;
    }
    final Granularity queryGranularity;
    final boolean descending;
    int timeseriesLimit = 0;
    final Map<String, Object> theContext = new HashMap<>();
    if (grouping.getDimensions().isEmpty()) {
        queryGranularity = Granularities.ALL;
        descending = false;
    } else if (grouping.getDimensions().size() == 1) {
        final DimensionExpression dimensionExpression = Iterables.getOnlyElement(grouping.getDimensions());
        queryGranularity = Expressions.toQueryGranularity(dimensionExpression.getDruidExpression(), plannerContext.getExprMacroTable());
        if (queryGranularity == null) {
            // Timeseries only applies if the single dimension is granular __time.
            return null;
        }
        theContext.put(TimeseriesQuery.CTX_TIMESTAMP_RESULT_FIELD, Iterables.getOnlyElement(grouping.getDimensions()).toDimensionSpec().getOutputName());
        if (sorting != null) {
            if (sorting.getOffsetLimit().hasLimit()) {
                final long limit = sorting.getOffsetLimit().getLimit();
                if (limit == 0) {
                    // Can't handle zero limit (the Timeseries query engine would treat it as unlimited).
                    return null;
                }
                timeseriesLimit = Ints.checkedCast(limit);
            }
            switch(sorting.getTimeSortKind(dimensionExpression.getOutputName())) {
                case UNORDERED:
                case TIME_ASCENDING:
                    descending = false;
                    break;
                case TIME_DESCENDING:
                    descending = true;
                    break;
                default:
                    // Sorting on a metric, maybe. Timeseries cannot handle.
                    return null;
            }
        } else {
            // No limitSpec.
            descending = false;
        }
    } else {
        // More than one dimension, timeseries cannot handle.
        return null;
    }
    // was originally a groupBy query, but with the grouping dimensions removed away in Grouping#applyProject
    if (!Granularities.ALL.equals(queryGranularity) || grouping.hasGroupingDimensionsDropped()) {
        theContext.put(TimeseriesQuery.SKIP_EMPTY_BUCKETS, true);
    }
    theContext.putAll(plannerContext.getQueryContext());
    final Pair<DataSource, Filtration> dataSourceFiltrationPair = getFiltration(dataSource, filter, virtualColumnRegistry);
    final DataSource newDataSource = dataSourceFiltrationPair.lhs;
    final Filtration filtration = dataSourceFiltrationPair.rhs;
    final List<PostAggregator> postAggregators = new ArrayList<>(grouping.getPostAggregators());
    if (sorting != null && sorting.getProjection() != null) {
        postAggregators.addAll(sorting.getProjection().getPostAggregators());
    }
    return new TimeseriesQuery(newDataSource, filtration.getQuerySegmentSpec(), descending, getVirtualColumns(false), filtration.getDimFilter(), queryGranularity, grouping.getAggregatorFactories(), postAggregators, timeseriesLimit, ImmutableSortedMap.copyOf(theContext));
}

Example 30 with DataSource

use of org.apache.druid.query.DataSource in project druid by druid-io.

the class DruidQueryTest method test_filtration_intervalInQueryFilter.

@Test
public void test_filtration_intervalInQueryFilter() {
    DataSource dataSource = new TableDataSource("test");
    Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(dataSource, filterWithInterval, VirtualColumnRegistry.create(RowSignature.empty(), TestExprMacroTable.INSTANCE));
    verify(pair, dataSource, selectorFilter, Intervals.utc(100, 200));
}