Examples with NotDimFilter io.druid.query.filter.NotDimFilter used on opensource projects

Example 6 with NotDimFilter

use of io.druid.query.filter.NotDimFilter in project druid by druid-io.

the class TimeseriesQueryRunnerTest method testTimeSeriesWithFilteredAggInvertedNullValue.

@Test
public void testTimeSeriesWithFilteredAggInvertedNullValue() {
    TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource(QueryRunnerTestHelper.dataSource).granularity(QueryRunnerTestHelper.allGran).intervals(QueryRunnerTestHelper.firstToThird).aggregators(Lists.newArrayList(Iterables.concat(QueryRunnerTestHelper.commonAggregators, Lists.newArrayList(new FilteredAggregatorFactory(new CountAggregatorFactory("filteredAgg"), new NotDimFilter(Druids.newSelectorDimFilterBuilder().dimension(QueryRunnerTestHelper.marketDimension).value(null).build())))))).postAggregators(Arrays.<PostAggregator>asList(QueryRunnerTestHelper.addRowsIndexConstant)).descending(descending).build();
    Iterable<Result<TimeseriesResultValue>> actualResults = Sequences.toList(runner.run(query, CONTEXT), Lists.<Result<TimeseriesResultValue>>newArrayList());
    List<Result<TimeseriesResultValue>> expectedResults = Arrays.asList(new Result<>(new DateTime("2011-04-01"), new TimeseriesResultValue(ImmutableMap.<String, Object>of("filteredAgg", 26L, "addRowsIndexConstant", 12486.361190795898d, "index", 12459.361190795898d, "uniques", 9.019833517963864d, "rows", 26L))));
    assertExpectedResults(expectedResults, actualResults);
}

Example 7 with NotDimFilter

use of io.druid.query.filter.NotDimFilter in project druid by druid-io.

the class TimeseriesQueryRunnerTest method testTimeSeriesWithFilteredAggValueNotPresent.

@Test
public void testTimeSeriesWithFilteredAggValueNotPresent() {
    TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource(QueryRunnerTestHelper.dataSource).granularity(QueryRunnerTestHelper.allGran).intervals(QueryRunnerTestHelper.firstToThird).aggregators(Lists.newArrayList(Iterables.concat(QueryRunnerTestHelper.commonAggregators, Lists.newArrayList(new FilteredAggregatorFactory(new CountAggregatorFactory("filteredAgg"), new NotDimFilter(Druids.newSelectorDimFilterBuilder().dimension(QueryRunnerTestHelper.marketDimension).value("LolLol").build())))))).postAggregators(Arrays.<PostAggregator>asList(QueryRunnerTestHelper.addRowsIndexConstant)).descending(descending).build();
    Iterable<Result<TimeseriesResultValue>> actualResults = Sequences.toList(runner.run(query, CONTEXT), Lists.<Result<TimeseriesResultValue>>newArrayList());
    List<Result<TimeseriesResultValue>> expectedResults = Arrays.asList(new Result<>(new DateTime("2011-04-01"), new TimeseriesResultValue(ImmutableMap.<String, Object>of("filteredAgg", 26L, "addRowsIndexConstant", 12486.361190795898d, "index", 12459.361190795898d, "uniques", 9.019833517963864d, "rows", 26L))));
    assertExpectedResults(expectedResults, actualResults);
}

Example 8 with NotDimFilter

use of io.druid.query.filter.NotDimFilter in project druid by druid-io.

the class TimeseriesQueryRunnerTest method testTimeseriesWithInvertedFilterOnNonExistentDimension.

@Test
public void testTimeseriesWithInvertedFilterOnNonExistentDimension() {
    TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource(QueryRunnerTestHelper.dataSource).granularity(QueryRunnerTestHelper.dayGran).filters(new NotDimFilter(new SelectorDimFilter("bobby", "sally", null))).intervals(QueryRunnerTestHelper.firstToThird).aggregators(QueryRunnerTestHelper.commonAggregators).postAggregators(Arrays.<PostAggregator>asList(QueryRunnerTestHelper.addRowsIndexConstant)).descending(descending).build();
    List<Result<TimeseriesResultValue>> expectedResults = Arrays.asList(new Result<>(new DateTime("2011-04-01"), new TimeseriesResultValue(ImmutableMap.<String, Object>of("rows", 13L, "index", 6626.151596069336, "addRowsIndexConstant", 6640.151596069336, "uniques", QueryRunnerTestHelper.UNIQUES_9))), new Result<>(new DateTime("2011-04-02"), new TimeseriesResultValue(ImmutableMap.<String, Object>of("rows", 13L, "index", 5833.2095947265625, "addRowsIndexConstant", 5847.2095947265625, "uniques", QueryRunnerTestHelper.UNIQUES_9))));
    Iterable<Result<TimeseriesResultValue>> results = Sequences.toList(runner.run(query, new HashMap<String, Object>()), Lists.<Result<TimeseriesResultValue>>newArrayList());
    assertExpectedResults(expectedResults, results);
}

Example 9 with NotDimFilter

use of io.druid.query.filter.NotDimFilter in project druid by druid-io.

the class QuantileSqlAggregatorTest method testQuantileOnComplexColumn.

@Test
public void testQuantileOnComplexColumn() throws Exception {
    try (final DruidPlanner planner = plannerFactory.createPlanner(null)) {
        final String sql = "SELECT\n" + "APPROX_QUANTILE(hist_m1, 0.01),\n" + "APPROX_QUANTILE(hist_m1, 0.5, 50),\n" + "APPROX_QUANTILE(hist_m1, 0.98, 200),\n" + "APPROX_QUANTILE(hist_m1, 0.99),\n" + "APPROX_QUANTILE(hist_m1, 0.99) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE(hist_m1, 0.999) FILTER(WHERE dim1 <> 'abc'),\n" + "APPROX_QUANTILE(hist_m1, 0.999) FILTER(WHERE dim1 = 'abc')\n" + "FROM foo";
        final PlannerResult plannerResult = planner.plan(sql);
        // Verify results
        final List<Object[]> results = Sequences.toList(plannerResult.run(), new ArrayList<Object[]>());
        final List<Object[]> expectedResults = ImmutableList.of(new Object[] { 1.0, 3.0, 5.880000114440918, 5.940000057220459, 6.0, 4.994999885559082, 6.0 });
        Assert.assertEquals(expectedResults.size(), results.size());
        for (int i = 0; i < expectedResults.size(); i++) {
            Assert.assertArrayEquals(expectedResults.get(i), results.get(i));
        }
        // Verify query
        Assert.assertEquals(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).aggregators(ImmutableList.of(new ApproximateHistogramFoldingAggregatorFactory("a0:agg", "hist_m1", null, null, null, null), new ApproximateHistogramFoldingAggregatorFactory("a2:agg", "hist_m1", 200, null, null, null), new FilteredAggregatorFactory(new ApproximateHistogramFoldingAggregatorFactory("a4:agg", "hist_m1", null, null, null, null), new SelectorDimFilter("dim1", "abc", null)), new FilteredAggregatorFactory(new ApproximateHistogramFoldingAggregatorFactory("a5:agg", "hist_m1", null, null, null, null), new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))))).postAggregators(ImmutableList.<PostAggregator>of(new QuantilePostAggregator("a0", "a0:agg", 0.01f), new QuantilePostAggregator("a1", "a0:agg", 0.50f), new QuantilePostAggregator("a2", "a2:agg", 0.98f), new QuantilePostAggregator("a3", "a0:agg", 0.99f), new QuantilePostAggregator("a4", "a4:agg", 0.99f), new QuantilePostAggregator("a5", "a5:agg", 0.999f), new QuantilePostAggregator("a6", "a4:agg", 0.999f))).context(ImmutableMap.<String, Object>of("skipEmptyBuckets", true)).build(), Iterables.getOnlyElement(queryLogHook.getRecordedQueries()));
    }
}

Example 10 with NotDimFilter

use of io.druid.query.filter.NotDimFilter in project druid by druid-io.

the class Expressions method toLeafFilter.

/**
   * Translates "condition" to a Druid filter, assuming it does not contain any boolean expressions. Returns null
   * if we cannot translate the condition.
   *
   * @param plannerContext planner context
   * @param rowSignature   row signature of the dataSource to be filtered
   * @param expression     Calcite row expression
   */
private static DimFilter toLeafFilter(final DruidOperatorTable operatorTable, final PlannerContext plannerContext, final RowSignature rowSignature, final RexNode expression) {
    if (expression.isAlwaysTrue()) {
        return Filtration.matchEverything();
    } else if (expression.isAlwaysFalse()) {
        return Filtration.matchNothing();
    }
    final SqlKind kind = expression.getKind();
    if (kind == SqlKind.LIKE) {
        final List<RexNode> operands = ((RexCall) expression).getOperands();
        final RowExtraction rex = toRowExtraction(operatorTable, plannerContext, rowSignature.getRowOrder(), operands.get(0));
        if (rex == null || !rex.isFilterable(rowSignature)) {
            return null;
        }
        return new LikeDimFilter(rex.getColumn(), RexLiteral.stringValue(operands.get(1)), operands.size() > 2 ? RexLiteral.stringValue(operands.get(2)) : null, rex.getExtractionFn());
    } else if (kind == SqlKind.EQUALS || kind == SqlKind.NOT_EQUALS || kind == SqlKind.GREATER_THAN || kind == SqlKind.GREATER_THAN_OR_EQUAL || kind == SqlKind.LESS_THAN || kind == SqlKind.LESS_THAN_OR_EQUAL) {
        final List<RexNode> operands = ((RexCall) expression).getOperands();
        Preconditions.checkState(operands.size() == 2, "WTF?! Expected 2 operands, got[%,d]", operands.size());
        boolean flip = false;
        RexNode lhs = operands.get(0);
        RexNode rhs = operands.get(1);
        if (lhs.getKind() == SqlKind.LITERAL && rhs.getKind() != SqlKind.LITERAL) {
            // swap lhs, rhs
            RexNode x = lhs;
            lhs = rhs;
            rhs = x;
            flip = true;
        }
        // rhs must be a literal
        if (rhs.getKind() != SqlKind.LITERAL) {
            return null;
        }
        // lhs must be translatable to a RowExtraction to be filterable
        final RowExtraction rex = toRowExtraction(operatorTable, plannerContext, rowSignature.getRowOrder(), lhs);
        if (rex == null || !rex.isFilterable(rowSignature)) {
            return null;
        }
        final String column = rex.getColumn();
        final ExtractionFn extractionFn = rex.getExtractionFn();
        if (column.equals(Column.TIME_COLUMN_NAME) && extractionFn instanceof TimeFormatExtractionFn) {
            // Check if we can strip the extractionFn and convert the filter to a direct filter on __time.
            // This allows potential conversion to query-level "intervals" later on, which is ideal for Druid queries.
            final Granularity granularity = ExtractionFns.toQueryGranularity(extractionFn);
            if (granularity != null) {
                // lhs is FLOOR(__time TO granularity); rhs must be a timestamp
                final long rhsMillis = toMillisLiteral(rhs, plannerContext.getTimeZone());
                final Interval rhsInterval = granularity.bucket(new DateTime(rhsMillis));
                // Is rhs aligned on granularity boundaries?
                final boolean rhsAligned = rhsInterval.getStartMillis() == rhsMillis;
                // Create a BoundRefKey that strips the extractionFn and compares __time as a number.
                final BoundRefKey boundRefKey = new BoundRefKey(column, null, StringComparators.NUMERIC);
                if (kind == SqlKind.EQUALS) {
                    return rhsAligned ? Bounds.interval(boundRefKey, rhsInterval) : Filtration.matchNothing();
                } else if (kind == SqlKind.NOT_EQUALS) {
                    return rhsAligned ? new NotDimFilter(Bounds.interval(boundRefKey, rhsInterval)) : Filtration.matchEverything();
                } else if ((!flip && kind == SqlKind.GREATER_THAN) || (flip && kind == SqlKind.LESS_THAN)) {
                    return Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else if ((!flip && kind == SqlKind.GREATER_THAN_OR_EQUAL) || (flip && kind == SqlKind.LESS_THAN_OR_EQUAL)) {
                    return rhsAligned ? Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) : Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else if ((!flip && kind == SqlKind.LESS_THAN) || (flip && kind == SqlKind.GREATER_THAN)) {
                    return rhsAligned ? Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) : Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else if ((!flip && kind == SqlKind.LESS_THAN_OR_EQUAL) || (flip && kind == SqlKind.GREATER_THAN_OR_EQUAL)) {
                    return Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else {
                    throw new IllegalStateException("WTF?! Shouldn't have got here...");
                }
            }
        }
        final String val;
        final RexLiteral rhsLiteral = (RexLiteral) rhs;
        if (SqlTypeName.NUMERIC_TYPES.contains(rhsLiteral.getTypeName())) {
            val = String.valueOf(RexLiteral.value(rhsLiteral));
        } else if (SqlTypeName.CHAR_TYPES.contains(rhsLiteral.getTypeName())) {
            val = String.valueOf(RexLiteral.stringValue(rhsLiteral));
        } else if (SqlTypeName.TIMESTAMP == rhsLiteral.getTypeName() || SqlTypeName.DATE == rhsLiteral.getTypeName()) {
            val = String.valueOf(toMillisLiteral(rhsLiteral, plannerContext.getTimeZone()));
        } else {
            // Don't know how to filter on this kind of literal.
            return null;
        }
        // Numeric lhs needs a numeric comparison.
        final boolean lhsIsNumeric = SqlTypeName.NUMERIC_TYPES.contains(lhs.getType().getSqlTypeName()) || SqlTypeName.TIMESTAMP == lhs.getType().getSqlTypeName() || SqlTypeName.DATE == lhs.getType().getSqlTypeName();
        final StringComparator comparator = lhsIsNumeric ? StringComparators.NUMERIC : StringComparators.LEXICOGRAPHIC;
        final BoundRefKey boundRefKey = new BoundRefKey(column, extractionFn, comparator);
        final DimFilter filter;
        // Always use BoundDimFilters, to simplify filter optimization later (it helps to remember the comparator).
        if (kind == SqlKind.EQUALS) {
            filter = Bounds.equalTo(boundRefKey, val);
        } else if (kind == SqlKind.NOT_EQUALS) {
            filter = new NotDimFilter(Bounds.equalTo(boundRefKey, val));
        } else if ((!flip && kind == SqlKind.GREATER_THAN) || (flip && kind == SqlKind.LESS_THAN)) {
            filter = Bounds.greaterThan(boundRefKey, val);
        } else if ((!flip && kind == SqlKind.GREATER_THAN_OR_EQUAL) || (flip && kind == SqlKind.LESS_THAN_OR_EQUAL)) {
            filter = Bounds.greaterThanOrEqualTo(boundRefKey, val);
        } else if ((!flip && kind == SqlKind.LESS_THAN) || (flip && kind == SqlKind.GREATER_THAN)) {
            filter = Bounds.lessThan(boundRefKey, val);
        } else if ((!flip && kind == SqlKind.LESS_THAN_OR_EQUAL) || (flip && kind == SqlKind.GREATER_THAN_OR_EQUAL)) {
            filter = Bounds.lessThanOrEqualTo(boundRefKey, val);
        } else {
            throw new IllegalStateException("WTF?! Shouldn't have got here...");
        }
        return filter;
    } else {
        return null;
    }
}