Examples with DimFilter org.apache.druid.query.filter.DimFilter used on opensource projects

Example 31 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project hive by apache.

the class DruidStorageHandlerUtils method toDruidFilter.

@Nullable
private static DimFilter toDruidFilter(ExprNodeDesc filterExpr, Configuration configuration, List<VirtualColumn> virtualColumns, boolean resolveDynamicValues) {
    if (filterExpr == null) {
        return null;
    }
    Class<? extends GenericUDF> genericUDFClass = getGenericUDFClassFromExprDesc(filterExpr);
    if (FunctionRegistry.isOpAnd(filterExpr)) {
        Iterator<ExprNodeDesc> iterator = filterExpr.getChildren().iterator();
        List<DimFilter> delegates = Lists.newArrayList();
        while (iterator.hasNext()) {
            DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues);
            if (filter != null) {
                delegates.add(filter);
            }
        }
        if (!delegates.isEmpty()) {
            return new AndDimFilter(delegates);
        }
    }
    if (FunctionRegistry.isOpOr(filterExpr)) {
        Iterator<ExprNodeDesc> iterator = filterExpr.getChildren().iterator();
        List<DimFilter> delegates = Lists.newArrayList();
        while (iterator.hasNext()) {
            DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues);
            if (filter != null) {
                delegates.add(filter);
            }
        }
        if (!delegates.isEmpty()) {
            return new OrDimFilter(delegates);
        }
    } else if (GenericUDFBetween.class == genericUDFClass) {
        List<ExprNodeDesc> child = filterExpr.getChildren();
        String col = extractColName(child.get(1), virtualColumns);
        if (col != null) {
            try {
                StringComparator comparator = stringTypeInfos.contains(child.get(1).getTypeInfo()) ? StringComparators.LEXICOGRAPHIC : StringComparators.NUMERIC;
                String lower = evaluate(child.get(2), configuration, resolveDynamicValues);
                String upper = evaluate(child.get(3), configuration, resolveDynamicValues);
                return new BoundDimFilter(col, lower, upper, false, false, null, null, comparator);
            } catch (HiveException e) {
                throw new RuntimeException(e);
            }
        }
    } else if (GenericUDFInBloomFilter.class == genericUDFClass) {
        List<ExprNodeDesc> child = filterExpr.getChildren();
        String col = extractColName(child.get(0), virtualColumns);
        if (col != null) {
            try {
                BloomKFilter bloomFilter = evaluateBloomFilter(child.get(1), configuration, resolveDynamicValues);
                return new BloomDimFilter(col, BloomKFilterHolder.fromBloomKFilter(bloomFilter), null);
            } catch (HiveException | IOException e) {
                throw new RuntimeException(e);
            }
        }
    }
    return null;
}

Example 32 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project druid by druid-io.

the class FilterPartitionBenchmark method readComplexOrFilter.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void readComplexOrFilter(Blackhole blackhole) {
    DimFilter dimFilter1 = new OrDimFilter(Arrays.asList(new SelectorDimFilter("dimSequential", "199", null), new AndDimFilter(Arrays.asList(new NoBitmapSelectorDimFilter("dimMultivalEnumerated2", "Corundum", null), new SelectorDimFilter("dimMultivalEnumerated", "Bar", null)))));
    DimFilter dimFilter2 = new OrDimFilter(Arrays.asList(new SelectorDimFilter("dimSequential", "299", null), new SelectorDimFilter("dimSequential", "399", null), new AndDimFilter(Arrays.asList(new NoBitmapSelectorDimFilter("dimMultivalEnumerated2", "Xylophone", null), new SelectorDimFilter("dimMultivalEnumerated", "Foo", null)))));
    DimFilter dimFilter3 = new OrDimFilter(Arrays.asList(dimFilter1, dimFilter2, new AndDimFilter(Arrays.asList(new NoBitmapSelectorDimFilter("dimMultivalEnumerated2", "Orange", null), new SelectorDimFilter("dimMultivalEnumerated", "World", null)))));
    StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
    Sequence<Cursor> cursors = makeCursors(sa, dimFilter3.toFilter());
    readCursors(cursors, blackhole);
}

Example 33 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project druid by druid-io.

the class MovingAverageIterableTest method testWithFilteredAggregation.

@Test
public void testWithFilteredAggregation() {
    Map<String, Object> event1 = new HashMap<>();
    Map<String, Object> event2 = new HashMap<>();
    List<DimensionSpec> ds = new ArrayList<>();
    ds.add(new DefaultDimensionSpec("gender", "gender"));
    event1.put("gender", "m");
    event1.put("pageViews", 10L);
    Row row1 = new MapBasedRow(JAN_1, event1);
    event2.put("gender", "m");
    event2.put("pageViews", 20L);
    Row row2 = new MapBasedRow(JAN_4, event2);
    Sequence<RowBucket> seq = Sequences.simple(Arrays.asList(new RowBucket(JAN_1, Collections.singletonList(row1)), new RowBucket(JAN_2, Collections.emptyList()), new RowBucket(JAN_3, Collections.emptyList()), new RowBucket(JAN_4, Collections.singletonList(row2))));
    AveragerFactory averagerfactory = new LongMeanAveragerFactory("movingAvgPageViews", 4, 1, "pageViews");
    AggregatorFactory aggregatorFactory = new LongSumAggregatorFactory("pageViews", "pageViews");
    DimFilter filter = new SelectorDimFilter("gender", "m", null);
    FilteredAggregatorFactory filteredAggregatorFactory = new FilteredAggregatorFactory(aggregatorFactory, filter);
    Iterator<Row> iter = new MovingAverageIterable(seq, ds, Collections.singletonList(averagerfactory), Collections.emptyList(), Collections.singletonList(filteredAggregatorFactory)).iterator();
    Assert.assertTrue(iter.hasNext());
    Row result = iter.next();
    Assert.assertEquals("m", (result.getDimension("gender")).get(0));
    Assert.assertEquals(2.5f, result.getMetric("movingAvgPageViews").floatValue(), 0.0f);
    Assert.assertTrue(iter.hasNext());
    result = iter.next();
    Assert.assertEquals("m", (result.getDimension("gender")).get(0));
    Assert.assertEquals(2.5f, result.getMetric("movingAvgPageViews").floatValue(), 0.0f);
    Assert.assertTrue(iter.hasNext());
    result = iter.next();
    Assert.assertEquals("m", (result.getDimension("gender")).get(0));
    Assert.assertEquals(2.5f, result.getMetric("movingAvgPageViews").floatValue(), 0.0f);
    Assert.assertTrue(iter.hasNext());
    result = iter.next();
    Assert.assertEquals("m", (result.getDimension("gender")).get(0));
    Assert.assertEquals(7.5f, result.getMetric("movingAvgPageViews").floatValue(), 0.0f);
    Assert.assertFalse(iter.hasNext());
}

Example 34 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project druid by druid-io.

the class IncrementalIndexReadBenchmark method readWithFilters.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void readWithFilters(Blackhole blackhole) {
    DimFilter filter = new OrDimFilter(Arrays.asList(new BoundDimFilter("dimSequential", "-1", "-1", true, true, null, null, StringComparators.ALPHANUMERIC), new JavaScriptDimFilter("dimSequential", "function(x) { return false }", null, JavaScriptConfig.getEnabledInstance()), new RegexDimFilter("dimSequential", "X", null), new SearchQueryDimFilter("dimSequential", new ContainsSearchQuerySpec("X", false), null), new InDimFilter("dimSequential", Collections.singletonList("X"), null)));
    IncrementalIndexStorageAdapter sa = new IncrementalIndexStorageAdapter(incIndex);
    Sequence<Cursor> cursors = makeCursors(sa, filter);
    Cursor cursor = cursors.limit(1).toList().get(0);
    List<DimensionSelector> selectors = new ArrayList<>();
    selectors.add(makeDimensionSelector(cursor, "dimSequential"));
    selectors.add(makeDimensionSelector(cursor, "dimZipf"));
    selectors.add(makeDimensionSelector(cursor, "dimUniform"));
    selectors.add(makeDimensionSelector(cursor, "dimSequentialHalfNull"));
    cursor.reset();
    while (!cursor.isDone()) {
        for (DimensionSelector selector : selectors) {
            IndexedInts row = selector.getRow();
            blackhole.consume(selector.lookupName(row.get(0)));
        }
        cursor.advance();
    }
}

Example 35 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project druid by druid-io.

the class SearchBenchmark method basicC.

private static SearchQueryBuilder basicC(final GeneratorSchemaInfo basicSchema) {
    final QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec(Collections.singletonList(basicSchema.getDataInterval()));
    final List<String> dimUniformFilterVals = new ArrayList<>();
    final int resultNum = (int) (100000 * 0.1);
    final int step = 100000 / resultNum;
    for (int i = 1; i < 100001 && dimUniformFilterVals.size() < resultNum; i += step) {
        dimUniformFilterVals.add(String.valueOf(i));
    }
    final String dimName = "dimUniform";
    final List<DimFilter> dimFilters = new ArrayList<>();
    dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, IdentityExtractionFn.getInstance()));
    dimFilters.add(new SelectorDimFilter(dimName, "3", StrlenExtractionFn.instance()));
    dimFilters.add(new BoundDimFilter(dimName, "100", "10000", true, true, true, new DimExtractionFn() {

        @Override
        public byte[] getCacheKey() {
            return new byte[] { 0xF };
        }

        @Override
        public String apply(String value) {
            return String.valueOf(Long.parseLong(value) + 1);
        }

        @Override
        public boolean preservesOrdering() {
            return false;
        }

        @Override
        public ExtractionType getExtractionType() {
            return ExtractionType.ONE_TO_ONE;
        }
    }, null));
    dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, new LowerExtractionFn(null)));
    dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, new UpperExtractionFn(null)));
    dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, new SubstringDimExtractionFn(1, 3)));
    return Druids.newSearchQueryBuilder().dataSource("blah").granularity(Granularities.ALL).intervals(intervalSpec).query("").dimensions(Collections.singletonList("dimUniform")).filters(new AndDimFilter(dimFilters));
}