Examples with BloomDimFilter org.apache.druid.query.filter.BloomDimFilter used on opensource projects

Example 6 with BloomDimFilter

use of org.apache.druid.query.filter.BloomDimFilter in project druid by druid-io.

the class BloomDimFilterSqlTest method testBloomFilterVirtualColumn.

@Test
public void testBloomFilterVirtualColumn() throws Exception {
    BloomKFilter filter = new BloomKFilter(1500);
    filter.addString("def-foo");
    byte[] bytes = BloomFilterSerializersModule.bloomKFilterToBytes(filter);
    String base64 = StringUtils.encodeBase64String(bytes);
    testQuery(StringUtils.format("SELECT COUNT(*) FROM druid.foo WHERE bloom_filter_test(concat(dim1, '-foo'), '%s')", base64), ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(querySegmentSpec(Filtration.eternity())).granularity(Granularities.ALL).virtualColumns(expressionVirtualColumn("v0", "concat(\"dim1\",'-foo')", ColumnType.STRING)).filters(new BloomDimFilter("v0", BloomKFilterHolder.fromBloomKFilter(filter), null)).aggregators(aggregators(new CountAggregatorFactory("a0"))).context(QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(new Object[] { 1L }));
}

Example 7 with BloomDimFilter

use of org.apache.druid.query.filter.BloomDimFilter in project druid by druid-io.

the class BloomFilterOperatorConversion method toDruidFilter.

@Nullable
@Override
public DimFilter toDruidFilter(final PlannerContext plannerContext, RowSignature rowSignature, @Nullable VirtualColumnRegistry virtualColumnRegistry, final RexNode rexNode) {
    final List<RexNode> operands = ((RexCall) rexNode).getOperands();
    final DruidExpression druidExpression = Expressions.toDruidExpression(plannerContext, rowSignature, operands.get(0));
    if (druidExpression == null) {
        return null;
    }
    String base64EncodedBloomKFilter = RexLiteral.stringValue(operands.get(1));
    final byte[] decoded = StringUtils.decodeBase64String(base64EncodedBloomKFilter);
    BloomKFilter filter;
    BloomKFilterHolder holder;
    try {
        filter = BloomFilterSerializersModule.bloomKFilterFromBytes(decoded);
        holder = BloomKFilterHolder.fromBloomKFilter(filter);
    } catch (IOException ioe) {
        throw new RuntimeException("Failed to deserialize bloom filter", ioe);
    }
    if (druidExpression.isSimpleExtraction()) {
        return new BloomDimFilter(druidExpression.getSimpleExtraction().getColumn(), holder, druidExpression.getSimpleExtraction().getExtractionFn(), null);
    } else if (virtualColumnRegistry != null) {
        String virtualColumnName = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(druidExpression, operands.get(0).getType());
        if (virtualColumnName == null) {
            return null;
        }
        return new BloomDimFilter(virtualColumnName, holder, null, null);
    } else {
        return null;
    }
}

Example 8 with BloomDimFilter

use of org.apache.druid.query.filter.BloomDimFilter in project hive by apache.

the class DruidStorageHandlerUtils method toDruidFilter.

@Nullable
private static DimFilter toDruidFilter(ExprNodeDesc filterExpr, Configuration configuration, List<VirtualColumn> virtualColumns, boolean resolveDynamicValues) {
    if (filterExpr == null) {
        return null;
    }
    Class<? extends GenericUDF> genericUDFClass = getGenericUDFClassFromExprDesc(filterExpr);
    if (FunctionRegistry.isOpAnd(filterExpr)) {
        Iterator<ExprNodeDesc> iterator = filterExpr.getChildren().iterator();
        List<DimFilter> delegates = Lists.newArrayList();
        while (iterator.hasNext()) {
            DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues);
            if (filter != null) {
                delegates.add(filter);
            }
        }
        if (!delegates.isEmpty()) {
            return new AndDimFilter(delegates);
        }
    }
    if (FunctionRegistry.isOpOr(filterExpr)) {
        Iterator<ExprNodeDesc> iterator = filterExpr.getChildren().iterator();
        List<DimFilter> delegates = Lists.newArrayList();
        while (iterator.hasNext()) {
            DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues);
            if (filter != null) {
                delegates.add(filter);
            }
        }
        if (!delegates.isEmpty()) {
            return new OrDimFilter(delegates);
        }
    } else if (GenericUDFBetween.class == genericUDFClass) {
        List<ExprNodeDesc> child = filterExpr.getChildren();
        String col = extractColName(child.get(1), virtualColumns);
        if (col != null) {
            try {
                StringComparator comparator = stringTypeInfos.contains(child.get(1).getTypeInfo()) ? StringComparators.LEXICOGRAPHIC : StringComparators.NUMERIC;
                String lower = evaluate(child.get(2), configuration, resolveDynamicValues);
                String upper = evaluate(child.get(3), configuration, resolveDynamicValues);
                return new BoundDimFilter(col, lower, upper, false, false, null, null, comparator);
            } catch (HiveException e) {
                throw new RuntimeException(e);
            }
        }
    } else if (GenericUDFInBloomFilter.class == genericUDFClass) {
        List<ExprNodeDesc> child = filterExpr.getChildren();
        String col = extractColName(child.get(0), virtualColumns);
        if (col != null) {
            try {
                BloomKFilter bloomFilter = evaluateBloomFilter(child.get(1), configuration, resolveDynamicValues);
                return new BloomDimFilter(col, BloomKFilterHolder.fromBloomKFilter(bloomFilter), null);
            } catch (HiveException | IOException e) {
                throw new RuntimeException(e);
            }
        }
    }
    return null;
}