Examples with BloomKFilter org.apache.druid.query.filter.BloomKFilter used on opensource projects

Example 11 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterAggregatorTest method testMergeValues.

@Test
public void testMergeValues() throws IOException {
    final TestBloomFilterBufferColumnSelector mergeDim = new TestBloomFilterBufferColumnSelector(ImmutableList.of(ByteBuffer.wrap(BloomFilterSerializersModule.bloomKFilterToBytes(filter1)), ByteBuffer.wrap(BloomFilterSerializersModule.bloomKFilterToBytes(filter2))));
    BloomFilterMergeAggregator mergeAggregator = new BloomFilterMergeAggregator(mergeDim, MAX_NUM_VALUES, true);
    for (int i = 0; i < 2; ++i) {
        aggregateColumn(Collections.singletonList(mergeDim), mergeAggregator);
    }
    BloomKFilter merged = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.getCombiningFactory().finalizeComputation(mergeAggregator.get()));
    String serialized = filterToString(merged);
    Assert.assertEquals(serializedCombinedFilter, serialized);
}

Example 12 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterAggregatorTest method testAggregateValues.

@Test
public void testAggregateValues() throws IOException {
    DimensionSelector dimSelector = new CardinalityAggregatorTest.TestDimensionSelector(VALUES1, null);
    StringBloomFilterAggregator agg = new StringBloomFilterAggregator(dimSelector, MAX_NUM_VALUES, true);
    for (int i = 0; i < VALUES1.size(); ++i) {
        aggregateDimension(Collections.singletonList(dimSelector), agg);
    }
    BloomKFilter bloomKFilter = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.finalizeComputation(agg.get()));
    String serialized = filterToString(bloomKFilter);
    Assert.assertEquals(serializedFilter1, serialized);
}

Example 13 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterSqlAggregatorTest method testBloomFilterAgg.

@Test
public void testBloomFilterAgg() throws Exception {
    cannotVectorize();
    BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
    for (InputRow row : CalciteTests.ROWS1_WITH_NUMERIC_DIMS) {
        String raw = NullHandling.emptyToNullIfNeeded((String) row.getRaw("dim1"));
        if (raw == null) {
            expected1.addBytes(null, 0, 0);
        } else {
            expected1.addString(raw);
        }
    }
    testQuery("SELECT\n" + "BLOOM_FILTER(dim1, 1000)\n" + "FROM numfoo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE3).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).aggregators(ImmutableList.of(new BloomFilterAggregatorFactory("a0:agg", new DefaultDimensionSpec("dim1", "a0:dim1"), TEST_NUM_ENTRIES))).context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(new Object[] { CalciteTests.getJsonMapper().writeValueAsString(expected1) }));
}

Example 14 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterSqlAggregatorTest method testBloomFilterAggLongVirtualColumn.

@Test
public void testBloomFilterAggLongVirtualColumn() throws Exception {
    cannotVectorize();
    BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
    for (InputRow row : CalciteTests.ROWS1_WITH_NUMERIC_DIMS) {
        Object raw = row.getRaw("l1");
        if (raw == null) {
            if (NullHandling.replaceWithDefault()) {
                expected1.addLong(NullHandling.defaultLongValue());
            } else {
                expected1.addBytes(null, 0, 0);
            }
        } else {
            expected1.addLong(2 * ((Number) raw).longValue());
        }
    }
    testQuery("SELECT\n" + "BLOOM_FILTER(l1 * 2, 1000)\n" + "FROM numfoo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE3).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "(\"l1\" * 2)", ColumnType.LONG, TestExprMacroTable.INSTANCE)).aggregators(ImmutableList.of(new BloomFilterAggregatorFactory("a0:agg", new DefaultDimensionSpec("v0", "a0:v0"), TEST_NUM_ENTRIES))).context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(new Object[] { CalciteTests.getJsonMapper().writeValueAsString(expected1) }));
}

Example 15 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterSqlAggregatorTest method testBloomFilterAggLong.

@Test
public void testBloomFilterAggLong() throws Exception {
    cannotVectorize();
    BloomKFilter expected3 = new BloomKFilter(TEST_NUM_ENTRIES);
    for (InputRow row : CalciteTests.ROWS1_WITH_NUMERIC_DIMS) {
        Object raw = row.getRaw("l1");
        if (raw == null) {
            if (NullHandling.replaceWithDefault()) {
                expected3.addLong(NullHandling.defaultLongValue());
            } else {
                expected3.addBytes(null, 0, 0);
            }
        } else {
            expected3.addLong(((Number) raw).longValue());
        }
    }
    testQuery("SELECT\n" + "BLOOM_FILTER(l1, 1000)\n" + "FROM numfoo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE3).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).aggregators(ImmutableList.of(new BloomFilterAggregatorFactory("a0:agg", new DefaultDimensionSpec("l1", "a0:l1", ColumnType.LONG), TEST_NUM_ENTRIES))).context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(new Object[] { CalciteTests.getJsonMapper().writeValueAsString(expected3) }));
}