Examples with BloomKFilter org.apache.druid.query.filter.BloomKFilter used on opensource projects

Example 36 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterAggregatorTest method testBufferAggregateStringValues.

@Test
public void testBufferAggregateStringValues() throws IOException {
    DimensionSelector dimSelector = new CardinalityAggregatorTest.TestDimensionSelector(VALUES2, null);
    StringBloomFilterAggregator agg = new StringBloomFilterAggregator(dimSelector, MAX_NUM_VALUES, true);
    int maxSize = valueAggregatorFactory.getMaxIntermediateSizeWithNulls();
    ByteBuffer buf = ByteBuffer.allocate(maxSize + 64);
    int pos = 10;
    buf.limit(pos + maxSize);
    agg.init(buf, pos);
    for (int i = 0; i < VALUES2.size(); ++i) {
        bufferAggregateDimension(Collections.singletonList(dimSelector), agg, buf, pos);
    }
    BloomKFilter bloomKFilter = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)));
    String serialized = filterToString(bloomKFilter);
    Assert.assertEquals(serializedFilter2, serialized);
}

Example 37 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterAggregatorTest method testBuferMergeValues.

@Test
public void testBuferMergeValues() throws IOException {
    final TestBloomFilterBufferColumnSelector mergeDim = new TestBloomFilterBufferColumnSelector(ImmutableList.of(ByteBuffer.wrap(BloomFilterSerializersModule.bloomKFilterToBytes(filter1)), ByteBuffer.wrap(BloomFilterSerializersModule.bloomKFilterToBytes(filter2))));
    BloomFilterMergeAggregator mergeAggregator = new BloomFilterMergeAggregator(mergeDim, MAX_NUM_VALUES, false);
    int maxSize = valueAggregatorFactory.getCombiningFactory().getMaxIntermediateSizeWithNulls();
    ByteBuffer buf = ByteBuffer.allocate(maxSize + 64);
    int pos = 10;
    buf.limit(pos + maxSize);
    mergeAggregator.init(buf, pos);
    for (int i = 0; i < 2; ++i) {
        bufferAggregateColumn(Collections.singletonList(mergeDim), mergeAggregator, buf, pos);
    }
    BloomKFilter merged = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.getCombiningFactory().finalizeComputation(mergeAggregator.get(buf, pos)));
    String serialized = filterToString(merged);
    Assert.assertEquals(serializedCombinedFilter, serialized);
}

Example 38 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterAggregatorTest method testBufferAggregateLongValues.

@Test
public void testBufferAggregateLongValues() throws IOException {
    TestLongColumnSelector selector = new TestLongColumnSelector(Arrays.asList(LONG_VALUES1));
    LongBloomFilterAggregator agg = new LongBloomFilterAggregator(selector, MAX_NUM_VALUES, true);
    int maxSize = valueAggregatorFactory.getMaxIntermediateSizeWithNulls();
    ByteBuffer buf = ByteBuffer.allocate(maxSize + 64);
    int pos = 10;
    buf.limit(pos + maxSize);
    agg.init(buf, pos);
    IntStream.range(0, LONG_VALUES1.length).forEach(i -> bufferAggregateColumn(Collections.singletonList(selector), agg, buf, pos));
    BloomKFilter bloomKFilter = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)));
    String serialized = filterToString(bloomKFilter);
    Assert.assertEquals(serializedLongFilter, serialized);
}

Example 39 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterSqlAggregatorTest method testBloomFilterAggFloatVirtualColumn.

@Test
public void testBloomFilterAggFloatVirtualColumn() throws Exception {
    cannotVectorize();
    BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
    for (InputRow row : CalciteTests.ROWS1_WITH_NUMERIC_DIMS) {
        Object raw = row.getRaw("f1");
        if (raw == null) {
            if (NullHandling.replaceWithDefault()) {
                expected1.addFloat(NullHandling.defaultFloatValue());
            } else {
                expected1.addBytes(null, 0, 0);
            }
        } else {
            expected1.addFloat(2 * ((Number) raw).floatValue());
        }
    }
    testQuery("SELECT\n" + "BLOOM_FILTER(f1 * 2, 1000)\n" + "FROM numfoo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE3).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "(\"f1\" * 2)", ColumnType.FLOAT, TestExprMacroTable.INSTANCE)).aggregators(ImmutableList.of(new BloomFilterAggregatorFactory("a0:agg", new DefaultDimensionSpec("v0", "a0:v0"), TEST_NUM_ENTRIES))).context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(new Object[] { CalciteTests.getJsonMapper().writeValueAsString(expected1) }));
}

Example 40 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterSqlAggregatorTest method testGroupByAggregatorDefaultValues.

@Test
public void testGroupByAggregatorDefaultValues() throws Exception {
    // makes empty bloom filters
    cannotVectorize();
    BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
    BloomKFilter expected2 = new BloomKFilter(TEST_NUM_ENTRIES);
    testQuery("SELECT\n" + "dim2,\n" + "BLOOM_FILTER(dim1, 1000) FILTER(WHERE dim1 = 'nonexistent'),\n" + "BLOOM_FILTER(l1, 1000) FILTER(WHERE dim1 = 'nonexistent')\n" + "FROM numfoo WHERE dim2 = 'a' GROUP BY dim2", ImmutableList.of(GroupByQuery.builder().setDataSource(CalciteTests.DATASOURCE3).setInterval(querySegmentSpec(Filtration.eternity())).setDimFilter(selector("dim2", "a", null)).setGranularity(Granularities.ALL).setVirtualColumns(expressionVirtualColumn("v0", "'a'", ColumnType.STRING)).setDimensions(new DefaultDimensionSpec("v0", "_d0", ColumnType.STRING)).setAggregatorSpecs(aggregators(new FilteredAggregatorFactory(new BloomFilterAggregatorFactory("a0:agg", new DefaultDimensionSpec("dim1", "a0:dim1"), TEST_NUM_ENTRIES), selector("dim1", "nonexistent", null)), new FilteredAggregatorFactory(new BloomFilterAggregatorFactory("a1:agg", new DefaultDimensionSpec("l1", "a1:l1", ColumnType.LONG), TEST_NUM_ENTRIES), selector("dim1", "nonexistent", null)))).setContext(QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(new Object[] { "a", CalciteTests.getJsonMapper().writeValueAsString(expected1), CalciteTests.getJsonMapper().writeValueAsString(expected2) }));
}