Examples with BloomKFilter org.apache.druid.query.filter.BloomKFilter used on opensource projects

Example 6 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterAggregatorTest method testBufferAggregateDoubleValues.

@Test
public void testBufferAggregateDoubleValues() throws IOException {
    TestDoubleColumnSelector selector = new TestDoubleColumnSelector(Arrays.asList(DOUBLE_VALUES1));
    DoubleBloomFilterAggregator agg = new DoubleBloomFilterAggregator(selector, MAX_NUM_VALUES, true);
    int maxSize = valueAggregatorFactory.getMaxIntermediateSizeWithNulls();
    ByteBuffer buf = ByteBuffer.allocate(maxSize + 64);
    int pos = 10;
    buf.limit(pos + maxSize);
    agg.init(buf, pos);
    IntStream.range(0, DOUBLE_VALUES1.length).forEach(i -> bufferAggregateColumn(Collections.singletonList(selector), agg, buf, pos));
    BloomKFilter bloomKFilter = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)));
    String serialized = filterToString(bloomKFilter);
    Assert.assertEquals(serializedDoubleFilter, serialized);
}

Example 7 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterAggregatorTest method testBufferAggregateFloatValues.

@Test
public void testBufferAggregateFloatValues() throws IOException {
    TestFloatColumnSelector selector = new TestFloatColumnSelector(Arrays.asList(FLOAT_VALUES1));
    FloatBloomFilterAggregator agg = new FloatBloomFilterAggregator(selector, MAX_NUM_VALUES, true);
    int maxSize = valueAggregatorFactory.getMaxIntermediateSizeWithNulls();
    ByteBuffer buf = ByteBuffer.allocate(maxSize + 64);
    int pos = 10;
    buf.limit(pos + maxSize);
    agg.init(buf, pos);
    IntStream.range(0, FLOAT_VALUES1.length).forEach(i -> bufferAggregateColumn(Collections.singletonList(selector), agg, buf, pos));
    BloomKFilter bloomKFilter = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)));
    String serialized = filterToString(bloomKFilter);
    Assert.assertEquals(serializedFloatFilter, serialized);
}

Example 8 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterAggregatorTest method testAggregateFloatValues.

@Test
public void testAggregateFloatValues() throws IOException {
    TestFloatColumnSelector selector = new TestFloatColumnSelector(Arrays.asList(FLOAT_VALUES1));
    FloatBloomFilterAggregator agg = new FloatBloomFilterAggregator(selector, MAX_NUM_VALUES, true);
    for (Float ignored : FLOAT_VALUES1) {
        aggregateColumn(Collections.singletonList(selector), agg);
    }
    BloomKFilter bloomKFilter = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.finalizeComputation(agg.get()));
    String serialized = filterToString(bloomKFilter);
    Assert.assertEquals(serializedFloatFilter, serialized);
}

Example 9 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterAggregatorTest method testMergeValuesWithBuffersForGroupByV1.

@Test
public void testMergeValuesWithBuffersForGroupByV1() throws IOException {
    final TestBloomFilterBufferColumnSelector mergeDim = new TestBloomFilterBufferColumnSelector(ImmutableList.of(ByteBuffer.wrap(BloomFilterSerializersModule.bloomKFilterToBytes(filter1)), ByteBuffer.wrap(BloomFilterSerializersModule.bloomKFilterToBytes(filter2))));
    BloomFilterMergeAggregator mergeAggregator = new BloomFilterMergeAggregator(mergeDim, MAX_NUM_VALUES, true);
    for (int i = 0; i < 2; ++i) {
        aggregateColumn(Collections.singletonList(mergeDim), mergeAggregator);
    }
    BloomKFilter merged = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.getCombiningFactory().finalizeComputation(mergeAggregator.get()));
    String serialized = filterToString(merged);
    Assert.assertEquals(serializedCombinedFilter, serialized);
}

Example 10 with BloomKFilter

use of org.apache.druid.query.filter.BloomKFilter in project druid by druid-io.

the class BloomFilterAggregatorTest method testCombineValues.

@Test
public void testCombineValues() throws IOException {
    DimensionSelector dimSelector1 = new CardinalityAggregatorTest.TestDimensionSelector(VALUES1, null);
    DimensionSelector dimSelector2 = new CardinalityAggregatorTest.TestDimensionSelector(VALUES2, null);
    StringBloomFilterAggregator agg1 = new StringBloomFilterAggregator(dimSelector1, MAX_NUM_VALUES, true);
    StringBloomFilterAggregator agg2 = new StringBloomFilterAggregator(dimSelector2, MAX_NUM_VALUES, true);
    for (int i = 0; i < VALUES1.size(); ++i) {
        aggregateDimension(Collections.singletonList(dimSelector1), agg1);
    }
    for (int i = 0; i < VALUES2.size(); ++i) {
        aggregateDimension(Collections.singletonList(dimSelector2), agg2);
    }
    BloomKFilter combined = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.finalizeComputation(valueAggregatorFactory.combine(agg1.get(), agg2.get())));
    String serialized = filterToString(combined);
    Assert.assertEquals(serializedCombinedFilter, serialized);
}