Examples with ImmutableBitmap org.apache.druid.collections.bitmap.ImmutableBitmap used on opensource projects

Example 61 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class ListFilteredVirtualColumnSelectorTest method testFilterListFilteredVirtualColumnAllowIndex.

@Test
public void testFilterListFilteredVirtualColumnAllowIndex() {
    ListFilteredVirtualColumn virtualColumn = new ListFilteredVirtualColumn(ALLOW_VIRTUAL_NAME, new DefaultDimensionSpec(COLUMN_NAME, COLUMN_NAME, ColumnType.STRING), ImmutableSet.of("b", "c"), true);
    ColumnSelector selector = EasyMock.createMock(ColumnSelector.class);
    ColumnHolder holder = EasyMock.createMock(ColumnHolder.class);
    BitmapIndex index = EasyMock.createMock(BitmapIndex.class);
    ImmutableBitmap bitmap = EasyMock.createMock(ImmutableBitmap.class);
    BitmapFactory bitmapFactory = EasyMock.createMock(BitmapFactory.class);
    EasyMock.expect(selector.getColumnHolder(COLUMN_NAME)).andReturn(holder).atLeastOnce();
    EasyMock.expect(holder.getBitmapIndex()).andReturn(index).atLeastOnce();
    EasyMock.expect(index.getCardinality()).andReturn(3).atLeastOnce();
    EasyMock.expect(index.getValue(0)).andReturn("a").atLeastOnce();
    EasyMock.expect(index.getValue(1)).andReturn("b").atLeastOnce();
    EasyMock.expect(index.getValue(2)).andReturn("c").atLeastOnce();
    EasyMock.expect(index.getBitmap(2)).andReturn(bitmap).once();
    EasyMock.expect(index.getBitmapFactory()).andReturn(bitmapFactory).once();
    EasyMock.expect(index.hasNulls()).andReturn(true).once();
    EasyMock.replay(selector, holder, index, bitmap, bitmapFactory);
    ColumnSelectorBitmapIndexSelector bitmapIndexSelector = new ColumnSelectorBitmapIndexSelector(new RoaringBitmapFactory(), VirtualColumns.create(Collections.singletonList(virtualColumn)), selector);
    SelectorFilter filter = new SelectorFilter(ALLOW_VIRTUAL_NAME, "a");
    Assert.assertTrue(filter.shouldUseBitmapIndex(bitmapIndexSelector));
    BitmapIndex listFilteredIndex = bitmapIndexSelector.getBitmapIndex(ALLOW_VIRTUAL_NAME);
    Assert.assertEquals(-1, listFilteredIndex.getIndex("a"));
    Assert.assertEquals(0, listFilteredIndex.getIndex("b"));
    Assert.assertEquals(1, listFilteredIndex.getIndex("c"));
    Assert.assertEquals(2, listFilteredIndex.getCardinality());
    Assert.assertEquals("b", listFilteredIndex.getValue(0));
    Assert.assertEquals("c", listFilteredIndex.getValue(1));
    Assert.assertEquals(bitmap, listFilteredIndex.getBitmap(1));
    Assert.assertEquals(bitmapFactory, listFilteredIndex.getBitmapFactory());
    Assert.assertTrue(listFilteredIndex.hasNulls());
    EasyMock.verify(selector, holder, index, bitmap, bitmapFactory);
}

Example 62 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class QueryableIndexStorageAdapter method analyzeFilter.

@VisibleForTesting
public FilterAnalysis analyzeFilter(@Nullable final Filter filter, ColumnSelectorBitmapIndexSelector indexSelector, @Nullable QueryMetrics queryMetrics) {
    final int totalRows = index.getNumRows();
    /*
     * Filters can be applied in two stages:
     * pre-filtering: Use bitmap indexes to prune the set of rows to be scanned.
     * post-filtering: Iterate through rows and apply the filter to the row values
     *
     * The pre-filter and post-filter step have an implicit AND relationship. (i.e., final rows are those that
     * were not pruned AND those that matched the filter during row scanning)
     *
     * An AND filter can have its subfilters partitioned across the two steps. The subfilters that can be
     * processed entirely with bitmap indexes (subfilter returns true for supportsBitmapIndex())
     * will be moved to the pre-filtering stage.
     *
     * Any subfilters that cannot be processed entirely with bitmap indexes will be moved to the post-filtering stage.
     */
    final List<Filter> preFilters;
    final List<Filter> postFilters = new ArrayList<>();
    int preFilteredRows = totalRows;
    if (filter == null) {
        preFilters = Collections.emptyList();
    } else {
        preFilters = new ArrayList<>();
        if (filter instanceof AndFilter) {
            // If we get an AndFilter, we can split the subfilters across both filtering stages
            for (Filter subfilter : ((AndFilter) filter).getFilters()) {
                if (subfilter.supportsBitmapIndex(indexSelector) && subfilter.shouldUseBitmapIndex(indexSelector)) {
                    preFilters.add(subfilter);
                } else {
                    postFilters.add(subfilter);
                }
            }
        } else {
            // If we get an OrFilter or a single filter, handle the filter in one stage
            if (filter.supportsBitmapIndex(indexSelector) && filter.shouldUseBitmapIndex(indexSelector)) {
                preFilters.add(filter);
            } else {
                postFilters.add(filter);
            }
        }
    }
    final ImmutableBitmap preFilterBitmap;
    if (preFilters.isEmpty()) {
        preFilterBitmap = null;
    } else {
        if (queryMetrics != null) {
            BitmapResultFactory<?> bitmapResultFactory = queryMetrics.makeBitmapResultFactory(indexSelector.getBitmapFactory());
            long bitmapConstructionStartNs = System.nanoTime();
            // Use AndFilter.getBitmapResult to intersect the preFilters to get its short-circuiting behavior.
            preFilterBitmap = AndFilter.getBitmapIndex(indexSelector, bitmapResultFactory, preFilters);
            preFilteredRows = preFilterBitmap.size();
            queryMetrics.reportBitmapConstructionTime(System.nanoTime() - bitmapConstructionStartNs);
        } else {
            BitmapResultFactory<?> bitmapResultFactory = new DefaultBitmapResultFactory(indexSelector.getBitmapFactory());
            preFilterBitmap = AndFilter.getBitmapIndex(indexSelector, bitmapResultFactory, preFilters);
        }
    }
    if (queryMetrics != null) {
        queryMetrics.preFilters(new ArrayList<>(preFilters));
        queryMetrics.postFilters(postFilters);
        queryMetrics.reportSegmentRows(totalRows);
        queryMetrics.reportPreFilteredRows(preFilteredRows);
    }
    return new FilterAnalysis(preFilterBitmap, Filters.maybeAnd(postFilters).orElse(null));
}

Example 63 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class NumericNullColumnSelectorTest method testLongSelectorWithNullsCanResetOffset.

@Test
public void testLongSelectorWithNullsCanResetOffset() {
    for (ImmutableBitmap bitmap : bitmaps) {
        ColumnarLongs longs = new ColumnarLongs() {

            @Override
            public int size() {
                return numRows;
            }

            @Override
            public long get(int index) {
                return ThreadLocalRandom.current().nextLong();
            }

            @Override
            public void close() {
            }
        };
        LongsColumn columnWithNulls = LongsColumn.create(longs, bitmap);
        ColumnValueSelector<?> selector = columnWithNulls.makeColumnValueSelector(offset);
        assertOffsetCanReset(selector, bitmap, offset);
        VectorValueSelector vectorSelector = columnWithNulls.makeVectorValueSelector(vectorOffset);
        assertVectorOffsetCanReset(vectorSelector, bitmap, vectorOffset);
    }
}

Example 64 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class BitmapCreationBenchmark method testToImmutableByteArray.

@BenchmarkOptions(warmupRounds = 10, benchmarkRounds = 1000)
@Test
public void testToImmutableByteArray() {
    ImmutableBitmap immutableBitmap = factory.makeImmutableBitmap(baseMutableBitmap);
    Assert.assertArrayEquals(baseBytes, immutableBitmap.toBytes());
}