Examples with ImmutableBitmap org.apache.druid.collections.bitmap.ImmutableBitmap used on opensource projects

Example 56 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class OrFilter method makeMatcher.

@Override
public ValueMatcher makeMatcher(BitmapIndexSelector selector, ColumnSelectorFactory columnSelectorFactory, RowOffsetMatcherFactory rowOffsetMatcherFactory) {
    final List<ValueMatcher> matchers = new ArrayList<>();
    final List<ImmutableBitmap> bitmaps = new ArrayList<>();
    for (Filter filter : filters) {
        if (filter.supportsBitmapIndex(selector)) {
            bitmaps.add(filter.getBitmapIndex(selector));
        } else {
            ValueMatcher matcher = filter.makeMatcher(columnSelectorFactory);
            matchers.add(matcher);
        }
    }
    if (bitmaps.size() > 0) {
        ImmutableBitmap combinedBitmap = selector.getBitmapFactory().union(bitmaps);
        ValueMatcher offsetMatcher = rowOffsetMatcherFactory.makeRowOffsetMatcher(combinedBitmap);
        matchers.add(0, offsetMatcher);
    }
    return makeMatcher(matchers.toArray(BooleanFilter.EMPTY_VALUE_MATCHER_ARRAY));
}

Example 57 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class IndexMergerNullHandlingTest method testStringColumnNullHandling.

@Test
public void testStringColumnNullHandling() throws Exception {
    List<Map<String, Object>> nonNullFlavors = new ArrayList<>();
    nonNullFlavors.add(ImmutableMap.of("d", "a"));
    nonNullFlavors.add(ImmutableMap.of("d", ImmutableList.of("a", "b")));
    List<Map<String, Object>> nullFlavors = new ArrayList<>();
    Map<String, Object> mMissing = ImmutableMap.of();
    Map<String, Object> mEmptyList = ImmutableMap.of("d", Collections.emptyList());
    Map<String, Object> mNull = new HashMap<>();
    mNull.put("d", null);
    Map<String, Object> mEmptyString = ImmutableMap.of("d", "");
    Map<String, Object> mListOfNull = ImmutableMap.of("d", Collections.singletonList(null));
    Map<String, Object> mListOfEmptyString = ImmutableMap.of("d", Collections.singletonList(""));
    nullFlavors.add(mMissing);
    nullFlavors.add(mEmptyList);
    nullFlavors.add(mNull);
    nullFlavors.add(mListOfNull);
    if (NullHandling.replaceWithDefault()) {
        nullFlavors.add(mEmptyString);
        nullFlavors.add(mListOfEmptyString);
    } else {
        nonNullFlavors.add(mEmptyString);
        nonNullFlavors.add(mListOfEmptyString);
    }
    Set<Map<String, Object>> allValues = new HashSet<>();
    allValues.addAll(nonNullFlavors);
    allValues.addAll(nullFlavors);
    for (Set<Map<String, Object>> subset : Sets.powerSet(allValues)) {
        if (subset.isEmpty()) {
            continue;
        }
        final List<Map<String, Object>> subsetList = new ArrayList<>(subset);
        IncrementalIndex toPersist = IncrementalIndexTest.createIndex(new AggregatorFactory[] {});
        for (Map<String, Object> m : subsetList) {
            toPersist.add(new MapBasedInputRow(0L, ImmutableList.of("d"), m));
        }
        final File tempDir = temporaryFolder.newFolder();
        try (QueryableIndex index = indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null))) {
            final ColumnHolder columnHolder = index.getColumnHolder("d");
            if (nullFlavors.containsAll(subsetList)) {
                // all null -> should be missing
                Assert.assertNull(subsetList.toString(), columnHolder);
            } else {
                Assert.assertNotNull(subsetList.toString(), columnHolder);
                // The column has multiple values if there are any lists with > 1 element in the input set.
                final boolean hasMultipleValues = subsetList.stream().anyMatch(m -> m.get("d") instanceof List && (((List) m.get("d")).size() > 1));
                // Compute all unique values, the same way that IndexMerger is expected to do it.
                final Set<String> uniqueValues = new HashSet<>();
                for (Map<String, Object> m : subsetList) {
                    final List<String> dValues = normalize(m.get("d"), hasMultipleValues);
                    uniqueValues.addAll(dValues);
                    if (nullFlavors.contains(m)) {
                        uniqueValues.add(null);
                    }
                }
                try (final DictionaryEncodedColumn<String> dictionaryColumn = (DictionaryEncodedColumn<String>) columnHolder.getColumn()) {
                    // Verify unique values against the dictionary.
                    Assert.assertEquals(subsetList.toString(), uniqueValues.stream().sorted(Comparators.naturalNullsFirst()).collect(Collectors.toList()), IntStream.range(0, dictionaryColumn.getCardinality()).mapToObj(dictionaryColumn::lookupName).collect(Collectors.toList()));
                    Assert.assertEquals(subsetList.toString(), hasMultipleValues, dictionaryColumn.hasMultipleValues());
                    Assert.assertEquals(subsetList.toString(), uniqueValues.size(), dictionaryColumn.getCardinality());
                    // Verify the expected set of rows was indexed, ignoring order.
                    Assert.assertEquals(subsetList.toString(), ImmutableMultiset.copyOf(subsetList.stream().map(m -> normalize(m.get("d"), hasMultipleValues)).distinct().collect(Collectors.toList())), ImmutableMultiset.copyOf(IntStream.range(0, index.getNumRows()).mapToObj(rowNumber -> getRow(dictionaryColumn, rowNumber)).distinct().collect(Collectors.toList())));
                    // Verify that the bitmap index for null is correct.
                    final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
                    // Read through the column to find all the rows that should match null.
                    final List<Integer> expectedNullRows = new ArrayList<>();
                    for (int i = 0; i < index.getNumRows(); i++) {
                        final List<String> row = getRow(dictionaryColumn, i);
                        if (row.isEmpty() || row.stream().anyMatch(NullHandling::isNullOrEquivalent)) {
                            expectedNullRows.add(i);
                        }
                    }
                    Assert.assertEquals(subsetList.toString(), expectedNullRows.size() > 0, bitmapIndex.hasNulls());
                    if (expectedNullRows.size() > 0) {
                        Assert.assertEquals(subsetList.toString(), 0, bitmapIndex.getIndex(null));
                        final ImmutableBitmap nullBitmap = bitmapIndex.getBitmap(bitmapIndex.getIndex(null));
                        final List<Integer> actualNullRows = new ArrayList<>();
                        final IntIterator iterator = nullBitmap.iterator();
                        while (iterator.hasNext()) {
                            actualNullRows.add(iterator.next());
                        }
                        Assert.assertEquals(subsetList.toString(), expectedNullRows, actualNullRows);
                    } else {
                        Assert.assertEquals(-1, bitmapIndex.getIndex(null));
                    }
                }
            }
        }
    }
}

Example 58 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class ColumnSelectorBitmapIndexSelectorTest method setup.

@Before
public void setup() {
    bitmapFactory = EasyMock.createMock(BitmapFactory.class);
    virtualColumns = EasyMock.createMock(VirtualColumns.class);
    index = EasyMock.createMock(ColumnSelector.class);
    bitmapIndexSelector = new ColumnSelectorBitmapIndexSelector(bitmapFactory, virtualColumns, index);
    EasyMock.expect(virtualColumns.getVirtualColumn(STRING_DICTIONARY_COLUMN_NAME)).andReturn(null).anyTimes();
    EasyMock.expect(virtualColumns.getVirtualColumn(NON_STRING_DICTIONARY_COLUMN_NAME)).andReturn(null).anyTimes();
    ColumnHolder holder = EasyMock.createMock(ColumnHolder.class);
    EasyMock.expect(index.getColumnHolder(STRING_DICTIONARY_COLUMN_NAME)).andReturn(holder).anyTimes();
    StringDictionaryEncodedColumn stringColumn = EasyMock.createMock(StringDictionaryEncodedColumn.class);
    EasyMock.expect(holder.getCapabilities()).andReturn(ColumnCapabilitiesImpl.createDefault().setType(ColumnType.STRING).setDictionaryEncoded(true).setDictionaryValuesUnique(true).setDictionaryValuesSorted(true).setHasBitmapIndexes(true)).anyTimes();
    EasyMock.expect(holder.getColumn()).andReturn(stringColumn).anyTimes();
    BitmapIndex someIndex = EasyMock.createMock(BitmapIndex.class);
    EasyMock.expect(holder.getBitmapIndex()).andReturn(someIndex).anyTimes();
    ImmutableBitmap someBitmap = EasyMock.createMock(ImmutableBitmap.class);
    EasyMock.expect(someIndex.getIndex("foo")).andReturn(0).anyTimes();
    EasyMock.expect(someIndex.getBitmap(0)).andReturn(someBitmap).anyTimes();
    ColumnHolder nonStringHolder = EasyMock.createMock(ColumnHolder.class);
    EasyMock.expect(index.getColumnHolder(NON_STRING_DICTIONARY_COLUMN_NAME)).andReturn(nonStringHolder).anyTimes();
    EasyMock.expect(nonStringHolder.getCapabilities()).andReturn(ColumnCapabilitiesImpl.createDefault().setType(ColumnType.ofComplex("testBlob")).setDictionaryEncoded(true).setDictionaryValuesUnique(true).setDictionaryValuesSorted(true).setHasBitmapIndexes(true).setFilterable(true)).anyTimes();
    EasyMock.replay(bitmapFactory, virtualColumns, index, holder, stringColumn, nonStringHolder, someIndex, someBitmap);
}

Example 59 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class ColumnSelectorBitmapIndexSelectorTest method testNonStringDictionaryDoNotUseIndex.

@Test
public void testNonStringDictionaryDoNotUseIndex() {
    BitmapIndex bitmapIndex = bitmapIndexSelector.getBitmapIndex(NON_STRING_DICTIONARY_COLUMN_NAME);
    Assert.assertNull(bitmapIndex);
    Indexed<String> vals = bitmapIndexSelector.getDimensionValues(NON_STRING_DICTIONARY_COLUMN_NAME);
    Assert.assertNull(vals);
    ImmutableBitmap valueIndex = bitmapIndexSelector.getBitmapIndex(NON_STRING_DICTIONARY_COLUMN_NAME, "foo");
    Assert.assertNull(valueIndex);
    EasyMock.verify(bitmapFactory, virtualColumns, index);
}

Example 60 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class VectorSelectorUtilsTest method testConciseImmutableNullVector.

@Test
public void testConciseImmutableNullVector() {
    final WrappedConciseBitmap bitmap = new WrappedConciseBitmap();
    populate(bitmap, NULLS);
    final ImmutableBitmap immutable = new WrappedImmutableConciseBitmap(ImmutableConciseSet.newImmutableFromMutable(bitmap.getBitmap()));
    assertNullVector(immutable, NULLS);
    final WrappedConciseBitmap bitmap2 = new WrappedConciseBitmap();
    populate(bitmap2, NULLS_PATTERN);
    final ImmutableBitmap immutable2 = new WrappedImmutableConciseBitmap(ImmutableConciseSet.newImmutableFromMutable(bitmap2.getBitmap()));
    assertNullVector(immutable2, NULLS_PATTERN);
}