Examples with ImmutableBitmap org.apache.druid.collections.bitmap.ImmutableBitmap used on opensource projects

Example 6 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class ExtractionDimFilterTest method testNormal.

@Test
public void testNormal() {
    Filter extractionFilter = new SelectorDimFilter("foo", "extractDimVal", DIM_EXTRACTION_FN).toFilter();
    ImmutableBitmap immutableBitmap = extractionFilter.getBitmapIndex(BITMAP_INDEX_SELECTOR);
    Assert.assertEquals(1, immutableBitmap.size());
}

Example 7 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class Filters method estimateSelectivity.

/**
 * Return an estimated selectivity for bitmaps for the dimension values given by dimValueIndexes.
 *
 * @param bitmapIndex  bitmap index
 * @param bitmaps      bitmaps to extract, by index
 * @param totalNumRows number of rows in the column associated with this bitmap index
 *
 * @return estimated selectivity
 */
public static double estimateSelectivity(final BitmapIndex bitmapIndex, final IntList bitmaps, final long totalNumRows) {
    long numMatchedRows = 0;
    for (int i = 0; i < bitmaps.size(); i++) {
        final ImmutableBitmap bitmap = bitmapIndex.getBitmap(bitmaps.getInt(i));
        numMatchedRows += bitmap.size();
    }
    return Math.min(1., (double) numMatchedRows / totalNumRows);
}

Example 8 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class BitmapOffsetTest method testSanity.

@Test
public void testSanity() {
    MutableBitmap mutable = factory.makeEmptyMutableBitmap();
    for (int val : TEST_VALS) {
        mutable.add(val);
    }
    ImmutableBitmap bitmap = factory.makeImmutableBitmap(mutable);
    final BitmapOffset offset = BitmapOffset.of(bitmap, descending, bitmap.size());
    final int[] expected = descending ? TEST_VALS_FLIP : TEST_VALS;
    int count = 0;
    while (offset.withinBounds()) {
        Assert.assertEquals(expected[count], offset.getOffset());
        int cloneCount = count;
        Offset clonedOffset = offset.clone();
        while (clonedOffset.withinBounds()) {
            Assert.assertEquals(expected[cloneCount], clonedOffset.getOffset());
            ++cloneCount;
            clonedOffset.increment();
        }
        ++count;
        offset.increment();
    }
    Assert.assertEquals(count, expected.length);
}

Example 9 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class ColumnSelectorBitmapIndexSelectorTest method testStringDictionaryUseIndex.

@Test
public void testStringDictionaryUseIndex() {
    BitmapIndex bitmapIndex = bitmapIndexSelector.getBitmapIndex(STRING_DICTIONARY_COLUMN_NAME);
    Assert.assertNotNull(bitmapIndex);
    Indexed<String> vals = bitmapIndexSelector.getDimensionValues(STRING_DICTIONARY_COLUMN_NAME);
    Assert.assertNotNull(vals);
    ImmutableBitmap valueIndex = bitmapIndexSelector.getBitmapIndex(STRING_DICTIONARY_COLUMN_NAME, "foo");
    Assert.assertNotNull(valueIndex);
    EasyMock.verify(bitmapFactory, virtualColumns, index);
}

Example 10 with ImmutableBitmap

use of org.apache.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class DictionaryEncodedColumnPartSerde method getDeserializer.

@Override
public Deserializer getDeserializer() {
    return new Deserializer() {

        @Override
        public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) {
            final VERSION rVersion = VERSION.fromByte(buffer.get());
            final int rFlags;
            if (rVersion.compareTo(VERSION.COMPRESSED) >= 0) {
                rFlags = buffer.getInt();
            } else {
                rFlags = rVersion.equals(VERSION.UNCOMPRESSED_MULTI_VALUE) ? Feature.MULTI_VALUE.getMask() : NO_FLAGS;
            }
            final boolean hasMultipleValues = Feature.MULTI_VALUE.isSet(rFlags) || Feature.MULTI_VALUE_V3.isSet(rFlags);
            // Duplicate the first buffer since we are reading the dictionary twice.
            final GenericIndexed<String> rDictionary = GenericIndexed.read(buffer.duplicate(), GenericIndexed.STRING_STRATEGY, builder.getFileMapper());
            final GenericIndexed<ByteBuffer> rDictionaryUtf8 = GenericIndexed.read(buffer, GenericIndexed.BYTE_BUFFER_STRATEGY, builder.getFileMapper());
            builder.setType(ValueType.STRING);
            final WritableSupplier<ColumnarInts> rSingleValuedColumn;
            final WritableSupplier<ColumnarMultiInts> rMultiValuedColumn;
            if (hasMultipleValues) {
                rMultiValuedColumn = readMultiValuedColumn(rVersion, buffer, rFlags);
                rSingleValuedColumn = null;
            } else {
                rSingleValuedColumn = readSingleValuedColumn(rVersion, buffer);
                rMultiValuedColumn = null;
            }
            final String firstDictionaryEntry = rDictionary.get(0);
            DictionaryEncodedColumnSupplier dictionaryEncodedColumnSupplier = new DictionaryEncodedColumnSupplier(rDictionary, rDictionaryUtf8, rSingleValuedColumn, rMultiValuedColumn, columnConfig.columnCacheSizeBytes());
            builder.setHasMultipleValues(hasMultipleValues).setHasNulls(firstDictionaryEntry == null).setDictionaryEncodedColumnSupplier(dictionaryEncodedColumnSupplier);
            if (!Feature.NO_BITMAP_INDEX.isSet(rFlags)) {
                GenericIndexed<ImmutableBitmap> rBitmaps = GenericIndexed.read(buffer, bitmapSerdeFactory.getObjectStrategy(), builder.getFileMapper());
                builder.setBitmapIndex(new StringBitmapIndexColumnPartSupplier(bitmapSerdeFactory.getBitmapFactory(), rBitmaps, rDictionary));
            }
            if (buffer.hasRemaining()) {
                ImmutableRTree rSpatialIndex = new ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()).fromByteBufferWithSize(buffer);
                builder.setSpatialIndex(new SpatialIndexColumnPartSupplier(rSpatialIndex));
            }
        }

        private WritableSupplier<ColumnarInts> readSingleValuedColumn(VERSION version, ByteBuffer buffer) {
            switch(version) {
                case UNCOMPRESSED_SINGLE_VALUE:
                case UNCOMPRESSED_WITH_FLAGS:
                    return VSizeColumnarInts.readFromByteBuffer(buffer);
                case COMPRESSED:
                    return CompressedVSizeColumnarIntsSupplier.fromByteBuffer(buffer, byteOrder);
                default:
                    throw new IAE("Unsupported single-value version[%s]", version);
            }
        }

        private WritableSupplier<ColumnarMultiInts> readMultiValuedColumn(VERSION version, ByteBuffer buffer, int flags) {
            switch(version) {
                case UNCOMPRESSED_MULTI_VALUE:
                    {
                        return VSizeColumnarMultiInts.readFromByteBuffer(buffer);
                    }
                case UNCOMPRESSED_WITH_FLAGS:
                    {
                        if (Feature.MULTI_VALUE.isSet(flags)) {
                            return VSizeColumnarMultiInts.readFromByteBuffer(buffer);
                        } else {
                            throw new IAE("Unrecognized multi-value flag[%d] for version[%s]", flags, version);
                        }
                    }
                case COMPRESSED:
                    {
                        if (Feature.MULTI_VALUE.isSet(flags)) {
                            return CompressedVSizeColumnarMultiIntsSupplier.fromByteBuffer(buffer, byteOrder);
                        } else if (Feature.MULTI_VALUE_V3.isSet(flags)) {
                            return V3CompressedVSizeColumnarMultiIntsSupplier.fromByteBuffer(buffer, byteOrder);
                        } else {
                            throw new IAE("Unrecognized multi-value flag[%d] for version[%s]", flags, version);
                        }
                    }
                default:
                    throw new IAE("Unsupported multi-value version[%s]", version);
            }
        }
    };
}