use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.
the class FiltersTest method makeNonOverlappedBitmapIndexes.
private static BitmapIndex makeNonOverlappedBitmapIndexes(final int bitmapNum, final List<ImmutableBitmap> bitmaps) {
final BitmapIndex bitmapIndex = getBitmapIndex(bitmaps);
final BitmapFactory factory = bitmapIndex.getBitmapFactory();
for (int i = 0; i < bitmapNum; i++) {
final MutableBitmap mutableBitmap = factory.makeEmptyMutableBitmap();
for (int j = 0; j < 10; j++) {
mutableBitmap.add(i * 10 + j);
}
bitmaps.add(factory.makeImmutableBitmap(mutableBitmap));
}
return bitmapIndex;
}
use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.
the class Filters method matchPredicateNoUnion.
/**
* Return an iterable of bitmaps for all values matching a particular predicate. Unioning these bitmaps
* yields the same result that {@link #matchPredicate(String, BitmapIndexSelector, BitmapResultFactory, Predicate)}
* would have returned.
*
* @param dimension dimension to look at
* @param selector bitmap selector
* @param predicate predicate to use
*
* @return iterable of bitmaps of matching rows
*/
public static Iterable<ImmutableBitmap> matchPredicateNoUnion(final String dimension, final BitmapIndexSelector selector, final Predicate<String> predicate) {
Preconditions.checkNotNull(dimension, "dimension");
Preconditions.checkNotNull(selector, "selector");
Preconditions.checkNotNull(predicate, "predicate");
// Missing dimension -> match all rows if the predicate matches null; match no rows otherwise
try (final CloseableIndexed<String> dimValues = selector.getDimensionValues(dimension)) {
if (dimValues == null || dimValues.size() == 0) {
return ImmutableList.of(predicate.apply(null) ? allTrue(selector) : allFalse(selector));
}
// Apply predicate to all dimension values and union the matching bitmaps
final BitmapIndex bitmapIndex = selector.getBitmapIndex(dimension);
return makePredicateQualifyingBitmapIterable(bitmapIndex, predicate, dimValues);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.
the class ColumnSelectorBitmapIndexSelectorTest method testStringDictionaryUseIndex.
@Test
public void testStringDictionaryUseIndex() {
BitmapIndex bitmapIndex = bitmapIndexSelector.getBitmapIndex(STRING_DICTIONARY_COLUMN_NAME);
Assert.assertNotNull(bitmapIndex);
Indexed<String> vals = bitmapIndexSelector.getDimensionValues(STRING_DICTIONARY_COLUMN_NAME);
Assert.assertNotNull(vals);
ImmutableBitmap valueIndex = bitmapIndexSelector.getBitmapIndex(STRING_DICTIONARY_COLUMN_NAME, "foo");
Assert.assertNotNull(valueIndex);
EasyMock.verify(bitmapFactory, virtualColumns, index);
}
use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.
the class DumpSegment method runBitmaps.
private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
final BitmapSerdeFactory bitmapSerdeFactory;
if (bitmapFactory instanceof ConciseBitmapFactory) {
bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
} else if (bitmapFactory instanceof RoaringBitmapFactory) {
bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
} else {
throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
}
final List<String> columnNames = getColumnsToInclude(index);
withOutputStream(new Function<OutputStream, Object>() {
@Override
public Object apply(final OutputStream out) {
try (final JsonGenerator jg = objectMapper.getFactory().createGenerator(out)) {
jg.writeStartObject();
{
jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
jg.writeFieldName("bitmaps");
jg.writeStartObject();
{
for (final String columnName : columnNames) {
final ColumnHolder columnHolder = index.getColumnHolder(columnName);
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
if (bitmapIndex == null) {
jg.writeNullField(columnName);
} else {
jg.writeFieldName(columnName);
jg.writeStartObject();
for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
String val = bitmapIndex.getValue(i);
// respect nulls if they are present in the dictionary
jg.writeFieldName(val == null ? "null" : val);
final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
if (decompressBitmaps) {
jg.writeStartArray();
final IntIterator iterator = bitmap.iterator();
while (iterator.hasNext()) {
final int rowNum = iterator.next();
jg.writeNumber(rowNum);
}
jg.writeEndArray();
} else {
byte[] bytes = bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap);
if (bytes != null) {
jg.writeBinary(bytes);
}
}
}
jg.writeEndObject();
}
}
}
jg.writeEndObject();
}
jg.writeEndObject();
} catch (IOException e) {
throw new RuntimeException(e);
}
return null;
}
});
}
use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.
the class ColumnSelectorBitmapIndexSelector method getDimensionValues.
@Nullable
@Override
public CloseableIndexed<String> getDimensionValues(String dimension) {
if (isVirtualColumn(dimension)) {
BitmapIndex bitmapIndex = virtualColumns.getBitmapIndex(dimension, index);
if (bitmapIndex == null) {
return null;
}
return new CloseableIndexed<String>() {
@Override
public int size() {
return bitmapIndex.getCardinality();
}
@Override
public String get(int index) {
return bitmapIndex.getValue(index);
}
@Override
public int indexOf(String value) {
return bitmapIndex.getIndex(value);
}
@Override
public Iterator<String> iterator() {
return IndexedIterable.create(this).iterator();
}
@Override
public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
inspector.visit("column", bitmapIndex);
}
@Override
public void close() {
}
};
}
final ColumnHolder columnHolder = index.getColumnHolder(dimension);
if (columnHolder == null) {
return null;
}
if (!columnHolder.getCapabilities().toColumnType().is(ValueType.STRING)) {
// work correctly here until reworking is done to support filtering/indexing other types of columns
return null;
}
BaseColumn col = columnHolder.getColumn();
if (!(col instanceof DictionaryEncodedColumn)) {
return null;
}
final DictionaryEncodedColumn<String> column = (DictionaryEncodedColumn<String>) col;
return new CloseableIndexed<String>() {
@Override
public int size() {
return column.getCardinality();
}
@Override
public String get(int index) {
return column.lookupName(index);
}
@Override
public int indexOf(String value) {
return column.lookupId(value);
}
@Override
public Iterator<String> iterator() {
return IndexedIterable.create(this).iterator();
}
@Override
public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
inspector.visit("column", column);
}
@Override
public void close() throws IOException {
column.close();
}
};
}
Aggregations