use of org.apache.druid.segment.data.BitmapSerdeFactory in project presto by prestodb.
the class V9SegmentIndexSource method loadIndex.
@Override
public QueryableIndex loadIndex(List<ColumnHandle> columnHandles) throws IOException {
ByteBuffer indexBuffer = ByteBuffer.wrap(segmentColumnSource.getColumnData(INDEX_METADATA_FILE_NAME));
GenericIndexed.read(indexBuffer, STRING_STRATEGY);
GenericIndexed<String> allDimensions = GenericIndexed.read(indexBuffer, STRING_STRATEGY);
Interval dataInterval = Intervals.utc(indexBuffer.getLong(), indexBuffer.getLong());
BitmapSerdeFactory segmentBitmapSerdeFactory;
if (indexBuffer.hasRemaining()) {
segmentBitmapSerdeFactory = JSON_MAPPER.readValue(SERIALIZER_UTILS.readString(indexBuffer), BitmapSerdeFactory.class);
} else {
segmentBitmapSerdeFactory = new BitmapSerde.LegacyBitmapSerdeFactory();
}
Metadata metadata = null;
ByteBuffer metadataBuffer = ByteBuffer.wrap(segmentColumnSource.getColumnData(SEGMENT_METADATA_FILE_NAME));
try {
metadata = JSON_MAPPER.readValue(SERIALIZER_UTILS.readBytes(metadataBuffer, metadataBuffer.remaining()), Metadata.class);
} catch (JsonParseException | JsonMappingException e) {
// Any jackson deserialization errors are ignored e.g. if metadata contains some aggregator which
// is no longer supported then it is OK to not use the metadata instead of failing segment loading
log.warn(e, "Failed to load metadata for segment");
}
Map<String, Supplier<ColumnHolder>> columns = new HashMap<>();
for (ColumnHandle columnHandle : columnHandles) {
String columnName = ((DruidColumnHandle) columnHandle).getColumnName();
columns.put(columnName, () -> createColumnHolder(columnName));
}
List<String> availableDimensions = Streams.stream(allDimensions.iterator()).filter(columns::containsKey).collect(toImmutableList());
columns.put(TIME_COLUMN_NAME, () -> createColumnHolder(TIME_COLUMN_NAME));
Indexed<String> indexed = new ListIndexed<>(availableDimensions);
// TODO: get rid of the time column by creating Presto's SimpleQueryableIndex impl
return new SimpleQueryableIndex(dataInterval, indexed, segmentBitmapSerdeFactory.getBitmapFactory(), columns, null, metadata, false);
}
use of org.apache.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.
the class DumpSegment method runBitmaps.
private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
final BitmapSerdeFactory bitmapSerdeFactory;
if (bitmapFactory instanceof ConciseBitmapFactory) {
bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
} else if (bitmapFactory instanceof RoaringBitmapFactory) {
bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
} else {
throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
}
final List<String> columnNames = getColumnsToInclude(index);
withOutputStream(new Function<OutputStream, Object>() {
@Override
public Object apply(final OutputStream out) {
try (final JsonGenerator jg = objectMapper.getFactory().createGenerator(out)) {
jg.writeStartObject();
{
jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
jg.writeFieldName("bitmaps");
jg.writeStartObject();
{
for (final String columnName : columnNames) {
final ColumnHolder columnHolder = index.getColumnHolder(columnName);
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
if (bitmapIndex == null) {
jg.writeNullField(columnName);
} else {
jg.writeFieldName(columnName);
jg.writeStartObject();
for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
String val = bitmapIndex.getValue(i);
// respect nulls if they are present in the dictionary
jg.writeFieldName(val == null ? "null" : val);
final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
if (decompressBitmaps) {
jg.writeStartArray();
final IntIterator iterator = bitmap.iterator();
while (iterator.hasNext()) {
final int rowNum = iterator.next();
jg.writeNumber(rowNum);
}
jg.writeEndArray();
} else {
byte[] bytes = bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap);
if (bytes != null) {
jg.writeBinary(bytes);
}
}
}
jg.writeEndObject();
}
}
}
jg.writeEndObject();
}
jg.writeEndObject();
} catch (IOException e) {
throw new RuntimeException(e);
}
return null;
}
});
}
use of org.apache.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.
the class LikeFilterBenchmark method setup.
@Setup
public void setup() {
step = (END_INT - START_INT) / cardinality;
final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
final List<Integer> ints = generateInts();
final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {
@Override
public String apply(Integer i) {
return i.toString();
}
}), GenericIndexed.STRING_STRATEGY);
final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {
@Override
public ImmutableBitmap apply(Integer i) {
final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
mutableBitmap.add((i - START_INT) / step);
return bitmapFactory.makeImmutableBitmap(mutableBitmap);
}
}), serdeFactory.getObjectStrategy()), dictionary).get();
selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
}
use of org.apache.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.
the class BoundFilterBenchmark method setup.
@Setup
public void setup() {
step = (END_INT - START_INT) / cardinality;
final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
final List<Integer> ints = generateInts();
final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(i -> i.toString()), GenericIndexed.STRING_STRATEGY);
final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {
@Override
public ImmutableBitmap apply(Integer i) {
final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
mutableBitmap.add((i - START_INT) / step);
return bitmapFactory.makeImmutableBitmap(mutableBitmap);
}
}), serdeFactory.getObjectStrategy()), dictionary).get();
selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
}
use of org.apache.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.
the class DimensionPredicateFilterBenchmark method setup.
@Setup
public void setup() {
final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
final List<Integer> ints = generateInts();
final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {
@Override
public String apply(Integer i) {
return i.toString();
}
}), GenericIndexed.STRING_STRATEGY);
final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {
@Override
public ImmutableBitmap apply(Integer i) {
final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
mutableBitmap.add(i - START_INT);
return bitmapFactory.makeImmutableBitmap(mutableBitmap);
}
}), serdeFactory.getObjectStrategy()), dictionary).get();
selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
}
Aggregations