use of org.apache.druid.segment.column.ColumnConfig in project druid by druid-io.
the class AggregationTestHelper method createGroupByQueryAggregationTestHelper.
public static AggregationTestHelper createGroupByQueryAggregationTestHelper(List<? extends Module> jsonModulesToRegister, GroupByQueryConfig config, TemporaryFolder tempFolder) {
final ObjectMapper mapper = TestHelper.makeJsonMapper();
final Pair<GroupByQueryRunnerFactory, Closer> factoryAndCloser = GroupByQueryRunnerTest.makeQueryRunnerFactory(mapper, config);
final GroupByQueryRunnerFactory factory = factoryAndCloser.lhs;
final Closer closer = factoryAndCloser.rhs;
IndexIO indexIO = new IndexIO(mapper, new ColumnConfig() {
@Override
public int columnCacheSizeBytes() {
return 0;
}
});
return new AggregationTestHelper(mapper, new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()), indexIO, factory.getToolchest(), factory, tempFolder, jsonModulesToRegister, closer, Collections.emptyMap());
}
use of org.apache.druid.segment.column.ColumnConfig in project druid by druid-io.
the class AggregationTestHelper method createTimeseriesQueryAggregationTestHelper.
public static AggregationTestHelper createTimeseriesQueryAggregationTestHelper(List<? extends Module> jsonModulesToRegister, TemporaryFolder tempFolder) {
ObjectMapper mapper = TestHelper.makeJsonMapper();
TimeseriesQueryQueryToolChest toolchest = new TimeseriesQueryQueryToolChest();
TimeseriesQueryRunnerFactory factory = new TimeseriesQueryRunnerFactory(toolchest, new TimeseriesQueryEngine(), QueryRunnerTestHelper.NOOP_QUERYWATCHER);
IndexIO indexIO = new IndexIO(mapper, new ColumnConfig() {
@Override
public int columnCacheSizeBytes() {
return 0;
}
});
return new AggregationTestHelper(mapper, new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()), indexIO, toolchest, factory, tempFolder, jsonModulesToRegister, Closer.create(), Collections.emptyMap());
}
use of org.apache.druid.segment.column.ColumnConfig in project druid by druid-io.
the class DictionaryEncodedColumnPartSerde method getDeserializer.
@Override
public Deserializer getDeserializer() {
return new Deserializer() {
@Override
public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) {
final VERSION rVersion = VERSION.fromByte(buffer.get());
final int rFlags;
if (rVersion.compareTo(VERSION.COMPRESSED) >= 0) {
rFlags = buffer.getInt();
} else {
rFlags = rVersion.equals(VERSION.UNCOMPRESSED_MULTI_VALUE) ? Feature.MULTI_VALUE.getMask() : NO_FLAGS;
}
final boolean hasMultipleValues = Feature.MULTI_VALUE.isSet(rFlags) || Feature.MULTI_VALUE_V3.isSet(rFlags);
// Duplicate the first buffer since we are reading the dictionary twice.
final GenericIndexed<String> rDictionary = GenericIndexed.read(buffer.duplicate(), GenericIndexed.STRING_STRATEGY, builder.getFileMapper());
final GenericIndexed<ByteBuffer> rDictionaryUtf8 = GenericIndexed.read(buffer, GenericIndexed.BYTE_BUFFER_STRATEGY, builder.getFileMapper());
builder.setType(ValueType.STRING);
final WritableSupplier<ColumnarInts> rSingleValuedColumn;
final WritableSupplier<ColumnarMultiInts> rMultiValuedColumn;
if (hasMultipleValues) {
rMultiValuedColumn = readMultiValuedColumn(rVersion, buffer, rFlags);
rSingleValuedColumn = null;
} else {
rSingleValuedColumn = readSingleValuedColumn(rVersion, buffer);
rMultiValuedColumn = null;
}
final String firstDictionaryEntry = rDictionary.get(0);
DictionaryEncodedColumnSupplier dictionaryEncodedColumnSupplier = new DictionaryEncodedColumnSupplier(rDictionary, rDictionaryUtf8, rSingleValuedColumn, rMultiValuedColumn, columnConfig.columnCacheSizeBytes());
builder.setHasMultipleValues(hasMultipleValues).setHasNulls(firstDictionaryEntry == null).setDictionaryEncodedColumnSupplier(dictionaryEncodedColumnSupplier);
if (!Feature.NO_BITMAP_INDEX.isSet(rFlags)) {
GenericIndexed<ImmutableBitmap> rBitmaps = GenericIndexed.read(buffer, bitmapSerdeFactory.getObjectStrategy(), builder.getFileMapper());
builder.setBitmapIndex(new StringBitmapIndexColumnPartSupplier(bitmapSerdeFactory.getBitmapFactory(), rBitmaps, rDictionary));
}
if (buffer.hasRemaining()) {
ImmutableRTree rSpatialIndex = new ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()).fromByteBufferWithSize(buffer);
builder.setSpatialIndex(new SpatialIndexColumnPartSupplier(rSpatialIndex));
}
}
private WritableSupplier<ColumnarInts> readSingleValuedColumn(VERSION version, ByteBuffer buffer) {
switch(version) {
case UNCOMPRESSED_SINGLE_VALUE:
case UNCOMPRESSED_WITH_FLAGS:
return VSizeColumnarInts.readFromByteBuffer(buffer);
case COMPRESSED:
return CompressedVSizeColumnarIntsSupplier.fromByteBuffer(buffer, byteOrder);
default:
throw new IAE("Unsupported single-value version[%s]", version);
}
}
private WritableSupplier<ColumnarMultiInts> readMultiValuedColumn(VERSION version, ByteBuffer buffer, int flags) {
switch(version) {
case UNCOMPRESSED_MULTI_VALUE:
{
return VSizeColumnarMultiInts.readFromByteBuffer(buffer);
}
case UNCOMPRESSED_WITH_FLAGS:
{
if (Feature.MULTI_VALUE.isSet(flags)) {
return VSizeColumnarMultiInts.readFromByteBuffer(buffer);
} else {
throw new IAE("Unrecognized multi-value flag[%d] for version[%s]", flags, version);
}
}
case COMPRESSED:
{
if (Feature.MULTI_VALUE.isSet(flags)) {
return CompressedVSizeColumnarMultiIntsSupplier.fromByteBuffer(buffer, byteOrder);
} else if (Feature.MULTI_VALUE_V3.isSet(flags)) {
return V3CompressedVSizeColumnarMultiIntsSupplier.fromByteBuffer(buffer, byteOrder);
} else {
throw new IAE("Unrecognized multi-value flag[%d] for version[%s]", flags, version);
}
}
default:
throw new IAE("Unsupported multi-value version[%s]", version);
}
}
};
}
use of org.apache.druid.segment.column.ColumnConfig in project druid by druid-io.
the class FloatNumericColumnPartSerde method getDeserializer.
@Override
public Deserializer getDeserializer() {
return new Deserializer() {
@Override
public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) {
final CompressedColumnarFloatsSupplier column = CompressedColumnarFloatsSupplier.fromByteBuffer(buffer, byteOrder);
FloatNumericColumnSupplier columnSupplier = new FloatNumericColumnSupplier(column, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap());
builder.setType(ValueType.FLOAT).setHasMultipleValues(false).setNumericColumnSupplier(columnSupplier);
}
};
}
use of org.apache.druid.segment.column.ColumnConfig in project druid by druid-io.
the class JoinAndLookupBenchmark method setup.
@Setup()
public void setup() throws IOException {
tmpDir = FileUtils.createTempDir();
ColumnConfig columnConfig = () -> columnCacheSizeBytes;
index = JoinTestHelper.createFactIndexBuilder(columnConfig, tmpDir, rows).buildMMappedIndex();
final String prefix = "c.";
baseSegment = new QueryableIndexSegment(index, SegmentId.dummy("join"));
List<JoinableClause> joinableClausesLookupStringKey = ImmutableList.of(new JoinableClause(prefix, LookupJoinable.wrap(JoinTestHelper.createCountryIsoCodeToNameLookup()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryIsoCode == \"%sk\"", prefix), prefix, ExprMacroTable.nil())));
JoinFilterPreAnalysis preAnalysisLookupStringKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesLookupStringKey, VirtualColumns.EMPTY, null));
hashJoinLookupStringKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesLookupStringKey, preAnalysisLookupStringKey);
List<JoinableClause> joinableClausesLookupLongKey = ImmutableList.of(new JoinableClause(prefix, LookupJoinable.wrap(JoinTestHelper.createCountryIsoCodeToNameLookup()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryIsoCode == \"%sk\"", prefix), prefix, ExprMacroTable.nil())));
JoinFilterPreAnalysis preAnalysisLookupLongKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesLookupLongKey, VirtualColumns.EMPTY, null));
hashJoinLookupLongKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesLookupLongKey, preAnalysisLookupLongKey);
List<JoinableClause> joinableClausesIndexedTableStringKey = ImmutableList.of(new JoinableClause(prefix, new IndexedTableJoinable(JoinTestHelper.createCountriesIndexedTable()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryIsoCode == \"%scountryIsoCode\"", prefix), prefix, ExprMacroTable.nil())));
JoinFilterPreAnalysis preAnalysisIndexedStringKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesLookupLongKey, VirtualColumns.EMPTY, null));
hashJoinIndexedTableStringKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesIndexedTableStringKey, preAnalysisIndexedStringKey);
List<JoinableClause> joinableClausesIndexedTableLongKey = ImmutableList.of(new JoinableClause(prefix, new IndexedTableJoinable(JoinTestHelper.createCountriesIndexedTable()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryNumber == \"%scountryNumber\"", prefix), prefix, ExprMacroTable.nil())));
JoinFilterPreAnalysis preAnalysisIndexedLongKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesIndexedTableLongKey, VirtualColumns.EMPTY, null));
hashJoinIndexedTableLongKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesIndexedTableLongKey, preAnalysisIndexedLongKey);
final Map<String, String> countryCodeToNameMap = JoinTestHelper.createCountryIsoCodeToNameLookup().getMap();
final Map<String, String> countryNumberToNameMap = JoinTestHelper.createCountryNumberToNameLookup().getMap();
final ExprMacroTable exprMacroTable = new ExprMacroTable(ImmutableList.of(new LookupExprMacro(new LookupExtractorFactoryContainerProvider() {
@Override
public Set<String> getAllLookupNames() {
return ImmutableSet.of(LOOKUP_COUNTRY_CODE_TO_NAME, LOOKUP_COUNTRY_NUMBER_TO_NAME);
}
@Override
public Optional<LookupExtractorFactoryContainer> get(String lookupName) {
if (LOOKUP_COUNTRY_CODE_TO_NAME.equals(lookupName)) {
return Optional.of(new LookupExtractorFactoryContainer("0", new MapLookupExtractorFactory(countryCodeToNameMap, false)));
} else if (LOOKUP_COUNTRY_NUMBER_TO_NAME.equals(lookupName)) {
return Optional.of(new LookupExtractorFactoryContainer("0", new MapLookupExtractorFactory(countryNumberToNameMap, false)));
} else {
return Optional.empty();
}
}
})));
lookupVirtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn(LOOKUP_COUNTRY_CODE_TO_NAME, "lookup(countryIsoCode, '" + LOOKUP_COUNTRY_CODE_TO_NAME + "')", ColumnType.STRING, exprMacroTable), new ExpressionVirtualColumn(LOOKUP_COUNTRY_NUMBER_TO_NAME, "lookup(countryNumber, '" + LOOKUP_COUNTRY_NUMBER_TO_NAME + "')", ColumnType.STRING, exprMacroTable)));
}
Aggregations