Search in sources :

Example 6 with ColumnInspector

use of org.apache.druid.segment.ColumnInspector in project druid by druid-io.

the class RowBasedGrouperHelper method createResultRowBasedColumnSelectorFactory.

/**
 * Creates a {@link ColumnSelectorFactory} that can read rows which originate as results of the provided "query".
 *
 * @param query        a groupBy query
 * @param supplier     supplier of result rows from the query
 * @param finalization whether the column capabilities reported by this factory should reflect finalized types
 */
public static ColumnSelectorFactory createResultRowBasedColumnSelectorFactory(final GroupByQuery query, final Supplier<ResultRow> supplier, final RowSignature.Finalization finalization) {
    final RowSignature signature = query.getResultRowSignature(finalization);
    final RowAdapter<ResultRow> adapter = new RowAdapter<ResultRow>() {

        @Override
        public ToLongFunction<ResultRow> timestampFunction() {
            if (query.getResultRowHasTimestamp()) {
                return row -> row.getLong(0);
            } else {
                final long timestamp = query.getUniversalTimestamp().getMillis();
                return row -> timestamp;
            }
        }

        @Override
        public Function<ResultRow, Object> columnFunction(final String columnName) {
            final int columnIndex = signature.indexOf(columnName);
            if (columnIndex < 0) {
                return row -> null;
            } else {
                return row -> row.get(columnIndex);
            }
        }
    };
    // Decorate "signature" so that it returns hasMultipleValues = false. (groupBy does not return multiple values.)
    final ColumnInspector decoratedSignature = new ColumnInspector() {

        @Nullable
        @Override
        public ColumnCapabilities getColumnCapabilities(String column) {
            final ColumnCapabilities baseCapabilities = signature.getColumnCapabilities(column);
            if (baseCapabilities == null || baseCapabilities.hasMultipleValues().isFalse()) {
                return baseCapabilities;
            } else {
                return ColumnCapabilitiesImpl.copyOf(baseCapabilities).setHasMultipleValues(false);
            }
        }
    };
    return RowBasedColumnSelectorFactory.create(adapter, supplier::get, decoratedSignature, false);
}
Also used : ResultRow(org.apache.druid.query.groupby.ResultRow) Arrays(java.util.Arrays) Comparators(org.apache.druid.java.util.common.guava.Comparators) IntArrayUtils(org.apache.druid.common.utils.IntArrayUtils) DimensionHandlerUtils(org.apache.druid.segment.DimensionHandlerUtils) ColumnValueSelector(org.apache.druid.segment.ColumnValueSelector) AllGranularity(org.apache.druid.java.util.common.granularity.AllGranularity) IndexedInts(org.apache.druid.segment.data.IndexedInts) ByteBuffer(java.nio.ByteBuffer) Pair(org.apache.druid.java.util.common.Pair) DefaultLimitSpec(org.apache.druid.query.groupby.orderby.DefaultLimitSpec) BaseFloatColumnValueSelector(org.apache.druid.segment.BaseFloatColumnValueSelector) OrderByColumnSpec(org.apache.druid.query.groupby.orderby.OrderByColumnSpec) ColumnSelectorFactory(org.apache.druid.segment.ColumnSelectorFactory) RowAdapter(org.apache.druid.segment.RowAdapter) ColumnSelectorStrategyFactory(org.apache.druid.query.dimension.ColumnSelectorStrategyFactory) JsonValue(com.fasterxml.jackson.annotation.JsonValue) GroupingAggregatorFactory(org.apache.druid.query.aggregation.GroupingAggregatorFactory) BufferComparator(org.apache.druid.query.groupby.epinephelinae.Grouper.BufferComparator) Object2IntOpenHashMap(it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap) IAE(org.apache.druid.java.util.common.IAE) ToLongFunction(java.util.function.ToLongFunction) Longs(com.google.common.primitives.Longs) RowBasedColumnSelectorFactory(org.apache.druid.segment.RowBasedColumnSelectorFactory) ResultRow(org.apache.druid.query.groupby.ResultRow) Predicate(java.util.function.Predicate) AggregatorFactory(org.apache.druid.query.aggregation.AggregatorFactory) Set(java.util.Set) ISE(org.apache.druid.java.util.common.ISE) ValueType(org.apache.druid.segment.column.ValueType) Collectors(java.util.stream.Collectors) List(java.util.List) ColumnCapabilitiesImpl(org.apache.druid.segment.column.ColumnCapabilitiesImpl) BooleanValueMatcher(org.apache.druid.segment.filter.BooleanValueMatcher) DimensionSpec(org.apache.druid.query.dimension.DimensionSpec) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) BaseDoubleColumnValueSelector(org.apache.druid.segment.BaseDoubleColumnValueSelector) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Accumulator(org.apache.druid.java.util.common.guava.Accumulator) IntStream(java.util.stream.IntStream) ColumnSelectorPlus(org.apache.druid.query.ColumnSelectorPlus) ComparableList(org.apache.druid.segment.data.ComparableList) Supplier(com.google.common.base.Supplier) BaseQuery(org.apache.druid.query.BaseQuery) Function(java.util.function.Function) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Interval(org.joda.time.Interval) SettableSupplier(org.apache.druid.common.guava.SettableSupplier) ColumnSelectorStrategy(org.apache.druid.query.dimension.ColumnSelectorStrategy) StringComparators(org.apache.druid.query.ordering.StringComparators) ComparableStringArray(org.apache.druid.segment.data.ComparableStringArray) GroupByQuery(org.apache.druid.query.groupby.GroupByQuery) DimensionSelector(org.apache.druid.segment.DimensionSelector) Nullable(javax.annotation.Nullable) ValueMatcher(org.apache.druid.query.filter.ValueMatcher) ColumnInspector(org.apache.druid.segment.ColumnInspector) StringComparator(org.apache.druid.query.ordering.StringComparator) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) GroupByQueryConfig(org.apache.druid.query.groupby.GroupByQueryConfig) DateTime(org.joda.time.DateTime) Ints(com.google.common.primitives.Ints) BaseLongColumnValueSelector(org.apache.druid.segment.BaseLongColumnValueSelector) Object2IntMap(it.unimi.dsi.fastutil.objects.Object2IntMap) NullHandling(org.apache.druid.common.config.NullHandling) RowSignature(org.apache.druid.segment.column.RowSignature) Closeable(java.io.Closeable) JsonCreator(com.fasterxml.jackson.annotation.JsonCreator) ColumnType(org.apache.druid.segment.column.ColumnType) Preconditions(com.google.common.base.Preconditions) BitSet(java.util.BitSet) IntArrays(it.unimi.dsi.fastutil.ints.IntArrays) Comparator(java.util.Comparator) Filters(org.apache.druid.segment.filter.Filters) ReferenceCountingResourceHolder(org.apache.druid.collections.ReferenceCountingResourceHolder) Filter(org.apache.druid.query.filter.Filter) RowAdapter(org.apache.druid.segment.RowAdapter) ColumnInspector(org.apache.druid.segment.ColumnInspector) RowSignature(org.apache.druid.segment.column.RowSignature) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities)

Example 7 with ColumnInspector

use of org.apache.druid.segment.ColumnInspector in project druid by druid-io.

the class ExpressionVectorSelectorBenchmark method setup.

@Setup(Level.Trial)
public void setup() {
    this.closer = Closer.create();
    final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench");
    final DataSegment dataSegment = DataSegment.builder().dataSource("foo").interval(schemaInfo.getDataInterval()).version("1").shardSpec(new LinearShardSpec(0)).size(0).build();
    final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
    this.index = closer.register(segmentGenerator.generate(dataSegment, schemaInfo, Granularities.HOUR, rowsPerSegment));
    Expr parsed = Parser.parse(expression, ExprMacroTable.nil());
    outputType = parsed.getOutputType(new ColumnInspector() {

        @Nullable
        @Override
        public ColumnCapabilities getColumnCapabilities(String column) {
            return QueryableIndexStorageAdapter.getColumnCapabilities(index, column);
        }
    });
    checkSanity();
}
Also used : SegmentGenerator(org.apache.druid.segment.generator.SegmentGenerator) Expr(org.apache.druid.math.expr.Expr) LinearShardSpec(org.apache.druid.timeline.partition.LinearShardSpec) GeneratorSchemaInfo(org.apache.druid.segment.generator.GeneratorSchemaInfo) ColumnInspector(org.apache.druid.segment.ColumnInspector) DataSegment(org.apache.druid.timeline.DataSegment) Setup(org.openjdk.jmh.annotations.Setup)

Example 8 with ColumnInspector

use of org.apache.druid.segment.ColumnInspector in project druid by druid-io.

the class VirtualColumnsTest method testGetColumnCapabilitiesDoubleBase.

@Test
public void testGetColumnCapabilitiesDoubleBase() {
    final VirtualColumns virtualColumns = makeVirtualColumns();
    final ColumnInspector baseInspector = column -> {
        if (REAL_COLUMN_NAME.equals(column)) {
            return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.DOUBLE);
        } else {
            return null;
        }
    };
    Assert.assertEquals(ValueType.FLOAT, virtualColumns.getColumnCapabilities(baseInspector, "expr").getType());
    Assert.assertEquals(ValueType.DOUBLE, virtualColumns.getColumnCapabilities(baseInspector, "expr2").getType());
    Assert.assertNull(virtualColumns.getColumnCapabilities(baseInspector, REAL_COLUMN_NAME));
}
Also used : Arrays(java.util.Arrays) RuntimeShapeInspector(org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector) ExtractionFn(org.apache.druid.query.extraction.ExtractionFn) ColumnValueSelector(org.apache.druid.segment.ColumnValueSelector) IdLookup(org.apache.druid.segment.IdLookup) IndexedInts(org.apache.druid.segment.data.IndexedInts) BaseFloatColumnValueSelector(org.apache.druid.segment.BaseFloatColumnValueSelector) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) ColumnSelectorFactory(org.apache.druid.segment.ColumnSelectorFactory) ImmutableList(com.google.common.collect.ImmutableList) DimensionSelector(org.apache.druid.segment.DimensionSelector) TestLongColumnSelector(org.apache.druid.segment.TestLongColumnSelector) BucketExtractionFn(org.apache.druid.query.extraction.BucketExtractionFn) ExtractionDimensionSpec(org.apache.druid.query.dimension.ExtractionDimensionSpec) BaseObjectColumnValueSelector(org.apache.druid.segment.BaseObjectColumnValueSelector) ExpectedException(org.junit.rules.ExpectedException) Nullable(javax.annotation.Nullable) ValueMatcher(org.apache.druid.query.filter.ValueMatcher) ColumnInspector(org.apache.druid.segment.ColumnInspector) Longs(com.google.common.primitives.Longs) VirtualColumns(org.apache.druid.segment.VirtualColumns) VirtualColumn(org.apache.druid.segment.VirtualColumn) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ValueType(org.apache.druid.segment.column.ValueType) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test) TestExprMacroTable(org.apache.druid.query.expression.TestExprMacroTable) DimensionDictionarySelector(org.apache.druid.segment.DimensionDictionarySelector) DimensionSelectorUtils(org.apache.druid.segment.DimensionSelectorUtils) ZeroIndexedInts(org.apache.druid.segment.data.ZeroIndexedInts) TestHelper(org.apache.druid.segment.TestHelper) List(java.util.List) BaseLongColumnValueSelector(org.apache.druid.segment.BaseLongColumnValueSelector) ColumnCapabilitiesImpl(org.apache.druid.segment.column.ColumnCapabilitiesImpl) Rule(org.junit.Rule) Predicate(com.google.common.base.Predicate) DimensionSpec(org.apache.druid.query.dimension.DimensionSpec) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) ColumnType(org.apache.druid.segment.column.ColumnType) Assert(org.junit.Assert) ColumnInspector(org.apache.druid.segment.ColumnInspector) VirtualColumns(org.apache.druid.segment.VirtualColumns) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 9 with ColumnInspector

use of org.apache.druid.segment.ColumnInspector in project druid by druid-io.

the class VirtualColumnsTest method testWrapInspectorDoubleBase.

@Test
public void testWrapInspectorDoubleBase() {
    final VirtualColumns virtualColumns = makeVirtualColumns();
    final ColumnInspector baseInspector = column -> {
        if (REAL_COLUMN_NAME.equals(column)) {
            return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.DOUBLE);
        } else {
            return null;
        }
    };
    final ColumnInspector wrappedInspector = virtualColumns.wrapInspector(baseInspector);
    Assert.assertEquals(ValueType.FLOAT, wrappedInspector.getColumnCapabilities("expr").getType());
    Assert.assertEquals(ValueType.DOUBLE, wrappedInspector.getColumnCapabilities("expr2").getType());
    Assert.assertEquals(ValueType.DOUBLE, wrappedInspector.getColumnCapabilities(REAL_COLUMN_NAME).getType());
}
Also used : Arrays(java.util.Arrays) RuntimeShapeInspector(org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector) ExtractionFn(org.apache.druid.query.extraction.ExtractionFn) ColumnValueSelector(org.apache.druid.segment.ColumnValueSelector) IdLookup(org.apache.druid.segment.IdLookup) IndexedInts(org.apache.druid.segment.data.IndexedInts) BaseFloatColumnValueSelector(org.apache.druid.segment.BaseFloatColumnValueSelector) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) ColumnSelectorFactory(org.apache.druid.segment.ColumnSelectorFactory) ImmutableList(com.google.common.collect.ImmutableList) DimensionSelector(org.apache.druid.segment.DimensionSelector) TestLongColumnSelector(org.apache.druid.segment.TestLongColumnSelector) BucketExtractionFn(org.apache.druid.query.extraction.BucketExtractionFn) ExtractionDimensionSpec(org.apache.druid.query.dimension.ExtractionDimensionSpec) BaseObjectColumnValueSelector(org.apache.druid.segment.BaseObjectColumnValueSelector) ExpectedException(org.junit.rules.ExpectedException) Nullable(javax.annotation.Nullable) ValueMatcher(org.apache.druid.query.filter.ValueMatcher) ColumnInspector(org.apache.druid.segment.ColumnInspector) Longs(com.google.common.primitives.Longs) VirtualColumns(org.apache.druid.segment.VirtualColumns) VirtualColumn(org.apache.druid.segment.VirtualColumn) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ValueType(org.apache.druid.segment.column.ValueType) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test) TestExprMacroTable(org.apache.druid.query.expression.TestExprMacroTable) DimensionDictionarySelector(org.apache.druid.segment.DimensionDictionarySelector) DimensionSelectorUtils(org.apache.druid.segment.DimensionSelectorUtils) ZeroIndexedInts(org.apache.druid.segment.data.ZeroIndexedInts) TestHelper(org.apache.druid.segment.TestHelper) List(java.util.List) BaseLongColumnValueSelector(org.apache.druid.segment.BaseLongColumnValueSelector) ColumnCapabilitiesImpl(org.apache.druid.segment.column.ColumnCapabilitiesImpl) Rule(org.junit.Rule) Predicate(com.google.common.base.Predicate) DimensionSpec(org.apache.druid.query.dimension.DimensionSpec) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) ColumnType(org.apache.druid.segment.column.ColumnType) Assert(org.junit.Assert) ColumnInspector(org.apache.druid.segment.ColumnInspector) VirtualColumns(org.apache.druid.segment.VirtualColumns) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 10 with ColumnInspector

use of org.apache.druid.segment.ColumnInspector in project druid by druid-io.

the class VirtualColumnsTest method testGetColumnCapabilitiesWithFallbackDoubleBase.

@Test
public void testGetColumnCapabilitiesWithFallbackDoubleBase() {
    final VirtualColumns virtualColumns = makeVirtualColumns();
    final ColumnInspector baseInspector = column -> {
        if (REAL_COLUMN_NAME.equals(column)) {
            return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.DOUBLE);
        } else {
            return null;
        }
    };
    Assert.assertEquals(ValueType.FLOAT, virtualColumns.getColumnCapabilitiesWithFallback(baseInspector, "expr").getType());
    Assert.assertEquals(ValueType.DOUBLE, virtualColumns.getColumnCapabilitiesWithFallback(baseInspector, "expr2").getType());
    Assert.assertEquals(ValueType.DOUBLE, virtualColumns.getColumnCapabilitiesWithFallback(baseInspector, REAL_COLUMN_NAME).getType());
}
Also used : Arrays(java.util.Arrays) RuntimeShapeInspector(org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector) ExtractionFn(org.apache.druid.query.extraction.ExtractionFn) ColumnValueSelector(org.apache.druid.segment.ColumnValueSelector) IdLookup(org.apache.druid.segment.IdLookup) IndexedInts(org.apache.druid.segment.data.IndexedInts) BaseFloatColumnValueSelector(org.apache.druid.segment.BaseFloatColumnValueSelector) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) ColumnSelectorFactory(org.apache.druid.segment.ColumnSelectorFactory) ImmutableList(com.google.common.collect.ImmutableList) DimensionSelector(org.apache.druid.segment.DimensionSelector) TestLongColumnSelector(org.apache.druid.segment.TestLongColumnSelector) BucketExtractionFn(org.apache.druid.query.extraction.BucketExtractionFn) ExtractionDimensionSpec(org.apache.druid.query.dimension.ExtractionDimensionSpec) BaseObjectColumnValueSelector(org.apache.druid.segment.BaseObjectColumnValueSelector) ExpectedException(org.junit.rules.ExpectedException) Nullable(javax.annotation.Nullable) ValueMatcher(org.apache.druid.query.filter.ValueMatcher) ColumnInspector(org.apache.druid.segment.ColumnInspector) Longs(com.google.common.primitives.Longs) VirtualColumns(org.apache.druid.segment.VirtualColumns) VirtualColumn(org.apache.druid.segment.VirtualColumn) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ValueType(org.apache.druid.segment.column.ValueType) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test) TestExprMacroTable(org.apache.druid.query.expression.TestExprMacroTable) DimensionDictionarySelector(org.apache.druid.segment.DimensionDictionarySelector) DimensionSelectorUtils(org.apache.druid.segment.DimensionSelectorUtils) ZeroIndexedInts(org.apache.druid.segment.data.ZeroIndexedInts) TestHelper(org.apache.druid.segment.TestHelper) List(java.util.List) BaseLongColumnValueSelector(org.apache.druid.segment.BaseLongColumnValueSelector) ColumnCapabilitiesImpl(org.apache.druid.segment.column.ColumnCapabilitiesImpl) Rule(org.junit.Rule) Predicate(com.google.common.base.Predicate) DimensionSpec(org.apache.druid.query.dimension.DimensionSpec) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) ColumnType(org.apache.druid.segment.column.ColumnType) Assert(org.junit.Assert) ColumnInspector(org.apache.druid.segment.ColumnInspector) VirtualColumns(org.apache.druid.segment.VirtualColumns) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Aggregations

ColumnInspector (org.apache.druid.segment.ColumnInspector)11 ColumnSelectorFactory (org.apache.druid.segment.ColumnSelectorFactory)8 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)7 Longs (com.google.common.primitives.Longs)7 Arrays (java.util.Arrays)7 List (java.util.List)7 Nullable (javax.annotation.Nullable)7 DefaultDimensionSpec (org.apache.druid.query.dimension.DefaultDimensionSpec)7 DimensionSpec (org.apache.druid.query.dimension.DimensionSpec)7 ValueMatcher (org.apache.druid.query.filter.ValueMatcher)7 BaseFloatColumnValueSelector (org.apache.druid.segment.BaseFloatColumnValueSelector)7 BaseLongColumnValueSelector (org.apache.druid.segment.BaseLongColumnValueSelector)7 ColumnValueSelector (org.apache.druid.segment.ColumnValueSelector)7 DimensionSelector (org.apache.druid.segment.DimensionSelector)7 ColumnCapabilities (org.apache.druid.segment.column.ColumnCapabilities)7 ColumnCapabilitiesImpl (org.apache.druid.segment.column.ColumnCapabilitiesImpl)7 ColumnType (org.apache.druid.segment.column.ColumnType)7 ValueType (org.apache.druid.segment.column.ValueType)7 IndexedInts (org.apache.druid.segment.data.IndexedInts)7 Predicate (com.google.common.base.Predicate)6