Search in sources :

Example 16 with ColumnCapabilities

use of org.apache.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class ExpressionVirtualColumnTest method testMultiObjectSelectorMakesRightSelector.

@Test
public void testMultiObjectSelectorMakesRightSelector() {
    DimensionSpec spec = new DefaultDimensionSpec("expr", "expr");
    // do some ugly faking to test if SingleStringInputDeferredEvaluationExpressionDimensionSelector is created for multi-value expressions when possible
    ColumnSelectorFactory factory = new ColumnSelectorFactory() {

        @Override
        public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) {
            DimensionSelector delegate = COLUMN_SELECTOR_FACTORY.makeDimensionSelector(dimensionSpec);
            DimensionSelector faker = new DimensionSelector() {

                @Override
                public IndexedInts getRow() {
                    return delegate.getRow();
                }

                @Override
                public ValueMatcher makeValueMatcher(@Nullable String value) {
                    return delegate.makeValueMatcher(value);
                }

                @Override
                public ValueMatcher makeValueMatcher(Predicate<String> predicate) {
                    return delegate.makeValueMatcher(predicate);
                }

                @Override
                public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
                    delegate.inspectRuntimeShape(inspector);
                }

                @Nullable
                @Override
                public Object getObject() {
                    return delegate.getObject();
                }

                @Override
                public Class<?> classOfObject() {
                    return delegate.classOfObject();
                }

                @Override
                public int getValueCardinality() {
                    // value doesn't matter as long as not CARDINALITY_UNKNOWN
                    return 3;
                }

                @Nullable
                @Override
                public String lookupName(int id) {
                    return null;
                }

                @Override
                public boolean nameLookupPossibleInAdvance() {
                    // fake this so when SingleStringInputDeferredEvaluationExpressionDimensionSelector it doesn't explode
                    return true;
                }

                @Nullable
                @Override
                public IdLookup idLookup() {
                    return name -> 0;
                }
            };
            return faker;
        }

        @Override
        public ColumnValueSelector makeColumnValueSelector(String columnName) {
            return COLUMN_SELECTOR_FACTORY.makeColumnValueSelector(columnName);
        }

        @Nullable
        @Override
        public ColumnCapabilities getColumnCapabilities(String column) {
            return new ColumnCapabilitiesImpl().setType(ColumnType.STRING).setHasMultipleValues(true).setDictionaryEncoded(true);
        }
    };
    final BaseObjectColumnValueSelector selectorImplicit = SCALE_LIST_SELF_IMPLICIT.makeDimensionSelector(spec, factory);
    final BaseObjectColumnValueSelector selectorExplicit = SCALE_LIST_SELF_EXPLICIT.makeDimensionSelector(spec, factory);
    Assert.assertTrue(selectorImplicit instanceof SingleStringInputDeferredEvaluationExpressionDimensionSelector);
    Assert.assertTrue(selectorExplicit instanceof ExpressionMultiValueDimensionSelector);
}
Also used : Arrays(java.util.Arrays) RuntimeShapeInspector(org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector) ConstantMultiValueDimensionSelector(org.apache.druid.segment.ConstantMultiValueDimensionSelector) MapBasedInputRow(org.apache.druid.data.input.MapBasedInputRow) ColumnValueSelector(org.apache.druid.segment.ColumnValueSelector) RowAdapters(org.apache.druid.segment.RowAdapters) Parser(org.apache.druid.math.expr.Parser) IdLookup(org.apache.druid.segment.IdLookup) IndexedInts(org.apache.druid.segment.data.IndexedInts) BaseFloatColumnValueSelector(org.apache.druid.segment.BaseFloatColumnValueSelector) Row(org.apache.druid.data.input.Row) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) ColumnSelectorFactory(org.apache.druid.segment.ColumnSelectorFactory) ImmutableList(com.google.common.collect.ImmutableList) Predicates(com.google.common.base.Predicates) DimensionSelector(org.apache.druid.segment.DimensionSelector) BucketExtractionFn(org.apache.druid.query.extraction.BucketExtractionFn) ExtractionDimensionSpec(org.apache.druid.query.dimension.ExtractionDimensionSpec) BaseObjectColumnValueSelector(org.apache.druid.segment.BaseObjectColumnValueSelector) Nullable(javax.annotation.Nullable) ValueMatcher(org.apache.druid.query.filter.ValueMatcher) DateTimes(org.apache.druid.java.util.common.DateTimes) RowBasedColumnSelectorFactory(org.apache.druid.segment.RowBasedColumnSelectorFactory) ImmutableMap(com.google.common.collect.ImmutableMap) ValueType(org.apache.druid.segment.column.ValueType) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test) TestExprMacroTable(org.apache.druid.query.expression.TestExprMacroTable) ExprEval(org.apache.druid.math.expr.ExprEval) InputRow(org.apache.druid.data.input.InputRow) BaseLongColumnValueSelector(org.apache.druid.segment.BaseLongColumnValueSelector) ColumnCapabilitiesImpl(org.apache.druid.segment.column.ColumnCapabilitiesImpl) Predicate(com.google.common.base.Predicate) NullHandling(org.apache.druid.common.config.NullHandling) RowSignature(org.apache.druid.segment.column.RowSignature) DimensionSpec(org.apache.druid.query.dimension.DimensionSpec) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) ColumnType(org.apache.druid.segment.column.ColumnType) Assert(org.junit.Assert) ConstantDimensionSelector(org.apache.druid.segment.ConstantDimensionSelector) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) ExtractionDimensionSpec(org.apache.druid.query.dimension.ExtractionDimensionSpec) DimensionSpec(org.apache.druid.query.dimension.DimensionSpec) ConstantMultiValueDimensionSelector(org.apache.druid.segment.ConstantMultiValueDimensionSelector) DimensionSelector(org.apache.druid.segment.DimensionSelector) ConstantDimensionSelector(org.apache.druid.segment.ConstantDimensionSelector) ColumnSelectorFactory(org.apache.druid.segment.ColumnSelectorFactory) RowBasedColumnSelectorFactory(org.apache.druid.segment.RowBasedColumnSelectorFactory) RuntimeShapeInspector(org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector) BaseObjectColumnValueSelector(org.apache.druid.segment.BaseObjectColumnValueSelector) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) Predicate(com.google.common.base.Predicate) Nullable(javax.annotation.Nullable) ColumnCapabilitiesImpl(org.apache.druid.segment.column.ColumnCapabilitiesImpl) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 17 with ColumnCapabilities

use of org.apache.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class ExpressionVectorSelectorsTest method sanityTestVectorizedExpressionSelectors.

public static void sanityTestVectorizedExpressionSelectors(String expression, @Nullable ExpressionType outputType, QueryableIndex index, Closer closer, int rowsPerSegment) {
    final List<Object> results = new ArrayList<>(rowsPerSegment);
    final VirtualColumns virtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v", expression, ExpressionType.toColumnType(outputType), TestExprMacroTable.INSTANCE)));
    final QueryableIndexStorageAdapter storageAdapter = new QueryableIndexStorageAdapter(index);
    VectorCursor cursor = storageAdapter.makeVectorCursor(null, index.getDataInterval(), virtualColumns, false, 512, null);
    ColumnCapabilities capabilities = virtualColumns.getColumnCapabilities(storageAdapter, "v");
    int rowCount = 0;
    if (capabilities.isDictionaryEncoded().isTrue()) {
        SingleValueDimensionVectorSelector selector = cursor.getColumnSelectorFactory().makeSingleValueDimensionSelector(DefaultDimensionSpec.of("v"));
        while (!cursor.isDone()) {
            int[] row = selector.getRowVector();
            for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                results.add(selector.lookupName(row[i]));
            }
            cursor.advance();
        }
    } else {
        VectorValueSelector selector = null;
        VectorObjectSelector objectSelector = null;
        if (outputType != null && outputType.isNumeric()) {
            selector = cursor.getColumnSelectorFactory().makeValueSelector("v");
        } else {
            objectSelector = cursor.getColumnSelectorFactory().makeObjectSelector("v");
        }
        while (!cursor.isDone()) {
            boolean[] nulls;
            switch(outputType.getType()) {
                case LONG:
                    nulls = selector.getNullVector();
                    long[] longs = selector.getLongVector();
                    for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                        results.add(nulls != null && nulls[i] ? null : longs[i]);
                    }
                    break;
                case DOUBLE:
                    // special case to test floats just to get coverage on getFloatVector
                    if ("float2".equals(expression)) {
                        nulls = selector.getNullVector();
                        float[] floats = selector.getFloatVector();
                        for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                            results.add(nulls != null && nulls[i] ? null : (double) floats[i]);
                        }
                    } else {
                        nulls = selector.getNullVector();
                        double[] doubles = selector.getDoubleVector();
                        for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                            results.add(nulls != null && nulls[i] ? null : doubles[i]);
                        }
                    }
                    break;
                case STRING:
                    Object[] objects = objectSelector.getObjectVector();
                    for (int i = 0; i < objectSelector.getCurrentVectorSize(); i++, rowCount++) {
                        results.add(objects[i]);
                    }
                    break;
            }
            cursor.advance();
        }
    }
    closer.register(cursor);
    Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(null, index.getDataInterval(), virtualColumns, Granularities.ALL, false, null);
    int rowCountCursor = cursors.map(nonVectorized -> {
        final ColumnValueSelector nonSelector = nonVectorized.getColumnSelectorFactory().makeColumnValueSelector("v");
        int rows = 0;
        while (!nonVectorized.isDone()) {
            Assert.assertEquals(StringUtils.format("Failed at row %s", rows), nonSelector.getObject(), results.get(rows));
            rows++;
            nonVectorized.advance();
        }
        return rows;
    }).accumulate(0, (acc, in) -> acc + in);
    Assert.assertTrue(rowCountCursor > 0);
    Assert.assertEquals(rowCountCursor, rowCount);
}
Also used : BeforeClass(org.junit.BeforeClass) SegmentGenerator(org.apache.druid.segment.generator.SegmentGenerator) RunWith(org.junit.runner.RunWith) ColumnValueSelector(org.apache.druid.segment.ColumnValueSelector) Parser(org.apache.druid.math.expr.Parser) ArrayList(java.util.ArrayList) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) GeneratorBasicSchemas(org.apache.druid.segment.generator.GeneratorBasicSchemas) VectorCursor(org.apache.druid.segment.vector.VectorCursor) ImmutableList(com.google.common.collect.ImmutableList) ExpressionType(org.apache.druid.math.expr.ExpressionType) Expr(org.apache.druid.math.expr.Expr) SingleValueDimensionVectorSelector(org.apache.druid.segment.vector.SingleValueDimensionVectorSelector) Parameterized(org.junit.runners.Parameterized) Nullable(javax.annotation.Nullable) Before(org.junit.Before) Sequence(org.apache.druid.java.util.common.guava.Sequence) ColumnInspector(org.apache.druid.segment.ColumnInspector) AfterClass(org.junit.AfterClass) QueryableIndexStorageAdapter(org.apache.druid.segment.QueryableIndexStorageAdapter) VirtualColumns(org.apache.druid.segment.VirtualColumns) Closer(org.apache.druid.java.util.common.io.Closer) QueryableIndex(org.apache.druid.segment.QueryableIndex) StringUtils(org.apache.druid.java.util.common.StringUtils) GeneratorSchemaInfo(org.apache.druid.segment.generator.GeneratorSchemaInfo) VectorObjectSelector(org.apache.druid.segment.vector.VectorObjectSelector) Test(org.junit.Test) IOException(java.io.IOException) TestExprMacroTable(org.apache.druid.query.expression.TestExprMacroTable) VectorValueSelector(org.apache.druid.segment.vector.VectorValueSelector) Collectors(java.util.stream.Collectors) Granularities(org.apache.druid.java.util.common.granularity.Granularities) ExprMacroTable(org.apache.druid.math.expr.ExprMacroTable) List(java.util.List) Cursor(org.apache.druid.segment.Cursor) LinearShardSpec(org.apache.druid.timeline.partition.LinearShardSpec) DataSegment(org.apache.druid.timeline.DataSegment) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) Assert(org.junit.Assert) ArrayList(java.util.ArrayList) QueryableIndexStorageAdapter(org.apache.druid.segment.QueryableIndexStorageAdapter) VectorCursor(org.apache.druid.segment.vector.VectorCursor) Cursor(org.apache.druid.segment.Cursor) VectorCursor(org.apache.druid.segment.vector.VectorCursor) ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) VectorValueSelector(org.apache.druid.segment.vector.VectorValueSelector) SingleValueDimensionVectorSelector(org.apache.druid.segment.vector.SingleValueDimensionVectorSelector) VectorObjectSelector(org.apache.druid.segment.vector.VectorObjectSelector) VirtualColumns(org.apache.druid.segment.VirtualColumns) ColumnValueSelector(org.apache.druid.segment.ColumnValueSelector)

Example 18 with ColumnCapabilities

use of org.apache.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class ExpressionVirtualColumnTest method testCapabilities.

@Test
public void testCapabilities() {
    ColumnCapabilities caps = X_PLUS_Y.capabilities("expr");
    Assert.assertEquals(ValueType.FLOAT, caps.getType());
    Assert.assertFalse(caps.hasBitmapIndexes());
    Assert.assertFalse(caps.isDictionaryEncoded().isTrue());
    Assert.assertFalse(caps.areDictionaryValuesSorted().isTrue());
    Assert.assertFalse(caps.areDictionaryValuesUnique().isTrue());
    Assert.assertTrue(caps.hasMultipleValues().isUnknown());
    Assert.assertTrue(caps.hasMultipleValues().isMaybeTrue());
    Assert.assertFalse(caps.hasSpatialIndexes());
    caps = Z_CONCAT_X.capabilities("expr");
    Assert.assertEquals(ValueType.STRING, caps.getType());
    Assert.assertFalse(caps.hasBitmapIndexes());
    Assert.assertFalse(caps.isDictionaryEncoded().isTrue());
    Assert.assertFalse(caps.areDictionaryValuesSorted().isTrue());
    Assert.assertFalse(caps.areDictionaryValuesUnique().isTrue());
    Assert.assertTrue(caps.hasMultipleValues().isUnknown());
    Assert.assertTrue(caps.hasMultipleValues().isMaybeTrue());
    Assert.assertFalse(caps.hasSpatialIndexes());
}
Also used : ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 19 with ColumnCapabilities

use of org.apache.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class HashJoinSegmentStorageAdapterTest method test_getColumnCapabilities_factToCountryNonexistentFactColumn.

@Test
public void test_getColumnCapabilities_factToCountryNonexistentFactColumn() {
    final ColumnCapabilities capabilities = makeFactToCountrySegment().getColumnCapabilities("nonexistent");
    Assert.assertNull(capabilities);
}
Also used : ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) Test(org.junit.Test)

Example 20 with ColumnCapabilities

use of org.apache.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class ExpressionPlannerTest method testScalarStringNondictionaryEncoded.

@Test
public void testScalarStringNondictionaryEncoded() {
    ExpressionPlan thePlan = plan("concat(scalar_string, 'x')");
    Assert.assertTrue(thePlan.is(ExpressionPlan.Trait.VECTORIZABLE));
    Assert.assertFalse(thePlan.is(ExpressionPlan.Trait.SINGLE_INPUT_SCALAR, ExpressionPlan.Trait.SINGLE_INPUT_MAPPABLE, ExpressionPlan.Trait.INCOMPLETE_INPUTS, ExpressionPlan.Trait.UNKNOWN_INPUTS, ExpressionPlan.Trait.NEEDS_APPLIED, ExpressionPlan.Trait.NON_SCALAR_INPUTS, ExpressionPlan.Trait.NON_SCALAR_OUTPUT));
    Assert.assertEquals("concat(\"scalar_string\", 'x')", thePlan.getAppliedExpression().stringify());
    Assert.assertEquals("concat(\"scalar_string\", 'x')", thePlan.getAppliedFoldExpression("__acc").stringify());
    Assert.assertEquals(ExpressionType.STRING, thePlan.getOutputType());
    ColumnCapabilities inferred = thePlan.inferColumnCapabilities(null);
    Assert.assertNotNull(inferred);
    Assert.assertEquals(ValueType.STRING, inferred.getType());
    Assert.assertTrue(inferred.hasNulls().isTrue());
    Assert.assertFalse(inferred.isDictionaryEncoded().isMaybeTrue());
    Assert.assertFalse(inferred.areDictionaryValuesSorted().isMaybeTrue());
    Assert.assertFalse(inferred.areDictionaryValuesUnique().isMaybeTrue());
    Assert.assertFalse(inferred.hasMultipleValues().isMaybeTrue());
    Assert.assertFalse(inferred.hasBitmapIndexes());
    Assert.assertFalse(inferred.hasSpatialIndexes());
}
Also used : ColumnCapabilities(org.apache.druid.segment.column.ColumnCapabilities) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Aggregations

ColumnCapabilities (org.apache.druid.segment.column.ColumnCapabilities)78 Test (org.junit.Test)52 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)29 ColumnCapabilitiesImpl (org.apache.druid.segment.column.ColumnCapabilitiesImpl)9 ArrayList (java.util.ArrayList)4 Nullable (javax.annotation.Nullable)4 AggregatorFactory (org.apache.druid.query.aggregation.AggregatorFactory)4 DimensionSpec (org.apache.druid.query.dimension.DimensionSpec)4 ColumnValueSelector (org.apache.druid.segment.ColumnValueSelector)4 QueryableIndex (org.apache.druid.segment.QueryableIndex)4 ColumnHolder (org.apache.druid.segment.column.ColumnHolder)4 ColumnType (org.apache.druid.segment.column.ColumnType)4 RowSignature (org.apache.druid.segment.column.RowSignature)4 ValueType (org.apache.druid.segment.column.ValueType)4 List (java.util.List)3 NullHandlingTest (org.apache.druid.common.config.NullHandlingTest)3 ISE (org.apache.druid.java.util.common.ISE)3 DefaultDimensionSpec (org.apache.druid.query.dimension.DefaultDimensionSpec)3 VectorValueSelector (org.apache.druid.segment.vector.VectorValueSelector)3 Before (org.junit.Before)3