Examples with VirtualColumns org.apache.druid.segment.VirtualColumns used on opensource projects

Example 1 with VirtualColumns

use of org.apache.druid.segment.VirtualColumns in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionFilterOnRHSRegionNameExprVirtualColumn.

@Test
public void test_filterPushDown_factToRegionFilterOnRHSRegionNameExprVirtualColumn() {
    // If our query had a filter that uses expressions, such as upper(r1.regionName) == 'VIRGINIA', this plans into
    // a selector filter on an ExpressionVirtualColumn
    Filter originalFilter = new SelectorFilter("v0", "VIRGINIA");
    VirtualColumns virtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v0", "upper(\"r1.regionName\")", ColumnType.STRING, TestExprMacroTable.INSTANCE)));
    JoinableClauses joinableClauses = JoinableClauses.fromList(ImmutableList.of(factToRegion(JoinType.LEFT)));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(originalFilter, joinableClauses.getJoinableClauses(), virtualColumns);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses.getJoinableClauses(), joinFilterPreAnalysis);
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, virtualColumns, Granularities.ALL, false, null), ImmutableList.of("page", "v0"), ImmutableList.of(new Object[] { "Old Anatolian Turkish", "VIRGINIA" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(null, new SelectorFilter("v0", "VIRGINIA"), ImmutableSet.of());
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 2 with VirtualColumns

use of org.apache.druid.segment.VirtualColumns in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftFilterOnChannelVirtualColumn.

@Test
public void test_filterPushDown_factToRegionToCountryLeftFilterOnChannelVirtualColumn() {
    List<JoinableClause> joinableClauses = ImmutableList.of(factToRegion(JoinType.LEFT), regionToCountry(JoinType.LEFT));
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("v1", "virtual-column-#en.wikipedia")));
    VirtualColumns virtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v1", "concat('virtual-column-', \"channel\")", ColumnType.STRING, TestExprMacroTable.INSTANCE)));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(originalFilter, joinableClauses, virtualColumns);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis);
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, virtualColumns, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Talk:Oswald Tilghman", null, null }, new Object[] { "Peremptory norm", "New South Wales", "Australia" }, new Object[] { "President of India", "California", "United States" }, new Object[] { "Glasgow", "Kingston upon Hull", "United Kingdom" }, new Object[] { "Otjiwarongo Airport", "California", "United States" }, new Object[] { "Sarah Michelle Gellar", "Ontario", "Canada" }, new Object[] { "DirecTV", "North Carolina", "United States" }, new Object[] { "Carlo Curti", "California", "United States" }, new Object[] { "Giusy Ferreri discography", "Provincia di Varese", "Italy" }, new Object[] { "Roma-Bangkok", "Provincia di Varese", "Italy" }, new Object[] { "Old Anatolian Turkish", "Virginia", "United States" }, new Object[] { "Cream Soda", "Ainigriv", "States United" }, new Object[] { "Orange Soda", null, null }, new Object[] { "History of Fourems", "Fourems Province", "Fourems" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new SelectorFilter("v1", "virtual-column-#en.wikipedia"), null, ImmutableSet.of());
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 3 with VirtualColumns

use of org.apache.druid.segment.VirtualColumns in project druid by druid-io.

the class ExpressionVectorSelectorsTest method sanityTestVectorizedExpressionSelectors.

public static void sanityTestVectorizedExpressionSelectors(String expression, @Nullable ExpressionType outputType, QueryableIndex index, Closer closer, int rowsPerSegment) {
    final List<Object> results = new ArrayList<>(rowsPerSegment);
    final VirtualColumns virtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v", expression, ExpressionType.toColumnType(outputType), TestExprMacroTable.INSTANCE)));
    final QueryableIndexStorageAdapter storageAdapter = new QueryableIndexStorageAdapter(index);
    VectorCursor cursor = storageAdapter.makeVectorCursor(null, index.getDataInterval(), virtualColumns, false, 512, null);
    ColumnCapabilities capabilities = virtualColumns.getColumnCapabilities(storageAdapter, "v");
    int rowCount = 0;
    if (capabilities.isDictionaryEncoded().isTrue()) {
        SingleValueDimensionVectorSelector selector = cursor.getColumnSelectorFactory().makeSingleValueDimensionSelector(DefaultDimensionSpec.of("v"));
        while (!cursor.isDone()) {
            int[] row = selector.getRowVector();
            for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                results.add(selector.lookupName(row[i]));
            }
            cursor.advance();
        }
    } else {
        VectorValueSelector selector = null;
        VectorObjectSelector objectSelector = null;
        if (outputType != null && outputType.isNumeric()) {
            selector = cursor.getColumnSelectorFactory().makeValueSelector("v");
        } else {
            objectSelector = cursor.getColumnSelectorFactory().makeObjectSelector("v");
        }
        while (!cursor.isDone()) {
            boolean[] nulls;
            switch(outputType.getType()) {
                case LONG:
                    nulls = selector.getNullVector();
                    long[] longs = selector.getLongVector();
                    for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                        results.add(nulls != null && nulls[i] ? null : longs[i]);
                    }
                    break;
                case DOUBLE:
                    // special case to test floats just to get coverage on getFloatVector
                    if ("float2".equals(expression)) {
                        nulls = selector.getNullVector();
                        float[] floats = selector.getFloatVector();
                        for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                            results.add(nulls != null && nulls[i] ? null : (double) floats[i]);
                        }
                    } else {
                        nulls = selector.getNullVector();
                        double[] doubles = selector.getDoubleVector();
                        for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) {
                            results.add(nulls != null && nulls[i] ? null : doubles[i]);
                        }
                    }
                    break;
                case STRING:
                    Object[] objects = objectSelector.getObjectVector();
                    for (int i = 0; i < objectSelector.getCurrentVectorSize(); i++, rowCount++) {
                        results.add(objects[i]);
                    }
                    break;
            }
            cursor.advance();
        }
    }
    closer.register(cursor);
    Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(null, index.getDataInterval(), virtualColumns, Granularities.ALL, false, null);
    int rowCountCursor = cursors.map(nonVectorized -> {
        final ColumnValueSelector nonSelector = nonVectorized.getColumnSelectorFactory().makeColumnValueSelector("v");
        int rows = 0;
        while (!nonVectorized.isDone()) {
            Assert.assertEquals(StringUtils.format("Failed at row %s", rows), nonSelector.getObject(), results.get(rows));
            rows++;
            nonVectorized.advance();
        }
        return rows;
    }).accumulate(0, (acc, in) -> acc + in);
    Assert.assertTrue(rowCountCursor > 0);
    Assert.assertEquals(rowCountCursor, rowCount);
}

Example 4 with VirtualColumns

use of org.apache.druid.segment.VirtualColumns in project druid by druid-io.

the class VirtualColumnsTest method testMakeSelectors.

@Test
public void testMakeSelectors() {
    final VirtualColumns virtualColumns = makeVirtualColumns();
    final BaseObjectColumnValueSelector objectSelector = virtualColumns.makeColumnValueSelector("expr", null);
    final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector(new DefaultDimensionSpec("expr", "x"), null);
    final DimensionSelector extractionDimensionSelector = virtualColumns.makeDimensionSelector(new ExtractionDimensionSpec("expr", "x", new BucketExtractionFn(1.0, 0.5)), null);
    final BaseFloatColumnValueSelector floatSelector = virtualColumns.makeColumnValueSelector("expr", null);
    final BaseLongColumnValueSelector longSelector = virtualColumns.makeColumnValueSelector("expr", null);
    Assert.assertEquals(1L, objectSelector.getObject());
    Assert.assertEquals("1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
    Assert.assertEquals("0.5", extractionDimensionSelector.lookupName(extractionDimensionSelector.getRow().get(0)));
    Assert.assertEquals(1.0f, floatSelector.getFloat(), 0.0f);
    Assert.assertEquals(1L, longSelector.getLong());
}

Example 5 with VirtualColumns

use of org.apache.druid.segment.VirtualColumns in project druid by druid-io.

the class VirtualColumnsTest method testGetColumnCapabilitiesWithFallbackNilBase.

@Test
public void testGetColumnCapabilitiesWithFallbackNilBase() {
    final VirtualColumns virtualColumns = makeVirtualColumns();
    final ColumnInspector baseInspector = column -> null;
    Assert.assertEquals(ValueType.FLOAT, virtualColumns.getColumnCapabilitiesWithFallback(baseInspector, "expr").getType());
    Assert.assertEquals(ValueType.LONG, virtualColumns.getColumnCapabilitiesWithFallback(baseInspector, "expr2").getType());
    Assert.assertNull(virtualColumns.getColumnCapabilitiesWithFallback(baseInspector, REAL_COLUMN_NAME));
}