Examples with ExpressionVirtualColumn org.apache.druid.segment.virtual.ExpressionVirtualColumn used on opensource projects

Example 41 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class JoinAndLookupBenchmark method setup.

@Setup()
public void setup() throws IOException {
    tmpDir = FileUtils.createTempDir();
    ColumnConfig columnConfig = () -> columnCacheSizeBytes;
    index = JoinTestHelper.createFactIndexBuilder(columnConfig, tmpDir, rows).buildMMappedIndex();
    final String prefix = "c.";
    baseSegment = new QueryableIndexSegment(index, SegmentId.dummy("join"));
    List<JoinableClause> joinableClausesLookupStringKey = ImmutableList.of(new JoinableClause(prefix, LookupJoinable.wrap(JoinTestHelper.createCountryIsoCodeToNameLookup()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryIsoCode == \"%sk\"", prefix), prefix, ExprMacroTable.nil())));
    JoinFilterPreAnalysis preAnalysisLookupStringKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesLookupStringKey, VirtualColumns.EMPTY, null));
    hashJoinLookupStringKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesLookupStringKey, preAnalysisLookupStringKey);
    List<JoinableClause> joinableClausesLookupLongKey = ImmutableList.of(new JoinableClause(prefix, LookupJoinable.wrap(JoinTestHelper.createCountryIsoCodeToNameLookup()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryIsoCode == \"%sk\"", prefix), prefix, ExprMacroTable.nil())));
    JoinFilterPreAnalysis preAnalysisLookupLongKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesLookupLongKey, VirtualColumns.EMPTY, null));
    hashJoinLookupLongKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesLookupLongKey, preAnalysisLookupLongKey);
    List<JoinableClause> joinableClausesIndexedTableStringKey = ImmutableList.of(new JoinableClause(prefix, new IndexedTableJoinable(JoinTestHelper.createCountriesIndexedTable()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryIsoCode == \"%scountryIsoCode\"", prefix), prefix, ExprMacroTable.nil())));
    JoinFilterPreAnalysis preAnalysisIndexedStringKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesLookupLongKey, VirtualColumns.EMPTY, null));
    hashJoinIndexedTableStringKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesIndexedTableStringKey, preAnalysisIndexedStringKey);
    List<JoinableClause> joinableClausesIndexedTableLongKey = ImmutableList.of(new JoinableClause(prefix, new IndexedTableJoinable(JoinTestHelper.createCountriesIndexedTable()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryNumber == \"%scountryNumber\"", prefix), prefix, ExprMacroTable.nil())));
    JoinFilterPreAnalysis preAnalysisIndexedLongKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesIndexedTableLongKey, VirtualColumns.EMPTY, null));
    hashJoinIndexedTableLongKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesIndexedTableLongKey, preAnalysisIndexedLongKey);
    final Map<String, String> countryCodeToNameMap = JoinTestHelper.createCountryIsoCodeToNameLookup().getMap();
    final Map<String, String> countryNumberToNameMap = JoinTestHelper.createCountryNumberToNameLookup().getMap();
    final ExprMacroTable exprMacroTable = new ExprMacroTable(ImmutableList.of(new LookupExprMacro(new LookupExtractorFactoryContainerProvider() {

        @Override
        public Set<String> getAllLookupNames() {
            return ImmutableSet.of(LOOKUP_COUNTRY_CODE_TO_NAME, LOOKUP_COUNTRY_NUMBER_TO_NAME);
        }

        @Override
        public Optional<LookupExtractorFactoryContainer> get(String lookupName) {
            if (LOOKUP_COUNTRY_CODE_TO_NAME.equals(lookupName)) {
                return Optional.of(new LookupExtractorFactoryContainer("0", new MapLookupExtractorFactory(countryCodeToNameMap, false)));
            } else if (LOOKUP_COUNTRY_NUMBER_TO_NAME.equals(lookupName)) {
                return Optional.of(new LookupExtractorFactoryContainer("0", new MapLookupExtractorFactory(countryNumberToNameMap, false)));
            } else {
                return Optional.empty();
            }
        }
    })));
    lookupVirtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn(LOOKUP_COUNTRY_CODE_TO_NAME, "lookup(countryIsoCode, '" + LOOKUP_COUNTRY_CODE_TO_NAME + "')", ColumnType.STRING, exprMacroTable), new ExpressionVirtualColumn(LOOKUP_COUNTRY_NUMBER_TO_NAME, "lookup(countryNumber, '" + LOOKUP_COUNTRY_NUMBER_TO_NAME + "')", ColumnType.STRING, exprMacroTable)));
}

Example 42 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class ExpressionSelectorBenchmark method strlenUsingExpressionAsLong.

@Benchmark
public void strlenUsingExpressionAsLong(Blackhole blackhole) {
    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(null, index.getDataInterval(), VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v", "strlen(s)", ColumnType.STRING, TestExprMacroTable.INSTANCE))), Granularities.ALL, false, null);
    final List<?> results = cursors.map(cursor -> {
        final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
        consumeLong(cursor, selector, blackhole);
        return null;
    }).toList();
    blackhole.consume(results);
}

Example 43 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class ExpressionSelectorBenchmark method stringConcatAndCompareOnLong.

@Benchmark
public void stringConcatAndCompareOnLong(Blackhole blackhole) {
    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(null, index.getDataInterval(), VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v", "concat(n, ' is my favorite number') == '3 is my favorite number'", ColumnType.LONG, TestExprMacroTable.INSTANCE))), Granularities.ALL, false, null);
    final List<?> results = cursors.map(cursor -> {
        final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
        consumeLong(cursor, selector, blackhole);
        return null;
    }).toList();
    blackhole.consume(results);
}

Example 44 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class ExpressionSelectorBenchmark method strlenUsingExpressionAsString.

@Benchmark
public void strlenUsingExpressionAsString(Blackhole blackhole) {
    final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors(null, index.getDataInterval(), VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v", "strlen(s)", ColumnType.STRING, TestExprMacroTable.INSTANCE))), Granularities.ALL, false, null);
    final List<?> results = cursors.map(cursor -> {
        final DimensionSelector selector = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("v", "v", ColumnType.STRING));
        consumeDimension(cursor, selector, blackhole);
        return null;
    }).toList();
    blackhole.consume(results);
}

Example 45 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project hive by apache.

the class DruidStorageHandlerUtils method extractColName.

@Nullable
public static String extractColName(ExprNodeDesc expr, List<VirtualColumn> virtualColumns) {
    if (!druidSupportedTypeInfos.contains(expr.getTypeInfo())) {
        // We cannot pass the bloom filter to druid since bloom filter tests for exact object bytes.
        return null;
    }
    if (expr instanceof ExprNodeColumnDesc) {
        return ((ExprNodeColumnDesc) expr).getColumn();
    }
    ExprNodeGenericFuncDesc funcDesc = null;
    if (expr instanceof ExprNodeGenericFuncDesc) {
        funcDesc = (ExprNodeGenericFuncDesc) expr;
    }
    if (null == funcDesc) {
        return null;
    }
    GenericUDF udf = funcDesc.getGenericUDF();
    // bail out if its not a simple cast expression.
    if (funcDesc.getChildren().size() == 1 && funcDesc.getChildren().get(0) instanceof ExprNodeColumnDesc) {
        return null;
    }
    String columnName = ((ExprNodeColumnDesc) (funcDesc.getChildren().get(0))).getColumn();
    ValueType targetType = null;
    if (udf instanceof GenericUDFBridge) {
        Class<? extends UDF> udfClass = ((GenericUDFBridge) udf).getUdfClass();
        if (udfClass.equals(UDFToDouble.class)) {
            targetType = ValueType.DOUBLE;
        } else if (udfClass.equals(UDFToFloat.class)) {
            targetType = ValueType.FLOAT;
        } else if (udfClass.equals(UDFToLong.class)) {
            targetType = ValueType.LONG;
        }
    } else if (udf instanceof GenericUDFToString) {
        targetType = ValueType.STRING;
    }
    if (targetType == null) {
        return null;
    }
    String virtualColumnExpr = DruidQuery.format("CAST(%s, '%s')", columnName, targetType.toString());
    for (VirtualColumn column : virtualColumns) {
        if (column instanceof ExpressionVirtualColumn && ((ExpressionVirtualColumn) column).getExpression().equals(virtualColumnExpr)) {
            // Found an existing virtual column with same expression, no need to add another virtual column
            return column.getOutputName();
        }
    }
    Set<String> usedColumnNames = virtualColumns.stream().map(col -> col.getOutputName()).collect(Collectors.toSet());
    final String name = SqlValidatorUtil.uniquify("vc", usedColumnNames, SqlValidatorUtil.EXPR_SUGGESTER);
    ExpressionVirtualColumn expressionVirtualColumn = new ExpressionVirtualColumn(name, virtualColumnExpr, targetType, ExprMacroTable.nil());
    virtualColumns.add(expressionVirtualColumn);
    return name;
}