Examples with VirtualColumn org.apache.druid.segment.VirtualColumn used on opensource projects

Example 6 with VirtualColumn

use of org.apache.druid.segment.VirtualColumn in project druid by druid-io.

the class VirtualColumnsTest method testSerde.

@Test
public void testSerde() throws Exception {
    final ObjectMapper mapper = TestHelper.makeJsonMapper();
    final ImmutableList<VirtualColumn> theColumns = ImmutableList.of(new ExpressionVirtualColumn("expr", "x + y", ColumnType.FLOAT, TestExprMacroTable.INSTANCE), new ExpressionVirtualColumn("expr2", "x + z", ColumnType.FLOAT, TestExprMacroTable.INSTANCE));
    final VirtualColumns virtualColumns = VirtualColumns.create(theColumns);
    Assert.assertEquals(virtualColumns, mapper.readValue(mapper.writeValueAsString(virtualColumns), VirtualColumns.class));
    Assert.assertEquals(theColumns, mapper.readValue(mapper.writeValueAsString(virtualColumns), mapper.getTypeFactory().constructParametricType(List.class, VirtualColumn.class)));
}

Example 7 with VirtualColumn

use of org.apache.druid.segment.VirtualColumn in project druid by druid-io.

the class ScanQueryQueryToolChest method resultArraySignature.

@Override
public RowSignature resultArraySignature(final ScanQuery query) {
    if (query.getColumns() == null || query.getColumns().isEmpty()) {
        // will include none of them.
        return RowSignature.empty();
    } else {
        final RowSignature.Builder builder = RowSignature.builder();
        if (query.withNonNullLegacy(scanQueryConfig).isLegacy()) {
            builder.add(ScanQueryEngine.LEGACY_TIMESTAMP_KEY, null);
        }
        for (String columnName : query.getColumns()) {
            // With the Scan query we only know the columnType for virtual columns. Let's report those, at least.
            final ColumnType columnType;
            final VirtualColumn virtualColumn = query.getVirtualColumns().getVirtualColumn(columnName);
            if (virtualColumn != null) {
                columnType = virtualColumn.capabilities(columnName).toColumnType();
            } else {
                // Unknown type. In the future, it would be nice to have a way to fill these in.
                columnType = null;
            }
            builder.add(columnName, columnType);
        }
        return builder.build();
    }
}

Example 8 with VirtualColumn

use of org.apache.druid.segment.VirtualColumn in project druid by druid-io.

the class ArrayConcatSqlAggregator method toDruidAggregation.

@Nullable
@Override
public Aggregation toDruidAggregation(PlannerContext plannerContext, RowSignature rowSignature, VirtualColumnRegistry virtualColumnRegistry, RexBuilder rexBuilder, String name, AggregateCall aggregateCall, Project project, List<Aggregation> existingAggregations, boolean finalizeAggregations) {
    final List<RexNode> arguments = aggregateCall.getArgList().stream().map(i -> Expressions.fromFieldAccess(rowSignature, project, i)).collect(Collectors.toList());
    Integer maxSizeBytes = null;
    if (arguments.size() > 1) {
        RexNode maxBytes = arguments.get(1);
        if (!maxBytes.isA(SqlKind.LITERAL)) {
            // maxBytes must be a literal
            return null;
        }
        maxSizeBytes = ((Number) RexLiteral.value(maxBytes)).intValue();
    }
    final DruidExpression arg = Expressions.toDruidExpression(plannerContext, rowSignature, arguments.get(0));
    final ExprMacroTable macroTable = plannerContext.getExprMacroTable();
    final String fieldName;
    final ColumnType druidType = Calcites.getValueTypeForRelDataTypeFull(aggregateCall.getType());
    if (druidType == null || !druidType.isArray()) {
        // must be an array
        return null;
    }
    final String initialvalue = ExpressionType.fromColumnTypeStrict(druidType).asTypeString() + "[]";
    if (arg.isDirectColumnAccess()) {
        fieldName = arg.getDirectColumn();
    } else {
        VirtualColumn vc = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, arg, druidType);
        fieldName = vc.getOutputName();
    }
    if (aggregateCall.isDistinct()) {
        return Aggregation.create(new ExpressionLambdaAggregatorFactory(name, ImmutableSet.of(fieldName), null, initialvalue, null, true, false, false, StringUtils.format("array_set_add_all(\"__acc\", \"%s\")", fieldName), StringUtils.format("array_set_add_all(\"__acc\", \"%s\")", name), null, null, maxSizeBytes != null ? new HumanReadableBytes(maxSizeBytes) : null, macroTable));
    } else {
        return Aggregation.create(new ExpressionLambdaAggregatorFactory(name, ImmutableSet.of(fieldName), null, initialvalue, null, true, false, false, StringUtils.format("array_concat(\"__acc\", \"%s\")", fieldName), StringUtils.format("array_concat(\"__acc\", \"%s\")", name), null, null, maxSizeBytes != null ? new HumanReadableBytes(maxSizeBytes) : null, macroTable));
    }
}

Example 9 with VirtualColumn

use of org.apache.druid.segment.VirtualColumn in project druid by druid-io.

the class Queries method computeRequiredColumns.

/**
 * Helper for implementations of {@link Query#getRequiredColumns()}. Returns the list of columns that will be read
 * out of a datasource by a query that uses the provided objects in the usual way.
 *
 * The returned set always contains {@code __time}, no matter what.
 *
 * If the virtual columns, filter, dimensions, aggregators, or additional columns refer to a virtual column, then the
 * inputs of the virtual column will be returned instead of the name of the virtual column itself. Therefore, the
 * returned list will never contain the names of any virtual columns.
 *
 * @param virtualColumns    virtual columns whose inputs should be included.
 * @param filter            optional filter whose inputs should be included.
 * @param dimensions        dimension specs whose inputs should be included.
 * @param aggregators       aggregators whose inputs should be included.
 * @param additionalColumns additional columns to include. Each of these will be added to the returned set, unless it
 *                          refers to a virtual column, in which case the virtual column inputs will be added instead.
 */
public static Set<String> computeRequiredColumns(final VirtualColumns virtualColumns, @Nullable final DimFilter filter, final List<DimensionSpec> dimensions, final List<AggregatorFactory> aggregators, final List<String> additionalColumns) {
    final Set<String> requiredColumns = new HashSet<>();
    // Everyone needs __time (it's used by intervals filters).
    requiredColumns.add(ColumnHolder.TIME_COLUMN_NAME);
    for (VirtualColumn virtualColumn : virtualColumns.getVirtualColumns()) {
        for (String column : virtualColumn.requiredColumns()) {
            if (!virtualColumns.exists(column)) {
                requiredColumns.addAll(virtualColumn.requiredColumns());
            }
        }
    }
    if (filter != null) {
        for (String column : filter.getRequiredColumns()) {
            if (!virtualColumns.exists(column)) {
                requiredColumns.add(column);
            }
        }
    }
    for (DimensionSpec dimensionSpec : dimensions) {
        if (!virtualColumns.exists(dimensionSpec.getDimension())) {
            requiredColumns.add(dimensionSpec.getDimension());
        }
    }
    for (AggregatorFactory aggregator : aggregators) {
        for (String column : aggregator.requiredFields()) {
            if (!virtualColumns.exists(column)) {
                requiredColumns.add(column);
            }
        }
    }
    for (String column : additionalColumns) {
        if (!virtualColumns.exists(column)) {
            requiredColumns.add(column);
        }
    }
    return requiredColumns;
}

Example 10 with VirtualColumn

use of org.apache.druid.segment.VirtualColumn in project hive by apache.

the class DruidStorageHandlerUtils method addDynamicFilters.

public static org.apache.druid.query.Query addDynamicFilters(org.apache.druid.query.Query query, ExprNodeGenericFuncDesc filterExpr, Configuration conf, boolean resolveDynamicValues) {
    List<VirtualColumn> virtualColumns = Arrays.asList(getVirtualColumns(query).getVirtualColumns());
    org.apache.druid.query.Query rv = query;
    DimFilter joinReductionFilter = toDruidFilter(filterExpr, conf, virtualColumns, resolveDynamicValues);
    if (joinReductionFilter != null) {
        String type = query.getType();
        DimFilter filter = new AndDimFilter(joinReductionFilter, query.getFilter());
        switch(type) {
            case org.apache.druid.query.Query.TIMESERIES:
                rv = Druids.TimeseriesQueryBuilder.copy((TimeseriesQuery) query).filters(filter).virtualColumns(VirtualColumns.create(virtualColumns)).build();
                break;
            case org.apache.druid.query.Query.TOPN:
                rv = new TopNQueryBuilder((TopNQuery) query).filters(filter).virtualColumns(VirtualColumns.create(virtualColumns)).build();
                break;
            case org.apache.druid.query.Query.GROUP_BY:
                rv = new GroupByQuery.Builder((GroupByQuery) query).setDimFilter(filter).setVirtualColumns(VirtualColumns.create(virtualColumns)).build();
                break;
            case org.apache.druid.query.Query.SCAN:
                rv = Druids.ScanQueryBuilder.copy((ScanQuery) query).filters(filter).virtualColumns(VirtualColumns.create(virtualColumns)).build();
                break;
            default:
                throw new UnsupportedOperationException("Unsupported Query type " + type);
        }
    }
    return rv;
}