Examples with SqlType io.confluent.ksql.schema.ksql.types.SqlType used on opensource projects

Example 76 with SqlType

use of io.confluent.ksql.schema.ksql.types.SqlType in project ksql by confluentinc.

the class QueryProjectNode method selectOutputSchema.

private LogicalSchema selectOutputSchema(final MetaStore metaStore, final List<SelectExpression> selectExpressions, final boolean isWindowed) {
    final Builder schemaBuilder = LogicalSchema.builder();
    final LogicalSchema parentSchema = getSource().getSchema();
    // Copy meta & key columns into the value schema as SelectValueMapper expects it:
    final LogicalSchema schema = parentSchema.withPseudoAndKeyColsInValue(isWindowed, ksqlConfig);
    final ExpressionTypeManager expressionTypeManager = new ExpressionTypeManager(schema, metaStore);
    for (final SelectExpression select : selectExpressions) {
        final SqlType type = expressionTypeManager.getExpressionSqlType(select.getExpression());
        if (parentSchema.isKeyColumn(select.getAlias()) || select.getAlias().equals(SystemColumns.WINDOWSTART_NAME) || select.getAlias().equals(SystemColumns.WINDOWEND_NAME)) {
            schemaBuilder.keyColumn(select.getAlias(), type);
        } else {
            schemaBuilder.valueColumn(select.getAlias(), type);
        }
    }
    return schemaBuilder.build();
}

Example 77 with SqlType

use of io.confluent.ksql.schema.ksql.types.SqlType in project ksql by confluentinc.

the class LogicalPlanner method buildAggregateSchema.

private LogicalSchema buildAggregateSchema(final PlanNode sourcePlanNode, final GroupBy groupBy, final List<SelectExpression> projectionExpressions) {
    final LogicalSchema sourceSchema = sourcePlanNode.getSchema();
    final LogicalSchema projectionSchema = SelectionUtil.buildProjectionSchema(sourceSchema.withPseudoAndKeyColsInValue(analysis.getWindowExpression().isPresent(), ksqlConfig), projectionExpressions, metaStore);
    final List<Expression> groupByExps = groupBy.getGroupingExpressions();
    final Function<Expression, Optional<ColumnName>> selectResolver = expression -> {
        final List<ColumnName> foundInProjection = projectionExpressions.stream().filter(e -> e.getExpression().equals(expression)).map(SelectExpression::getAlias).collect(Collectors.toList());
        switch(foundInProjection.size()) {
            case 0:
                return Optional.empty();
            case 1:
                return Optional.of(foundInProjection.get(0));
            default:
                final String keys = GrammaticalJoiner.and().join(foundInProjection);
                throw new KsqlException("The projection contains a key column more than once: " + keys + "." + System.lineSeparator() + "Each key column must only be in the projection once. " + "If you intended to copy the key into the value, then consider using the " + AsValue.NAME + " function to indicate which key reference should be copied.");
        }
    };
    final List<Column> valueColumns;
    if (analysis.getInto().isPresent()) {
        // Persistent query:
        final Set<ColumnName> keyColumnNames = groupBy.getGroupingExpressions().stream().map(selectResolver).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toSet());
        valueColumns = projectionSchema.value().stream().filter(col -> !keyColumnNames.contains(col.name())).collect(Collectors.toList());
        if (valueColumns.isEmpty()) {
            throw new KsqlException("The projection contains no value columns.");
        }
    } else {
        // Transient query:
        // Transient queries only return value columns, so must have key columns in the value:
        valueColumns = projectionSchema.columns();
    }
    final Builder builder = LogicalSchema.builder();
    final ExpressionTypeManager typeManager = new ExpressionTypeManager(sourceSchema, metaStore);
    for (final Expression expression : groupByExps) {
        final SqlType keyType = typeManager.getExpressionSqlType(expression);
        final ColumnName keyName = selectResolver.apply(expression).orElseGet(() -> expression instanceof ColumnReferenceExp ? ((ColumnReferenceExp) expression).getColumnName() : ColumnNames.uniqueAliasFor(expression, sourceSchema));
        builder.keyColumn(keyName, keyType);
    }
    return builder.valueColumns(valueColumns).build();
}

Example 78 with SqlType

use of io.confluent.ksql.schema.ksql.types.SqlType in project ksql by confluentinc.

the class SelectionUtil method buildProjectionSchema.

/*
   * The algorithm behind this method feels unnecessarily complicated and is begging
   * for someone to come along and improve it, but until that time here is
   * a description of what's going on.
   *
   * Essentially, we need to build a logical schema that mirrors the physical
   * schema until https://github.com/confluentinc/ksql/issues/6374 is addressed.
   * That means that the keys must be ordered in the same way as the parent schema
   * (e.g. if the source schema was K1 INT KEY, K2 INT KEY and the projection is
   * SELECT K2, K1 this method will produce an output schema that is K1, K2
   * despite the way that the keys were ordered in the projection) - see
   * https://github.com/confluentinc/ksql/pull/7477 for context on the bug.
   *
   * But we cannot simply select all the keys and then the values, we must maintain
   * the interleaving of key and values because transient queries return all columns
   * to the user as "value columns". If someone issues a SELECT VALUE, * FROM FOO
   * it is expected that VALUE shows up _before_ the key fields. This means we need to
   * reorder the key columns within the list of projections without affecting the
   * relative order the keys/values.
   *
   * To spice things up even further, there's the possibility that the same key is
   * aliased multiple times (SELECT K1 AS X, K2 AS Y FROM ...), which is not supported
   * but is verified later when building the final projection - so we maintain it here.
   *
   * Now on to the algorithm itself: we make two passes through the list of projections.
   * The first pass builds a mapping from source key to all the projections for that key.
   * We will use this mapping to sort the keys in the second pass. This mapping is two
   * dimensional to address the possibility of the same key with multiple aliases.
   *
   * The second pass goes through the list of projections again and builds the logical schema,
   * but this time if we encounter a projection that references a key column, we instead take
   * it from the list we built in the first pass (in order defined by the parent schema).
   */
public static LogicalSchema buildProjectionSchema(final LogicalSchema parentSchema, final List<SelectExpression> projection, final FunctionRegistry functionRegistry) {
    final ExpressionTypeManager expressionTypeManager = new ExpressionTypeManager(parentSchema, functionRegistry);
    // keyExpressions[i] represents the expressions found in projection
    // that are associated with parentSchema's key at index i
    final List<List<SelectExpression>> keyExpressions = new ArrayList<>(parentSchema.key().size());
    for (int i = 0; i < parentSchema.key().size(); i++) {
        keyExpressions.add(new ArrayList<>());
    }
    // first pass to construct keyExpressions, keyExpressionMembership
    // is just a convenience data structure so that we don't have to do
    // the isKey check in the second iteration below
    final Set<SelectExpression> keyExpressionMembership = new HashSet<>();
    for (final SelectExpression select : projection) {
        final Expression expression = select.getExpression();
        if (expression instanceof ColumnReferenceExp) {
            final ColumnName name = ((ColumnReferenceExp) expression).getColumnName();
            parentSchema.findColumn(name).filter(c -> c.namespace() == Namespace.KEY).ifPresent(c -> {
                keyExpressions.get(c.index()).add(select);
                keyExpressionMembership.add(select);
            });
        }
    }
    // second pass, which iterates the projections but ignores any key expressions,
    // instead taking them from the ordered keyExpressions list
    final Builder builder = LogicalSchema.builder();
    int currKeyIdx = 0;
    for (final SelectExpression select : projection) {
        if (keyExpressionMembership.contains(select)) {
            while (keyExpressions.get(currKeyIdx).isEmpty()) {
                currKeyIdx++;
            }
            final SelectExpression keyExp = keyExpressions.get(currKeyIdx).remove(0);
            final SqlType type = expressionTypeManager.getExpressionSqlType(keyExp.getExpression());
            builder.keyColumn(keyExp.getAlias(), type);
        } else {
            final Expression expression = select.getExpression();
            final SqlType type = expressionTypeManager.getExpressionSqlType(expression);
            if (type == null) {
                throw new IllegalArgumentException("Can't infer a type of null. Please explicitly cast " + "it to a required type, e.g. CAST(null AS VARCHAR).");
            }
            builder.valueColumn(select.getAlias(), type);
        }
    }
    return builder.build();
}

Example 79 with SqlType

use of io.confluent.ksql.schema.ksql.types.SqlType in project ksql by confluentinc.

the class UdafUtil method resolveAggregateFunction.

public static KsqlAggregateFunction<?, ?, ?> resolveAggregateFunction(final FunctionRegistry functionRegistry, final FunctionCall functionCall, final LogicalSchema schema, final KsqlConfig config) {
    try {
        final ExpressionTypeManager expressionTypeManager = new ExpressionTypeManager(schema, functionRegistry);
        final SqlType argumentType = expressionTypeManager.getExpressionSqlType(functionCall.getArguments().get(0));
        // UDAFs only support one non-constant argument, and that argument must be a column reference
        final Expression arg = functionCall.getArguments().get(0);
        final Optional<Column> possibleValueColumn = arg instanceof UnqualifiedColumnReferenceExp ? schema.findValueColumn(((UnqualifiedColumnReferenceExp) arg).getColumnName()) : // assume that it is a column reference with no alias
        schema.findValueColumn(ColumnName.of(arg.toString()));
        final Column valueColumn = possibleValueColumn.orElseThrow(() -> new KsqlException("Could not find column for expression: " + arg));
        final AggregateFunctionInitArguments aggregateFunctionInitArguments = createAggregateFunctionInitArgs(valueColumn.index(), functionCall, config);
        return functionRegistry.getAggregateFunction(functionCall.getName(), argumentType, aggregateFunctionInitArguments);
    } catch (final Exception e) {
        throw new KsqlException("Failed to create aggregate function: " + functionCall, e);
    }
}

Example 80 with SqlType

use of io.confluent.ksql.schema.ksql.types.SqlType in project ksql by confluentinc.

the class ArithmeticInterpreter method getNonDecimalArithmeticFunction.

private static ArithmeticBinaryFunction getNonDecimalArithmeticFunction(final Operator operator, final SqlType leftType, final SqlType rightType) {
    final SqlBaseType leftBaseType = leftType.baseType();
    final SqlBaseType rightBaseType = rightType.baseType();
    if (leftBaseType == SqlBaseType.STRING && rightBaseType == SqlBaseType.STRING) {
        return (o1, o2) -> (String) o1 + (String) o2;
    } else if (leftBaseType == SqlBaseType.DOUBLE || rightBaseType == SqlBaseType.DOUBLE) {
        final TypedArithmeticBinaryFunction<Double> fn = getDoubleFunction(operator);
        final ComparableCastFunction<Double> castLeft = castToDoubleFunction(leftType);
        final ComparableCastFunction<Double> castRight = castToDoubleFunction(rightType);
        return (o1, o2) -> fn.doFunction(castLeft.cast(o1), castRight.cast(o2));
    } else if (leftBaseType == SqlBaseType.BIGINT || rightBaseType == SqlBaseType.BIGINT) {
        final TypedArithmeticBinaryFunction<Long> fn = getLongFunction(operator);
        final ComparableCastFunction<Long> castLeft = castToLongFunction(leftType);
        final ComparableCastFunction<Long> castRight = castToLongFunction(rightType);
        return (o1, o2) -> fn.doFunction(castLeft.cast(o1), castRight.cast(o2));
    } else if (leftBaseType == SqlBaseType.INTEGER || rightBaseType == SqlBaseType.INTEGER) {
        final TypedArithmeticBinaryFunction<Integer> fn = getIntegerFunction(operator);
        final ComparableCastFunction<Integer> castLeft = castToIntegerFunction(leftType);
        final ComparableCastFunction<Integer> castRight = castToIntegerFunction(rightType);
        return (o1, o2) -> fn.doFunction(castLeft.cast(o1), castRight.cast(o2));
    } else {
        throw new KsqlException("Can't do arithmetic for types " + leftType + " and " + rightType);
    }
}