Examples with Builder io.confluent.ksql.schema.ksql.LogicalSchema.Builder used on opensource projects

Example 16 with Builder

use of io.confluent.ksql.schema.ksql.LogicalSchema.Builder in project ksql by confluentinc.

the class TableElements method toLogicalSchema.

/**
 * @return the logical schema.
 */
public LogicalSchema toLogicalSchema() {
    if (elements.isEmpty()) {
        throw new KsqlException("No columns supplied.");
    }
    final Builder builder = LogicalSchema.builder();
    for (final TableElement tableElement : this) {
        final ColumnName fieldName = tableElement.getName();
        final SqlType fieldType = tableElement.getType().getSqlType();
        if (tableElement.getConstraints().isKey() || tableElement.getConstraints().isPrimaryKey()) {
            builder.keyColumn(fieldName, fieldType);
        } else if (tableElement.getConstraints().isHeaders()) {
            builder.headerColumn(fieldName, tableElement.getConstraints().getHeaderKey());
        } else {
            builder.valueColumn(fieldName, fieldType);
        }
    }
    return builder.build();
}

Example 17 with Builder

use of io.confluent.ksql.schema.ksql.LogicalSchema.Builder in project ksql by confluentinc.

the class QueryProjectNode method selectOutputSchema.

private LogicalSchema selectOutputSchema(final MetaStore metaStore, final List<SelectExpression> selectExpressions, final boolean isWindowed) {
    final Builder schemaBuilder = LogicalSchema.builder();
    final LogicalSchema parentSchema = getSource().getSchema();
    // Copy meta & key columns into the value schema as SelectValueMapper expects it:
    final LogicalSchema schema = parentSchema.withPseudoAndKeyColsInValue(isWindowed, ksqlConfig);
    final ExpressionTypeManager expressionTypeManager = new ExpressionTypeManager(schema, metaStore);
    for (final SelectExpression select : selectExpressions) {
        final SqlType type = expressionTypeManager.getExpressionSqlType(select.getExpression());
        if (parentSchema.isKeyColumn(select.getAlias()) || select.getAlias().equals(SystemColumns.WINDOWSTART_NAME) || select.getAlias().equals(SystemColumns.WINDOWEND_NAME)) {
            schemaBuilder.keyColumn(select.getAlias(), type);
        } else {
            schemaBuilder.valueColumn(select.getAlias(), type);
        }
    }
    return schemaBuilder.build();
}

Example 18 with Builder

use of io.confluent.ksql.schema.ksql.LogicalSchema.Builder in project ksql by confluentinc.

the class QueryProjectNode method buildPullQuerySelectStarSchema.

private LogicalSchema buildPullQuerySelectStarSchema(final LogicalSchema schema, final boolean windowed) {
    final Builder builder = LogicalSchema.builder().keyColumns(schema.key());
    if (windowed) {
        builder.keyColumn(SystemColumns.WINDOWSTART_NAME, SqlTypes.BIGINT);
        builder.keyColumn(SystemColumns.WINDOWEND_NAME, SqlTypes.BIGINT);
    }
    return builder.headerColumns(schema.headers()).valueColumns(schema.value()).build();
}

Example 19 with Builder

use of io.confluent.ksql.schema.ksql.LogicalSchema.Builder in project ksql by confluentinc.

the class LogicalPlanner method buildAggregateSchema.

private LogicalSchema buildAggregateSchema(final PlanNode sourcePlanNode, final GroupBy groupBy, final List<SelectExpression> projectionExpressions) {
    final LogicalSchema sourceSchema = sourcePlanNode.getSchema();
    final LogicalSchema projectionSchema = SelectionUtil.buildProjectionSchema(sourceSchema.withPseudoAndKeyColsInValue(analysis.getWindowExpression().isPresent(), ksqlConfig), projectionExpressions, metaStore);
    final List<Expression> groupByExps = groupBy.getGroupingExpressions();
    final Function<Expression, Optional<ColumnName>> selectResolver = expression -> {
        final List<ColumnName> foundInProjection = projectionExpressions.stream().filter(e -> e.getExpression().equals(expression)).map(SelectExpression::getAlias).collect(Collectors.toList());
        switch(foundInProjection.size()) {
            case 0:
                return Optional.empty();
            case 1:
                return Optional.of(foundInProjection.get(0));
            default:
                final String keys = GrammaticalJoiner.and().join(foundInProjection);
                throw new KsqlException("The projection contains a key column more than once: " + keys + "." + System.lineSeparator() + "Each key column must only be in the projection once. " + "If you intended to copy the key into the value, then consider using the " + AsValue.NAME + " function to indicate which key reference should be copied.");
        }
    };
    final List<Column> valueColumns;
    if (analysis.getInto().isPresent()) {
        // Persistent query:
        final Set<ColumnName> keyColumnNames = groupBy.getGroupingExpressions().stream().map(selectResolver).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toSet());
        valueColumns = projectionSchema.value().stream().filter(col -> !keyColumnNames.contains(col.name())).collect(Collectors.toList());
        if (valueColumns.isEmpty()) {
            throw new KsqlException("The projection contains no value columns.");
        }
    } else {
        // Transient query:
        // Transient queries only return value columns, so must have key columns in the value:
        valueColumns = projectionSchema.columns();
    }
    final Builder builder = LogicalSchema.builder();
    final ExpressionTypeManager typeManager = new ExpressionTypeManager(sourceSchema, metaStore);
    for (final Expression expression : groupByExps) {
        final SqlType keyType = typeManager.getExpressionSqlType(expression);
        final ColumnName keyName = selectResolver.apply(expression).orElseGet(() -> expression instanceof ColumnReferenceExp ? ((ColumnReferenceExp) expression).getColumnName() : ColumnNames.uniqueAliasFor(expression, sourceSchema));
        builder.keyColumn(keyName, keyType);
    }
    return builder.valueColumns(valueColumns).build();
}

Example 20 with Builder

use of io.confluent.ksql.schema.ksql.LogicalSchema.Builder in project ksql by confluentinc.

the class SelectionUtil method buildProjectionSchema.

/*
   * The algorithm behind this method feels unnecessarily complicated and is begging
   * for someone to come along and improve it, but until that time here is
   * a description of what's going on.
   *
   * Essentially, we need to build a logical schema that mirrors the physical
   * schema until https://github.com/confluentinc/ksql/issues/6374 is addressed.
   * That means that the keys must be ordered in the same way as the parent schema
   * (e.g. if the source schema was K1 INT KEY, K2 INT KEY and the projection is
   * SELECT K2, K1 this method will produce an output schema that is K1, K2
   * despite the way that the keys were ordered in the projection) - see
   * https://github.com/confluentinc/ksql/pull/7477 for context on the bug.
   *
   * But we cannot simply select all the keys and then the values, we must maintain
   * the interleaving of key and values because transient queries return all columns
   * to the user as "value columns". If someone issues a SELECT VALUE, * FROM FOO
   * it is expected that VALUE shows up _before_ the key fields. This means we need to
   * reorder the key columns within the list of projections without affecting the
   * relative order the keys/values.
   *
   * To spice things up even further, there's the possibility that the same key is
   * aliased multiple times (SELECT K1 AS X, K2 AS Y FROM ...), which is not supported
   * but is verified later when building the final projection - so we maintain it here.
   *
   * Now on to the algorithm itself: we make two passes through the list of projections.
   * The first pass builds a mapping from source key to all the projections for that key.
   * We will use this mapping to sort the keys in the second pass. This mapping is two
   * dimensional to address the possibility of the same key with multiple aliases.
   *
   * The second pass goes through the list of projections again and builds the logical schema,
   * but this time if we encounter a projection that references a key column, we instead take
   * it from the list we built in the first pass (in order defined by the parent schema).
   */
public static LogicalSchema buildProjectionSchema(final LogicalSchema parentSchema, final List<SelectExpression> projection, final FunctionRegistry functionRegistry) {
    final ExpressionTypeManager expressionTypeManager = new ExpressionTypeManager(parentSchema, functionRegistry);
    // keyExpressions[i] represents the expressions found in projection
    // that are associated with parentSchema's key at index i
    final List<List<SelectExpression>> keyExpressions = new ArrayList<>(parentSchema.key().size());
    for (int i = 0; i < parentSchema.key().size(); i++) {
        keyExpressions.add(new ArrayList<>());
    }
    // first pass to construct keyExpressions, keyExpressionMembership
    // is just a convenience data structure so that we don't have to do
    // the isKey check in the second iteration below
    final Set<SelectExpression> keyExpressionMembership = new HashSet<>();
    for (final SelectExpression select : projection) {
        final Expression expression = select.getExpression();
        if (expression instanceof ColumnReferenceExp) {
            final ColumnName name = ((ColumnReferenceExp) expression).getColumnName();
            parentSchema.findColumn(name).filter(c -> c.namespace() == Namespace.KEY).ifPresent(c -> {
                keyExpressions.get(c.index()).add(select);
                keyExpressionMembership.add(select);
            });
        }
    }
    // second pass, which iterates the projections but ignores any key expressions,
    // instead taking them from the ordered keyExpressions list
    final Builder builder = LogicalSchema.builder();
    int currKeyIdx = 0;
    for (final SelectExpression select : projection) {
        if (keyExpressionMembership.contains(select)) {
            while (keyExpressions.get(currKeyIdx).isEmpty()) {
                currKeyIdx++;
            }
            final SelectExpression keyExp = keyExpressions.get(currKeyIdx).remove(0);
            final SqlType type = expressionTypeManager.getExpressionSqlType(keyExp.getExpression());
            builder.keyColumn(keyExp.getAlias(), type);
        } else {
            final Expression expression = select.getExpression();
            final SqlType type = expressionTypeManager.getExpressionSqlType(expression);
            if (type == null) {
                throw new IllegalArgumentException("Can't infer a type of null. Please explicitly cast " + "it to a required type, e.g. CAST(null AS VARCHAR).");
            }
            builder.valueColumn(select.getAlias(), type);
        }
    }
    return builder.build();
}