Examples with Projection io.confluent.ksql.planner.Projection used on opensource projects

Example 1 with Projection

use of io.confluent.ksql.planner.Projection in project ksql by confluentinc.

the class FinalProjectNode method validateProjection.

/**
 * Called to validate that columns referenced in the projection are valid.
 *
 * <p>This is necessary as some joins can create synthetic key columns that do not come
 * from any data source.  This means the normal column validation done during analysis can not
 * fail on unknown column with generated column names.
 *
 * <p>Once the logical model has been built the synthetic key names are known and generated
 * column names can be validated.
 */
private void validateProjection() {
    // Validate any column in the projection that might be a synthetic
    // Only really need to include any that might be, but we include all:
    final RequiredColumns requiredColumns = RequiredColumns.builder().addAll(projection.singleExpressions()).build();
    final Set<ColumnReferenceExp> unknown = getSource().validateColumns(requiredColumns);
    if (!unknown.isEmpty()) {
        final String errors = unknown.stream().map(columnRef -> NodeLocation.asPrefix(columnRef.getLocation()) + "Column '" + columnRef + "' cannot be resolved.").collect(Collectors.joining(System.lineSeparator()));
        throw new KsqlException(errors);
    }
}

Example 2 with Projection

use of io.confluent.ksql.planner.Projection in project ksql by confluentinc.

the class FinalProjectNode method build.

private Pair<LogicalSchema, List<SelectExpression>> build(final MetaStore metaStore, final KsqlConfig ksqlConfig) {
    final LogicalSchema parentSchema = getSource().getSchema();
    final Optional<LogicalSchema> targetSchema = getTargetSchema(metaStore);
    final List<SelectExpression> selectExpressions = SelectionUtil.buildSelectExpressions(getSource(), projection.selectItems(), targetSchema);
    final LogicalSchema schema = SelectionUtil.buildProjectionSchema(parentSchema, selectExpressions, metaStore);
    if (into.isPresent()) {
        // Persistent queries have key columns as value columns - final projection can exclude them:
        final Map<ColumnName, Set<ColumnName>> seenKeyColumns = new HashMap<>();
        selectExpressions.removeIf(se -> {
            if (se.getExpression() instanceof UnqualifiedColumnReferenceExp) {
                final ColumnName columnName = ((UnqualifiedColumnReferenceExp) se.getExpression()).getColumnName();
                // Window bounds columns are currently removed if not aliased:
                if (SystemColumns.isWindowBound(columnName) && se.getAlias().equals(columnName)) {
                    return true;
                }
                if (parentSchema.isKeyColumn(columnName)) {
                    seenKeyColumns.computeIfAbsent(columnName, k -> new HashSet<>()).add(se.getAlias());
                    return true;
                }
            }
            return false;
        });
        for (final Entry<ColumnName, Set<ColumnName>> seenKey : seenKeyColumns.entrySet()) {
            if (seenKey.getValue().size() > 1) {
                final String keys = GrammaticalJoiner.and().join(seenKey.getValue().stream().map(Name::text).sorted());
                throw new KsqlException("The projection contains a key column (" + seenKey.getKey() + ") more than once, aliased as: " + keys + "." + System.lineSeparator() + "Each key column must only be in the projection once. " + "If you intended to copy the key into the value, then consider using the " + AsValue.NAME + " function to indicate which key reference should be copied.");
            }
        }
    }
    final LogicalSchema nodeSchema;
    if (into.isPresent()) {
        nodeSchema = schema.withoutPseudoAndKeyColsInValue(ksqlConfig);
    } else {
        // Transient queries return key columns in the value, so the projection includes them, and
        // the schema needs to include them too:
        final Builder builder = LogicalSchema.builder();
        builder.keyColumns(parentSchema.key());
        schema.columns().forEach(builder::valueColumn);
        nodeSchema = builder.build();
    }
    return Pair.of(nodeSchema, selectExpressions);
}

Example 3 with Projection

use of io.confluent.ksql.planner.Projection in project ksql by confluentinc.

the class JoinNode method validateKeyPresent.

@Override
void validateKeyPresent(final SourceName sinkName, final Projection projection) {
    if (joinKey.isForeignKey()) {
        final DataSourceNode leftInputTable = getLeftmostSourceNode();
        final SourceName leftInputTableName = leftInputTable.getAlias();
        final List<Column> leftInputTableKeys = leftInputTable.getDataSource().getSchema().key();
        final List<Column> missingKeys = leftInputTableKeys.stream().filter(k -> !projection.containsExpression(new QualifiedColumnReferenceExp(leftInputTableName, k.name())) && !projection.containsExpression(new UnqualifiedColumnReferenceExp(ColumnNames.generatedJoinColumnAlias(leftInputTableName, k.name())))).collect(Collectors.toList());
        if (!missingKeys.isEmpty()) {
            throwMissingKeyColumnForFkJoinException(missingKeys, leftInputTableName);
        }
    } else {
        final boolean atLeastOneKey = joinKey.getAllViableKeys(schema).stream().anyMatch(projection::containsExpression);
        if (!atLeastOneKey) {
            final boolean synthetic = joinKey.isSynthetic();
            final List<? extends Expression> viable = joinKey.getOriginalViableKeys(schema);
            throwKeysNotIncludedError(sinkName, "join expression", viable, false, synthetic);
        }
    }
}

Example 4 with Projection

use of io.confluent.ksql.planner.Projection in project ksql by confluentinc.

the class QueryProjectNode method buildOutputSchema.

/**
 * Builds the output schema of the project node.
 * The output schema comprises of exactly the columns that appear in the SELECT clause of the
 * query.
 * @param metaStore the metastore
 * @return the project node's output schema
 */
private LogicalSchema buildOutputSchema(final MetaStore metaStore) {
    final LogicalSchema outputSchema;
    final LogicalSchema parentSchema = getSource().getSchema();
    final boolean isWindowed = analysis.getFrom().getDataSource().getKsqlTopic().getKeyFormat().isWindowed();
    if (isSelectStar()) {
        outputSchema = buildPullQuerySelectStarSchema(parentSchema.withoutPseudoAndKeyColsInValue(ksqlConfig), isWindowed);
    } else {
        final List<SelectExpression> projects = projection.selectItems().stream().map(SingleColumn.class::cast).map(si -> SelectExpression.of(si.getAlias().orElseThrow(IllegalStateException::new), si.getExpression())).collect(Collectors.toList());
        outputSchema = selectOutputSchema(metaStore, projects, isWindowed);
    }
    if (isScalablePush) {
        // Transient queries return key columns in the value, so the projection includes them, and
        // the schema needs to include them too:
        final Builder builder = LogicalSchema.builder();
        outputSchema.columns().forEach(builder::valueColumn);
        return builder.build();
    }
    return outputSchema;
}