Search in sources :

Example 1 with Column

use of io.confluent.ksql.schema.ksql.Column in project ksql by confluentinc.

the class SelectionUtil method resolveSelectItem.

private static Stream<SelectExpression> resolveSelectItem(final int idx, final List<? extends SelectItem> selectItems, final PlanNode parentNode, final Optional<LogicalSchema> targetSchema) {
    final SelectItem selectItem = selectItems.get(idx);
    if (selectItem instanceof SingleColumn) {
        final SingleColumn column = (SingleColumn) selectItem;
        // if the column we are trying to coerce into a target schema is beyond
        // the target schema's max columns ignore it. this will generate a failure
        // down the line when we check that the result schema is identical to
        // the schema of the source we are attempting to fit
        final Optional<Column> targetColumn = targetSchema.filter(schema -> schema.columns().size() > idx).map(schema -> schema.columns().get(idx));
        return resolveSingleColumn(idx, parentNode, column, targetColumn);
    }
    if (selectItem instanceof AllColumns) {
        return resolveAllColumns(parentNode, (AllColumns) selectItem);
    }
    throw new IllegalArgumentException("Unsupported SelectItem type: " + selectItem.getClass().getName());
}
Also used : IntStream(java.util.stream.IntStream) Expression(io.confluent.ksql.execution.expression.tree.Expression) ColumnName(io.confluent.ksql.name.ColumnName) FunctionRegistry(io.confluent.ksql.function.FunctionRegistry) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) Set(java.util.Set) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) SelectExpression(io.confluent.ksql.execution.plan.SelectExpression) SelectItem(io.confluent.ksql.parser.tree.SelectItem) Namespace(io.confluent.ksql.schema.ksql.Column.Namespace) Builder(io.confluent.ksql.schema.ksql.LogicalSchema.Builder) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) List(java.util.List) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) Stream(java.util.stream.Stream) ExpressionTypeManager(io.confluent.ksql.execution.util.ExpressionTypeManager) Optional(java.util.Optional) AllColumns(io.confluent.ksql.parser.tree.AllColumns) ColumnReferenceExp(io.confluent.ksql.execution.expression.tree.ColumnReferenceExp) Column(io.confluent.ksql.schema.ksql.Column) SqlType(io.confluent.ksql.schema.ksql.types.SqlType) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) Column(io.confluent.ksql.schema.ksql.Column) SelectItem(io.confluent.ksql.parser.tree.SelectItem) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) AllColumns(io.confluent.ksql.parser.tree.AllColumns)

Example 2 with Column

use of io.confluent.ksql.schema.ksql.Column in project ksql by confluentinc.

the class SelectionUtil method resolveSingleColumn.

private static Stream<SelectExpression> resolveSingleColumn(final int idx, final PlanNode parentNode, final SingleColumn column, final Optional<Column> targetColumn) {
    final Expression expression = parentNode.resolveSelect(idx, column.getExpression());
    final ColumnName alias = column.getAlias().orElseThrow(() -> new IllegalStateException("Alias should be present by this point"));
    return Stream.of(SelectExpression.of(alias, targetColumn.map(col -> ImplicitlyCastResolver.resolve(expression, col.type())).orElse(expression)));
}
Also used : IntStream(java.util.stream.IntStream) Expression(io.confluent.ksql.execution.expression.tree.Expression) ColumnName(io.confluent.ksql.name.ColumnName) FunctionRegistry(io.confluent.ksql.function.FunctionRegistry) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) Set(java.util.Set) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) SelectExpression(io.confluent.ksql.execution.plan.SelectExpression) SelectItem(io.confluent.ksql.parser.tree.SelectItem) Namespace(io.confluent.ksql.schema.ksql.Column.Namespace) Builder(io.confluent.ksql.schema.ksql.LogicalSchema.Builder) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) List(java.util.List) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) Stream(java.util.stream.Stream) ExpressionTypeManager(io.confluent.ksql.execution.util.ExpressionTypeManager) Optional(java.util.Optional) AllColumns(io.confluent.ksql.parser.tree.AllColumns) ColumnReferenceExp(io.confluent.ksql.execution.expression.tree.ColumnReferenceExp) Column(io.confluent.ksql.schema.ksql.Column) SqlType(io.confluent.ksql.schema.ksql.types.SqlType) ColumnName(io.confluent.ksql.name.ColumnName) Expression(io.confluent.ksql.execution.expression.tree.Expression) SelectExpression(io.confluent.ksql.execution.plan.SelectExpression)

Example 3 with Column

use of io.confluent.ksql.schema.ksql.Column in project ksql by confluentinc.

the class JoinNode method validateKeyPresent.

@Override
void validateKeyPresent(final SourceName sinkName, final Projection projection) {
    if (joinKey.isForeignKey()) {
        final DataSourceNode leftInputTable = getLeftmostSourceNode();
        final SourceName leftInputTableName = leftInputTable.getAlias();
        final List<Column> leftInputTableKeys = leftInputTable.getDataSource().getSchema().key();
        final List<Column> missingKeys = leftInputTableKeys.stream().filter(k -> !projection.containsExpression(new QualifiedColumnReferenceExp(leftInputTableName, k.name())) && !projection.containsExpression(new UnqualifiedColumnReferenceExp(ColumnNames.generatedJoinColumnAlias(leftInputTableName, k.name())))).collect(Collectors.toList());
        if (!missingKeys.isEmpty()) {
            throwMissingKeyColumnForFkJoinException(missingKeys, leftInputTableName);
        }
    } else {
        final boolean atLeastOneKey = joinKey.getAllViableKeys(schema).stream().anyMatch(projection::containsExpression);
        if (!atLeastOneKey) {
            final boolean synthetic = joinKey.isSynthetic();
            final List<? extends Expression> viable = joinKey.getOriginalViableKeys(schema);
            throwKeysNotIncludedError(sinkName, "join expression", viable, false, synthetic);
        }
    }
}
Also used : DataSource(io.confluent.ksql.metastore.model.DataSource) Arrays(java.util.Arrays) ColumnName(io.confluent.ksql.name.ColumnName) SourceName(io.confluent.ksql.name.SourceName) WithinExpression(io.confluent.ksql.parser.tree.WithinExpression) BiFunction(java.util.function.BiFunction) Pair(io.confluent.ksql.util.Pair) Immutable(com.google.errorprone.annotations.Immutable) Map(java.util.Map) ColumnReferenceExp(io.confluent.ksql.execution.expression.tree.ColumnReferenceExp) ColumnNames(io.confluent.ksql.schema.ksql.ColumnNames) ImmutableMap(com.google.common.collect.ImmutableMap) Expression(io.confluent.ksql.execution.expression.tree.Expression) Collection(java.util.Collection) Set(java.util.Set) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) Streams(com.google.common.collect.Streams) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) Stream(java.util.stream.Stream) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) Column(io.confluent.ksql.schema.ksql.Column) FormatInfo(io.confluent.ksql.serde.FormatInfo) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) SchemaKStream(io.confluent.ksql.structured.SchemaKStream) SchemaKTable(io.confluent.ksql.structured.SchemaKTable) KeyFormat(io.confluent.ksql.serde.KeyFormat) QueryContext(io.confluent.ksql.execution.context.QueryContext) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) KafkaTopicClient(io.confluent.ksql.services.KafkaTopicClient) Supplier(java.util.function.Supplier) NoneFormat(io.confluent.ksql.serde.none.NoneFormat) ImmutableList(com.google.common.collect.ImmutableList) ForeignKeyJoinParamsFactory(io.confluent.ksql.execution.streams.ForeignKeyJoinParamsFactory) Objects.requireNonNull(java.util.Objects.requireNonNull) QualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.QualifiedColumnReferenceExp) Projection(io.confluent.ksql.planner.Projection) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) RequiredColumns(io.confluent.ksql.planner.RequiredColumns) JoinParamsFactory(io.confluent.ksql.execution.streams.JoinParamsFactory) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) Context(io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter.Context) ExpressionTreeRewriter(io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter) Collections(java.util.Collections) Column(io.confluent.ksql.schema.ksql.Column) QualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.QualifiedColumnReferenceExp) SourceName(io.confluent.ksql.name.SourceName) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp)

Example 4 with Column

use of io.confluent.ksql.schema.ksql.Column in project ksql by confluentinc.

the class KsqlStructuredDataOutputNode method validate.

private static void validate(final PlanNode source, final SourceName sinkName) {
    if (!(source instanceof VerifiableNode)) {
        throw new IllegalArgumentException("VerifiableNode required");
    }
    ((VerifiableNode) source).validateKeyPresent(sinkName);
    final LogicalSchema schema = source.getSchema();
    final String duplicates = schema.columns().stream().map(Column::name).collect(Collectors.groupingBy(Function.identity(), Collectors.counting())).entrySet().stream().filter(e -> e.getValue() > 1).map(Entry::getKey).map(ColumnName::toString).collect(Collectors.joining(", "));
    if (!duplicates.isEmpty()) {
        throw new IllegalArgumentException("Value columns clash with key columns: " + duplicates);
    }
}
Also used : SchemaKStream(io.confluent.ksql.structured.SchemaKStream) ColumnName(io.confluent.ksql.name.ColumnName) SourceName(io.confluent.ksql.name.SourceName) QueryContext(io.confluent.ksql.execution.context.QueryContext) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) OptionalInt(java.util.OptionalInt) Function(java.util.function.Function) Collectors(java.util.stream.Collectors) TimestampColumn(io.confluent.ksql.execution.timestamp.TimestampColumn) Objects.requireNonNull(java.util.Objects.requireNonNull) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) Entry(java.util.Map.Entry) Optional(java.util.Optional) Column(io.confluent.ksql.schema.ksql.Column) Entry(java.util.Map.Entry) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema)

Example 5 with Column

use of io.confluent.ksql.schema.ksql.Column in project ksql by confluentinc.

the class ProjectOperator method createRowForSelectStar.

// Optimization for select star, to avoid having to do code generation
private List<?> createRowForSelectStar(final GenericRow intermediate) {
    final List<Object> rowList = new ArrayList<>();
    for (SelectExpression selectExpression : logicalNode.getSelectExpressions()) {
        final Optional<Column> column = logicalNode.getIntermediateSchema().findValueColumn(selectExpression.getAlias());
        if (!column.isPresent()) {
            throw new IllegalStateException("Couldn't find alias in intermediate schema " + selectExpression.getAlias());
        }
        final int i = column.get().index();
        rowList.add(intermediate.get(i));
    }
    return rowList;
}
Also used : Column(io.confluent.ksql.schema.ksql.Column) ArrayList(java.util.ArrayList) SelectExpression(io.confluent.ksql.execution.plan.SelectExpression)

Aggregations

Column (io.confluent.ksql.schema.ksql.Column)29 ColumnName (io.confluent.ksql.name.ColumnName)14 LogicalSchema (io.confluent.ksql.schema.ksql.LogicalSchema)14 Collectors (java.util.stream.Collectors)9 Expression (io.confluent.ksql.execution.expression.tree.Expression)8 UnqualifiedColumnReferenceExp (io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp)8 SqlType (io.confluent.ksql.schema.ksql.types.SqlType)8 KsqlException (io.confluent.ksql.util.KsqlException)8 List (java.util.List)8 Optional (java.util.Optional)8 ColumnReferenceExp (io.confluent.ksql.execution.expression.tree.ColumnReferenceExp)7 TimestampColumn (io.confluent.ksql.execution.timestamp.TimestampColumn)7 ExpressionTypeManager (io.confluent.ksql.execution.util.ExpressionTypeManager)7 Set (java.util.Set)7 SelectExpression (io.confluent.ksql.execution.plan.SelectExpression)6 KsqlTopic (io.confluent.ksql.execution.ddl.commands.KsqlTopic)5 DataSource (io.confluent.ksql.metastore.model.DataSource)5 SourceName (io.confluent.ksql.name.SourceName)5 Builder (io.confluent.ksql.schema.ksql.LogicalSchema.Builder)5 FormatInfo (io.confluent.ksql.serde.FormatInfo)5