Examples with ColumnName io.confluent.ksql.name.ColumnName used on opensource projects

Example 56 with ColumnName

use of io.confluent.ksql.name.ColumnName in project ksql by confluentinc.

the class AggregateNode method aggregate.

private SchemaKTable<?> aggregate(final SchemaKGroupedStream grouped, final InternalSchema internalSchema, final Stacker contextStacker) {
    final List<FunctionCall> functions = internalSchema.updateFunctionList(functionList);
    final Stacker aggregationContext = contextStacker.push(AGGREGATION_OP_NAME);
    final List<ColumnName> requiredColumnNames = requiredColumns.stream().map(e -> (UnqualifiedColumnReferenceExp) internalSchema.resolveToInternal(e)).map(UnqualifiedColumnReferenceExp::getColumnName).collect(Collectors.toList());
    return grouped.aggregate(requiredColumnNames, functions, windowExpression, valueFormat.getFormatInfo(), aggregationContext);
}

Example 57 with ColumnName

use of io.confluent.ksql.name.ColumnName in project ksql by confluentinc.

the class SelectionUtil method buildProjectionSchema.

/*
   * The algorithm behind this method feels unnecessarily complicated and is begging
   * for someone to come along and improve it, but until that time here is
   * a description of what's going on.
   *
   * Essentially, we need to build a logical schema that mirrors the physical
   * schema until https://github.com/confluentinc/ksql/issues/6374 is addressed.
   * That means that the keys must be ordered in the same way as the parent schema
   * (e.g. if the source schema was K1 INT KEY, K2 INT KEY and the projection is
   * SELECT K2, K1 this method will produce an output schema that is K1, K2
   * despite the way that the keys were ordered in the projection) - see
   * https://github.com/confluentinc/ksql/pull/7477 for context on the bug.
   *
   * But we cannot simply select all the keys and then the values, we must maintain
   * the interleaving of key and values because transient queries return all columns
   * to the user as "value columns". If someone issues a SELECT VALUE, * FROM FOO
   * it is expected that VALUE shows up _before_ the key fields. This means we need to
   * reorder the key columns within the list of projections without affecting the
   * relative order the keys/values.
   *
   * To spice things up even further, there's the possibility that the same key is
   * aliased multiple times (SELECT K1 AS X, K2 AS Y FROM ...), which is not supported
   * but is verified later when building the final projection - so we maintain it here.
   *
   * Now on to the algorithm itself: we make two passes through the list of projections.
   * The first pass builds a mapping from source key to all the projections for that key.
   * We will use this mapping to sort the keys in the second pass. This mapping is two
   * dimensional to address the possibility of the same key with multiple aliases.
   *
   * The second pass goes through the list of projections again and builds the logical schema,
   * but this time if we encounter a projection that references a key column, we instead take
   * it from the list we built in the first pass (in order defined by the parent schema).
   */
public static LogicalSchema buildProjectionSchema(final LogicalSchema parentSchema, final List<SelectExpression> projection, final FunctionRegistry functionRegistry) {
    final ExpressionTypeManager expressionTypeManager = new ExpressionTypeManager(parentSchema, functionRegistry);
    // keyExpressions[i] represents the expressions found in projection
    // that are associated with parentSchema's key at index i
    final List<List<SelectExpression>> keyExpressions = new ArrayList<>(parentSchema.key().size());
    for (int i = 0; i < parentSchema.key().size(); i++) {
        keyExpressions.add(new ArrayList<>());
    }
    // first pass to construct keyExpressions, keyExpressionMembership
    // is just a convenience data structure so that we don't have to do
    // the isKey check in the second iteration below
    final Set<SelectExpression> keyExpressionMembership = new HashSet<>();
    for (final SelectExpression select : projection) {
        final Expression expression = select.getExpression();
        if (expression instanceof ColumnReferenceExp) {
            final ColumnName name = ((ColumnReferenceExp) expression).getColumnName();
            parentSchema.findColumn(name).filter(c -> c.namespace() == Namespace.KEY).ifPresent(c -> {
                keyExpressions.get(c.index()).add(select);
                keyExpressionMembership.add(select);
            });
        }
    }
    // second pass, which iterates the projections but ignores any key expressions,
    // instead taking them from the ordered keyExpressions list
    final Builder builder = LogicalSchema.builder();
    int currKeyIdx = 0;
    for (final SelectExpression select : projection) {
        if (keyExpressionMembership.contains(select)) {
            while (keyExpressions.get(currKeyIdx).isEmpty()) {
                currKeyIdx++;
            }
            final SelectExpression keyExp = keyExpressions.get(currKeyIdx).remove(0);
            final SqlType type = expressionTypeManager.getExpressionSqlType(keyExp.getExpression());
            builder.keyColumn(keyExp.getAlias(), type);
        } else {
            final Expression expression = select.getExpression();
            final SqlType type = expressionTypeManager.getExpressionSqlType(expression);
            if (type == null) {
                throw new IllegalArgumentException("Can't infer a type of null. Please explicitly cast " + "it to a required type, e.g. CAST(null AS VARCHAR).");
            }
            builder.valueColumn(select.getAlias(), type);
        }
    }
    return builder.build();
}

Example 58 with ColumnName

use of io.confluent.ksql.name.ColumnName in project ksql by confluentinc.

the class CreateStreamTest method shouldThrowOnNonePrimaryKey.

@Test
public void shouldThrowOnNonePrimaryKey() {
    // Given:
    final NodeLocation loc = new NodeLocation(2, 3);
    final ColumnName name = ColumnName.of("PK");
    final TableElements invalidElements = TableElements.of(new TableElement(Optional.of(loc), name, new Type(SqlTypes.STRING), PRIMARY_KEY_CONSTRAINT), new TableElement(Optional.of(new NodeLocation(3, 4)), ColumnName.of("values are always valid"), new Type(SqlTypes.STRING), ColumnConstraints.NO_COLUMN_CONSTRAINTS));
    // When:
    final ParseFailedException e = assertThrows(ParseFailedException.class, () -> new CreateStream(SOME_NAME, invalidElements, false, false, SOME_PROPS, false));
    // Then:
    assertThat(e.getMessage(), containsString("Line: 2, Col: 4: Column `PK` is a 'PRIMARY KEY' column: " + "please use 'KEY' for streams.\n" + "Tables have PRIMARY KEYs, which are unique and NON NULL.\n" + "Streams have KEYs, which have no uniqueness or NON NULL constraints."));
}

Example 59 with ColumnName

use of io.confluent.ksql.name.ColumnName in project ksql by confluentinc.

the class CreateTableTest method shouldThrowOnNonePrimaryKey.

@Test
public void shouldThrowOnNonePrimaryKey() {
    // Given:
    final NodeLocation loc = new NodeLocation(2, 3);
    final ColumnName name = ColumnName.of("K");
    final TableElements invalidElements = TableElements.of(new TableElement(Optional.of(loc), name, new Type(SqlTypes.STRING), KEY_CONSTRAINT), new TableElement(Optional.of(new NodeLocation(3, 4)), ColumnName.of("values are always valid"), new Type(SqlTypes.STRING), ColumnConstraints.NO_COLUMN_CONSTRAINTS));
    // When:
    final ParseFailedException e = assertThrows(ParseFailedException.class, () -> new CreateTable(SOME_NAME, invalidElements, false, false, SOME_PROPS, false));
    // Then:
    assertThat(e.getMessage(), containsString("Line: 2, Col: 4: Column `K` is a 'KEY' column: " + "please use 'PRIMARY KEY' for tables.\n" + "Tables have PRIMARY KEYs, which are unique and NON NULL.\n" + "Streams have KEYs, which have no uniqueness or NON NULL constraints."));
}

Example 60 with ColumnName

use of io.confluent.ksql.name.ColumnName in project ksql by confluentinc.

the class StreamAggregateBuilder method build.

@SuppressWarnings({ "rawtypes", "unchecked" })
static KTableHolder<Windowed<GenericKey>> build(final KGroupedStreamHolder groupedStream, final StreamWindowedAggregate aggregate, final RuntimeBuildContext buildContext, final MaterializedFactory materializedFactory, final AggregateParamsFactory aggregateParamsFactory) {
    final LogicalSchema sourceSchema = groupedStream.getSchema();
    final List<ColumnName> nonFuncColumns = aggregate.getNonAggregateColumns();
    final AggregateParams aggregateParams = aggregateParamsFactory.create(sourceSchema, nonFuncColumns, buildContext.getFunctionRegistry(), aggregate.getAggregationFunctions(), true, buildContext.getKsqlConfig());
    final LogicalSchema aggregateSchema = aggregateParams.getAggregateSchema();
    final LogicalSchema resultSchema = aggregateParams.getSchema();
    final KsqlWindowExpression ksqlWindowExpression = aggregate.getWindowExpression();
    final KTable<Windowed<GenericKey>, GenericRow> aggregated = ksqlWindowExpression.accept(new WindowedAggregator(groupedStream.getGroupedStream(), aggregate, aggregateSchema, buildContext, materializedFactory, aggregateParams), null);
    final KudafAggregator<Windowed<GenericKey>> aggregator = aggregateParams.getAggregator();
    KTable<Windowed<GenericKey>, GenericRow> reduced = aggregated.transformValues(() -> new KsTransformer<>(aggregator.getResultMapper()), Named.as(StreamsUtil.buildOpName(AggregateBuilderUtils.outputContext(aggregate))));
    final MaterializationInfo.Builder materializationBuilder = AggregateBuilderUtils.materializationInfoBuilder(aggregateParams.getAggregator(), aggregate, aggregateSchema, resultSchema);
    reduced = reduced.transformValues(() -> new KsTransformer<>(new WindowBoundsPopulator()), Named.as(StreamsUtil.buildOpName(AggregateBuilderUtils.windowSelectContext(aggregate))));
    materializationBuilder.map(pl -> (KsqlTransformer) new WindowBoundsPopulator(), resultSchema, AggregateBuilderUtils.windowSelectContext(aggregate));
    return KTableHolder.materialized(reduced, resultSchema, ExecutionKeyFactory.windowed(buildContext, ksqlWindowExpression.getWindowInfo()), materializationBuilder);
}