Examples with LogicalSchema io.confluent.ksql.schema.ksql.LogicalSchema used on opensource projects

Example 96 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class SinkBuilder method build.

public static <K> void build(final LogicalSchema schema, final Formats formats, final Optional<TimestampColumn> timestampColumn, final String topicName, final KStream<K, GenericRow> stream, final ExecutionKeyFactory<K> executionKeyFactory, final QueryContext queryContext, final RuntimeBuildContext buildContext) {
    final PhysicalSchema physicalSchema = PhysicalSchema.from(schema, formats.getKeyFeatures(), formats.getValueFeatures());
    final Serde<K> keySerde = executionKeyFactory.buildKeySerde(formats.getKeyFormat(), physicalSchema, queryContext);
    final Serde<GenericRow> valueSerde = buildContext.buildValueSerde(formats.getValueFormat(), physicalSchema, queryContext);
    final Optional<TransformTimestamp<K>> tsTransformer = timestampTransformer(buildContext, queryContext, schema, timestampColumn);
    final KStream<K, GenericRow> transformed = tsTransformer.map(t -> stream.transform(t, Named.as(TIMESTAMP_TRANSFORM_NAME + StreamsUtil.buildOpName(queryContext)))).orElse(stream);
    transformed.to(topicName, Produced.with(keySerde, valueSerde));
}

Example 97 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class StreamSelectBuilder method build.

public static <K> KStreamHolder<K> build(final KStreamHolder<K> stream, final StreamSelect<K> step, final RuntimeBuildContext buildContext) {
    final QueryContext queryContext = step.getProperties().getQueryContext();
    final LogicalSchema sourceSchema = stream.getSchema();
    final Selection<K> selection = Selection.of(sourceSchema, step.getKeyColumnNames(), step.getSelectExpressions(), buildContext.getKsqlConfig(), buildContext.getFunctionRegistry());
    final SelectValueMapper<K> selectMapper = selection.getMapper();
    final ProcessingLogger logger = buildContext.getProcessingLogger(queryContext);
    final Named selectName = Named.as(StreamsUtil.buildOpName(queryContext));
    return stream.withStream(stream.getStream().transformValues(() -> new KsTransformer<>(selectMapper.getTransformer(logger)), selectName), selection.getSchema());
}

Example 98 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class StreamSelectKeyBuilderV1 method build.

public static KStreamHolder<GenericKey> build(final KStreamHolder<?> stream, final StreamSelectKeyV1 selectKey, final RuntimeBuildContext buildContext) {
    final LogicalSchema sourceSchema = stream.getSchema();
    final CompiledExpression expression = buildExpressionEvaluator(selectKey, buildContext, sourceSchema);
    final ProcessingLogger processingLogger = buildContext.getProcessingLogger(selectKey.getProperties().getQueryContext());
    final String errorMsg = "Error extracting new key using expression " + selectKey.getKeyExpression();
    final Function<GenericRow, Object> evaluator = val -> expression.evaluate(val, null, processingLogger, () -> errorMsg);
    final LogicalSchema resultSchema = new StepSchemaResolver(buildContext.getKsqlConfig(), buildContext.getFunctionRegistry()).resolve(selectKey, sourceSchema);
    final KStream<?, GenericRow> kstream = stream.getStream();
    final KStream<GenericKey, GenericRow> rekeyed = kstream.filter((key, val) -> val != null && evaluator.apply(val) != null).selectKey((key, val) -> GenericKey.genericKey(evaluator.apply(val)));
    return new KStreamHolder<>(rekeyed, resultSchema, ExecutionKeyFactory.unwindowed(buildContext));
}

Example 99 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class TableAggregateBuilder method build.

public static KTableHolder<GenericKey> build(final KGroupedTableHolder groupedTable, final TableAggregate aggregate, final RuntimeBuildContext buildContext, final MaterializedFactory materializedFactory, final AggregateParamsFactory aggregateParamsFactory) {
    final LogicalSchema sourceSchema = groupedTable.getSchema();
    final List<ColumnName> nonFuncColumns = aggregate.getNonAggregateColumns();
    final AggregateParams aggregateParams = aggregateParamsFactory.createUndoable(sourceSchema, nonFuncColumns, buildContext.getFunctionRegistry(), aggregate.getAggregationFunctions(), buildContext.getKsqlConfig());
    final LogicalSchema aggregateSchema = aggregateParams.getAggregateSchema();
    final LogicalSchema resultSchema = aggregateParams.getSchema();
    final Materialized<GenericKey, GenericRow, KeyValueStore<Bytes, byte[]>> materialized = MaterializationUtil.buildMaterialized(aggregate, aggregateSchema, aggregate.getInternalFormats(), buildContext, materializedFactory, ExecutionKeyFactory.unwindowed(buildContext));
    final KTable<GenericKey, GenericRow> aggregated = groupedTable.getGroupedTable().aggregate(aggregateParams.getInitializer(), aggregateParams.getAggregator(), aggregateParams.getUndoAggregator().get(), materialized).transformValues(() -> new KsTransformer<>(aggregateParams.<GenericKey>getAggregator().getResultMapper()), Named.as(StreamsUtil.buildOpName(AggregateBuilderUtils.outputContext(aggregate))));
    final MaterializationInfo.Builder materializationBuilder = AggregateBuilderUtils.materializationInfoBuilder(aggregateParams.getAggregator(), aggregate, aggregateSchema, resultSchema);
    return KTableHolder.materialized(aggregated, resultSchema, ExecutionKeyFactory.unwindowed(buildContext), materializationBuilder);
}

Example 100 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class TableSelectKeyBuilder method build.

@VisibleForTesting
static <K> KTableHolder<K> build(final KTableHolder<K> table, final TableSelectKey<K> selectKey, final RuntimeBuildContext buildContext, final MaterializedFactory materializedFactory, final PartitionByParamsBuilder paramsBuilder) {
    final LogicalSchema sourceSchema = table.getSchema();
    final QueryContext queryContext = selectKey.getProperties().getQueryContext();
    final ProcessingLogger logger = buildContext.getProcessingLogger(queryContext);
    final PartitionByParams<K> params = paramsBuilder.build(sourceSchema, table.getExecutionKeyFactory(), selectKey.getKeyExpressions(), buildContext.getKsqlConfig(), buildContext.getFunctionRegistry(), logger);
    final Mapper<K> mapper = params.getMapper();
    final KTable<K, GenericRow> kTable = table.getTable();
    final Materialized<K, GenericRow, KeyValueStore<Bytes, byte[]>> materialized = MaterializationUtil.buildMaterialized(selectKey, params.getSchema(), selectKey.getInternalFormats(), buildContext, materializedFactory, table.getExecutionKeyFactory());
    final KTable<K, GenericRow> reKeyed = kTable.toStream().map(mapper, Named.as(queryContext.formatContext() + "-SelectKey-Mapper")).toTable(Named.as(queryContext.formatContext() + "-SelectKey"), materialized);
    final MaterializationInfo.Builder materializationBuilder = MaterializationInfo.builder(StreamsUtil.buildOpName(MaterializationUtil.materializeContext(selectKey)), params.getSchema());
    return KTableHolder.materialized(reKeyed, params.getSchema(), table.getExecutionKeyFactory().withQueryBuilder(buildContext), materializationBuilder);
}