Examples with KStreamHolder io.confluent.ksql.execution.plan.KStreamHolder used on opensource projects

Example 1 with KStreamHolder

use of io.confluent.ksql.execution.plan.KStreamHolder in project ksql by confluentinc.

the class StreamSelectKeyBuilderV1 method build.

public static KStreamHolder<GenericKey> build(final KStreamHolder<?> stream, final StreamSelectKeyV1 selectKey, final RuntimeBuildContext buildContext) {
    final LogicalSchema sourceSchema = stream.getSchema();
    final CompiledExpression expression = buildExpressionEvaluator(selectKey, buildContext, sourceSchema);
    final ProcessingLogger processingLogger = buildContext.getProcessingLogger(selectKey.getProperties().getQueryContext());
    final String errorMsg = "Error extracting new key using expression " + selectKey.getKeyExpression();
    final Function<GenericRow, Object> evaluator = val -> expression.evaluate(val, null, processingLogger, () -> errorMsg);
    final LogicalSchema resultSchema = new StepSchemaResolver(buildContext.getKsqlConfig(), buildContext.getFunctionRegistry()).resolve(selectKey, sourceSchema);
    final KStream<?, GenericRow> kstream = stream.getStream();
    final KStream<GenericKey, GenericRow> rekeyed = kstream.filter((key, val) -> val != null && evaluator.apply(val) != null).selectKey((key, val) -> GenericKey.genericKey(evaluator.apply(val)));
    return new KStreamHolder<>(rekeyed, resultSchema, ExecutionKeyFactory.unwindowed(buildContext));
}

Example 2 with KStreamHolder

use of io.confluent.ksql.execution.plan.KStreamHolder in project ksql by confluentinc.

the class SourceBuilderV1 method buildStream.

public KStreamHolder<GenericKey> buildStream(final RuntimeBuildContext buildContext, final StreamSource source, final ConsumedFactory consumedFactory) {
    final PhysicalSchema physicalSchema = getPhysicalSchema(source);
    final Serde<GenericRow> valueSerde = getValueSerde(buildContext, source, physicalSchema);
    final Serde<GenericKey> keySerde = getKeySerde(source, physicalSchema, buildContext);
    final Consumed<GenericKey, GenericRow> consumed = buildSourceConsumed(source, keySerde, valueSerde, AutoOffsetReset.LATEST, buildContext, consumedFactory);
    final KStream<GenericKey, GenericRow> kstream = buildKStream(source, buildContext, consumed, nonWindowedKeyGenerator(source.getSourceSchema()));
    return new KStreamHolder<>(kstream, buildSchema(source, false), ExecutionKeyFactory.unwindowed(buildContext));
}

Example 3 with KStreamHolder

use of io.confluent.ksql.execution.plan.KStreamHolder in project ksql by confluentinc.

the class SourceBuilderV1 method buildWindowedStream.

public KStreamHolder<Windowed<GenericKey>> buildWindowedStream(final RuntimeBuildContext buildContext, final WindowedStreamSource source, final ConsumedFactory consumedFactory) {
    final PhysicalSchema physicalSchema = getPhysicalSchema(source);
    final Serde<GenericRow> valueSerde = getValueSerde(buildContext, source, physicalSchema);
    final WindowInfo windowInfo = source.getWindowInfo();
    final Serde<Windowed<GenericKey>> keySerde = getWindowedKeySerde(source, physicalSchema, buildContext, windowInfo);
    final Consumed<Windowed<GenericKey>, GenericRow> consumed = buildSourceConsumed(source, keySerde, valueSerde, AutoOffsetReset.LATEST, buildContext, consumedFactory);
    final KStream<Windowed<GenericKey>, GenericRow> kstream = buildKStream(source, buildContext, consumed, windowedKeyGenerator(source.getSourceSchema()));
    return new KStreamHolder<>(kstream, buildSchema(source, true), ExecutionKeyFactory.windowed(buildContext, windowInfo));
}

Example 4 with KStreamHolder

use of io.confluent.ksql.execution.plan.KStreamHolder in project ksql by confluentinc.

the class SchemaKStream method selectKey.

/**
 * @param valueFormat value format used in constructing serdes. Unchanged by this step.
 * @param keyExpression expression for the key being selected
 * @param forceInternalKeyFormat new key format to be used, if present
 * @param contextStacker context for this step
 * @param forceRepartition if true, this step will repartition even if there is no change in
 *                         either key format or value. Used to ensure co-partitioning for
 *                         joins on Schema-Registry-enabled key formats
 * @return result stream: repartitioned if needed or forced, else this stream unchanged
 */
public SchemaKStream<K> selectKey(final FormatInfo valueFormat, final List<Expression> keyExpression, final Optional<KeyFormat> forceInternalKeyFormat, final Stacker contextStacker, final boolean forceRepartition) {
    final boolean keyFormatChange = forceInternalKeyFormat.isPresent() && !forceInternalKeyFormat.get().equals(keyFormat);
    final boolean repartitionNeeded = repartitionNeeded(keyExpression);
    if (!keyFormatChange && !forceRepartition && !repartitionNeeded) {
        return this;
    }
    if ((repartitionNeeded || !forceRepartition) && keyFormat.isWindowed()) {
        throw new KsqlException("Implicit repartitioning of windowed sources is not supported. " + "See https://github.com/confluentinc/ksql/issues/4385.");
    }
    final ExecutionStep<KStreamHolder<K>> step = ExecutionStepFactory.streamSelectKey(contextStacker, sourceStep, keyExpression);
    final KeyFormat newKeyFormat = forceInternalKeyFormat.orElse(keyFormat);
    return new SchemaKStream<>(step, resolveSchema(step), SerdeFeaturesFactory.sanitizeKeyFormat(newKeyFormat, toSqlTypes(keyExpression), true), ksqlConfig, functionRegistry);
}

Example 5 with KStreamHolder

use of io.confluent.ksql.execution.plan.KStreamHolder in project ksql by confluentinc.

the class StreamSelectKeyBuilder method build.

@VisibleForTesting
static <K> KStreamHolder<K> build(final KStreamHolder<K> stream, final StreamSelectKey<K> selectKey, final RuntimeBuildContext buildContext, final PartitionByParamsBuilder paramsBuilder) {
    final LogicalSchema sourceSchema = stream.getSchema();
    final QueryContext queryContext = selectKey.getProperties().getQueryContext();
    final ProcessingLogger logger = buildContext.getProcessingLogger(queryContext);
    final PartitionByParams<K> params = paramsBuilder.build(sourceSchema, stream.getExecutionKeyFactory(), selectKey.getKeyExpressions(), buildContext.getKsqlConfig(), buildContext.getFunctionRegistry(), logger);
    final Mapper<K> mapper = params.getMapper();
    final KStream<K, GenericRow> kStream = stream.getStream();
    final KStream<K, GenericRow> reKeyed = kStream.map(mapper, Named.as(queryContext.formatContext() + "-SelectKey"));
    return new KStreamHolder<>(reKeyed, params.getSchema(), stream.getExecutionKeyFactory().withQueryBuilder(buildContext));
}