Examples with RuntimeBuildContext io.confluent.ksql.execution.runtime.RuntimeBuildContext used on opensource projects

Example 1 with RuntimeBuildContext

use of io.confluent.ksql.execution.runtime.RuntimeBuildContext in project ksql by confluentinc.

the class TableSelectBuilder method build.

@SuppressWarnings("unchecked")
public static <K> KTableHolder<K> build(final KTableHolder<K> table, final TableSelect<K> step, final RuntimeBuildContext buildContext, final Optional<Formats> formats, final MaterializedFactory materializedFactory) {
    final LogicalSchema sourceSchema = table.getSchema();
    final QueryContext queryContext = step.getProperties().getQueryContext();
    final Selection<K> selection = Selection.of(sourceSchema, step.getKeyColumnNames(), step.getSelectExpressions(), buildContext.getKsqlConfig(), buildContext.getFunctionRegistry());
    final SelectValueMapper<K> selectMapper = selection.getMapper();
    final ProcessingLogger logger = buildContext.getProcessingLogger(queryContext);
    final Named selectName = Named.as(StreamsUtil.buildOpName(queryContext));
    final Optional<MaterializationInfo.Builder> matBuilder = table.getMaterializationBuilder();
    final boolean forceMaterialize = !matBuilder.isPresent();
    final Serde<K> keySerde;
    final Serde<GenericRow> valSerde;
    if (formats.isPresent()) {
        final Formats materializationFormat = formats.get();
        final PhysicalSchema physicalSchema = PhysicalSchema.from(selection.getSchema(), materializationFormat.getKeyFeatures(), materializationFormat.getValueFeatures());
        keySerde = (Serde<K>) buildContext.buildKeySerde(materializationFormat.getKeyFormat(), physicalSchema, queryContext);
        valSerde = buildContext.buildValueSerde(materializationFormat.getValueFormat(), physicalSchema, queryContext);
        if (forceMaterialize) {
            final Stacker stacker = Stacker.of(step.getProperties().getQueryContext());
            final String stateStoreName = StreamsUtil.buildOpName(stacker.push(PROJECT_OP).getQueryContext());
            final Materialized<K, GenericRow, KeyValueStore<Bytes, byte[]>> materialized = materializedFactory.create(keySerde, valSerde, stateStoreName);
            final KTable<K, GenericRow> transFormedTable = table.getTable().transformValues(() -> new KsTransformer<>(selectMapper.getTransformer(logger)), materialized);
            return KTableHolder.materialized(transFormedTable, selection.getSchema(), table.getExecutionKeyFactory(), MaterializationInfo.builder(stateStoreName, selection.getSchema()));
        }
    } else {
        keySerde = null;
        valSerde = null;
    }
    final KTable<K, GenericRow> transFormedTable = table.getTable().transformValues(() -> new KsTransformer<>(selectMapper.getTransformer(logger)), Materialized.with(keySerde, valSerde), selectName);
    final Optional<MaterializationInfo.Builder> materialization = matBuilder.map(b -> b.map(pl -> (KsqlTransformer<Object, GenericRow>) selectMapper.getTransformer(pl), selection.getSchema(), queryContext));
    return table.withTable(transFormedTable, selection.getSchema()).withMaterialization(materialization);
}

Example 2 with RuntimeBuildContext

use of io.confluent.ksql.execution.runtime.RuntimeBuildContext in project ksql by confluentinc.

the class TableSuppressBuilder method build.

@VisibleForTesting
@SuppressWarnings("unchecked")
<K> KTableHolder<K> build(final KTableHolder<K> table, final TableSuppress<K> step, final RuntimeBuildContext buildContext, final ExecutionKeyFactory<K> executionKeyFactory, final PhysicalSchemaFactory physicalSchemaFactory, final BiFunction<Serde<K>, Serde<GenericRow>, Materialized> materializedFactory) {
    final PhysicalSchema physicalSchema = physicalSchemaFactory.create(table.getSchema(), step.getInternalFormats().getKeyFeatures(), step.getInternalFormats().getValueFeatures());
    final QueryContext queryContext = QueryContext.Stacker.of(step.getProperties().getQueryContext()).push(SUPPRESS_OP_NAME).getQueryContext();
    final Serde<K> keySerde = executionKeyFactory.buildKeySerde(step.getInternalFormats().getKeyFormat(), physicalSchema, queryContext);
    final Serde<GenericRow> valueSerde = buildContext.buildValueSerde(step.getInternalFormats().getValueFormat(), physicalSchema, queryContext);
    final Materialized<K, GenericRow, KeyValueStore<Bytes, byte[]>> materialized = materializedFactory.apply(keySerde, valueSerde);
    final Suppressed.StrictBufferConfig strictBufferConfig;
    final long maxBytes = buildContext.getKsqlConfig().getLong(KsqlConfig.KSQL_SUPPRESS_BUFFER_SIZE_BYTES);
    if (maxBytes < 0) {
        strictBufferConfig = Suppressed.BufferConfig.unbounded();
    } else {
        strictBufferConfig = Suppressed.BufferConfig.maxBytes(maxBytes).shutDownWhenFull();
    }
    /* This is a dummy transformValues() call, we do this to ensure that the correct materialized
    with the correct key and val serdes is passed on when we call suppress
     */
    final KTable<K, GenericRow> suppressed = table.getTable().transformValues((() -> new KsTransformer<>((k, v, ctx) -> v)), materialized).suppress((Suppressed<? super K>) Suppressed.untilWindowCloses(strictBufferConfig).withName(SUPPRESS_OP_NAME));
    return table.withTable(suppressed, table.getSchema());
}

Example 3 with RuntimeBuildContext

use of io.confluent.ksql.execution.runtime.RuntimeBuildContext in project ksql by confluentinc.

the class TableFilterBuilder method build.

static <K> KTableHolder<K> build(final KTableHolder<K> table, final TableFilter<K> step, final RuntimeBuildContext buildContext, final SqlPredicateFactory sqlPredicateFactory) {
    final SqlPredicate predicate = sqlPredicateFactory.create(step.getFilterExpression(), table.getSchema(), buildContext.getKsqlConfig(), buildContext.getFunctionRegistry());
    final ProcessingLogger processingLogger = buildContext.getProcessingLogger(step.getProperties().getQueryContext());
    final Stacker stacker = Stacker.of(step.getProperties().getQueryContext());
    final KTable<K, GenericRow> filtered = table.getTable().transformValues(() -> new KsTransformer<>(predicate.getTransformer(processingLogger)), Named.as(StreamsUtil.buildOpName(stacker.push(PRE_PROCESS_OP).getQueryContext()))).filter((k, v) -> v.isPresent(), Named.as(StreamsUtil.buildOpName(stacker.push(FILTER_OP).getQueryContext()))).mapValues(Optional::get, Named.as(StreamsUtil.buildOpName(stacker.push(POST_PROCESS_OP).getQueryContext())));
    return table.withTable(filtered, table.getSchema()).withMaterialization(table.getMaterializationBuilder().map(b -> b.filter(predicate::getTransformer, step.getProperties().getQueryContext())));
}

Example 4 with RuntimeBuildContext

use of io.confluent.ksql.execution.runtime.RuntimeBuildContext in project ksql by confluentinc.

the class SourceBuilderV1 method buildKTable.

@Override
<K> KTable<K, GenericRow> buildKTable(final SourceStep<?> streamSource, final RuntimeBuildContext buildContext, final Consumed<K, GenericRow> consumed, final Function<K, Collection<?>> keyGenerator, final Materialized<K, GenericRow, KeyValueStore<Bytes, byte[]>> materialized, final Serde<GenericRow> valueSerde, final String stateStoreName, final PlanInfo planInfo) {
    validateNotUsingOldExecutionStepWithNewQueries(streamSource);
    final boolean forceChangelog = streamSource instanceof TableSourceV1 && ((TableSourceV1) streamSource).isForceChangelog();
    final KTable<K, GenericRow> table;
    if (!forceChangelog) {
        final String changelogTopic = changelogTopic(buildContext, stateStoreName);
        final Callback onFailure = getRegisterCallback(buildContext, streamSource.getFormats().getValueFormat());
        table = buildContext.getStreamsBuilder().table(streamSource.getTopicName(), consumed.withValueSerde(StaticTopicSerde.wrap(changelogTopic, valueSerde, onFailure)), materialized);
    } else {
        final KTable<K, GenericRow> source = buildContext.getStreamsBuilder().table(streamSource.getTopicName(), consumed);
        final boolean forceMaterialization = !planInfo.isRepartitionedInPlan(streamSource);
        if (forceMaterialization) {
            // add this identity mapValues call to prevent the source-changelog
            // optimization in kafka streams - we don't want this optimization to
            // be enabled because we cannot require symmetric serialization between
            // producer and KSQL (see https://issues.apache.org/jira/browse/KAFKA-10179
            // and https://github.com/confluentinc/ksql/issues/5673 for more details)
            table = source.mapValues(row -> row, materialized);
        } else {
            // if we know this table source is repartitioned later in the topology,
            // we do not need to force a materialization at this source step since the
            // re-partitioned topic will be used for any subsequent state stores, in lieu
            // of the original source topic, thus avoiding the issues above.
            // See https://github.com/confluentinc/ksql/issues/6650
            table = source.mapValues(row -> row);
        }
    }
    return table.transformValues(new AddKeyAndPseudoColumns<>(keyGenerator, streamSource.getPseudoColumnVersion(), streamSource.getSourceSchema().headers()));
}

Example 5 with RuntimeBuildContext

use of io.confluent.ksql.execution.runtime.RuntimeBuildContext in project ksql by confluentinc.

the class SinkBuilder method build.

public static <K> void build(final LogicalSchema schema, final Formats formats, final Optional<TimestampColumn> timestampColumn, final String topicName, final KStream<K, GenericRow> stream, final ExecutionKeyFactory<K> executionKeyFactory, final QueryContext queryContext, final RuntimeBuildContext buildContext) {
    final PhysicalSchema physicalSchema = PhysicalSchema.from(schema, formats.getKeyFeatures(), formats.getValueFeatures());
    final Serde<K> keySerde = executionKeyFactory.buildKeySerde(formats.getKeyFormat(), physicalSchema, queryContext);
    final Serde<GenericRow> valueSerde = buildContext.buildValueSerde(formats.getValueFormat(), physicalSchema, queryContext);
    final Optional<TransformTimestamp<K>> tsTransformer = timestampTransformer(buildContext, queryContext, schema, timestampColumn);
    final KStream<K, GenericRow> transformed = tsTransformer.map(t -> stream.transform(t, Named.as(TIMESTAMP_TRANSFORM_NAME + StreamsUtil.buildOpName(queryContext)))).orElse(stream);
    transformed.to(topicName, Produced.with(keySerde, valueSerde));
}