Examples with SourceStep io.confluent.ksql.execution.plan.SourceStep used on opensource projects

Example 1 with SourceStep

use of io.confluent.ksql.execution.plan.SourceStep in project ksql by confluentinc.

the class SourceBuilderV1 method buildKTable.

@Override
<K> KTable<K, GenericRow> buildKTable(final SourceStep<?> streamSource, final RuntimeBuildContext buildContext, final Consumed<K, GenericRow> consumed, final Function<K, Collection<?>> keyGenerator, final Materialized<K, GenericRow, KeyValueStore<Bytes, byte[]>> materialized, final Serde<GenericRow> valueSerde, final String stateStoreName, final PlanInfo planInfo) {
    validateNotUsingOldExecutionStepWithNewQueries(streamSource);
    final boolean forceChangelog = streamSource instanceof TableSourceV1 && ((TableSourceV1) streamSource).isForceChangelog();
    final KTable<K, GenericRow> table;
    if (!forceChangelog) {
        final String changelogTopic = changelogTopic(buildContext, stateStoreName);
        final Callback onFailure = getRegisterCallback(buildContext, streamSource.getFormats().getValueFormat());
        table = buildContext.getStreamsBuilder().table(streamSource.getTopicName(), consumed.withValueSerde(StaticTopicSerde.wrap(changelogTopic, valueSerde, onFailure)), materialized);
    } else {
        final KTable<K, GenericRow> source = buildContext.getStreamsBuilder().table(streamSource.getTopicName(), consumed);
        final boolean forceMaterialization = !planInfo.isRepartitionedInPlan(streamSource);
        if (forceMaterialization) {
            // add this identity mapValues call to prevent the source-changelog
            // optimization in kafka streams - we don't want this optimization to
            // be enabled because we cannot require symmetric serialization between
            // producer and KSQL (see https://issues.apache.org/jira/browse/KAFKA-10179
            // and https://github.com/confluentinc/ksql/issues/5673 for more details)
            table = source.mapValues(row -> row, materialized);
        } else {
            // if we know this table source is repartitioned later in the topology,
            // we do not need to force a materialization at this source step since the
            // re-partitioned topic will be used for any subsequent state stores, in lieu
            // of the original source topic, thus avoiding the issues above.
            // See https://github.com/confluentinc/ksql/issues/6650
            table = source.mapValues(row -> row);
        }
    }
    return table.transformValues(new AddKeyAndPseudoColumns<>(keyGenerator, streamSource.getPseudoColumnVersion(), streamSource.getSourceSchema().headers()));
}

Example 2 with SourceStep

use of io.confluent.ksql.execution.plan.SourceStep in project ksql by confluentinc.

the class SchemaKSourceFactory method buildTable.

private static SchemaKTable<?> buildTable(final PlanBuildContext buildContext, final DataSource dataSource, final Stacker contextStacker) {
    final KeyFormat keyFormat = dataSource.getKsqlTopic().getKeyFormat();
    if (keyFormat.isWindowed()) {
        throw new IllegalArgumentException("windowed");
    }
    final SourceStep<KTableHolder<GenericKey>> step;
    final int pseudoColumnVersionToUse = determinePseudoColumnVersionToUse(buildContext);
    // If the old query has a v1 table step, continue to use it.
    // See https://github.com/confluentinc/ksql/pull/7990
    boolean useOldExecutionStepVersion = false;
    if (buildContext.getPlanInfo().isPresent()) {
        final Set<ExecutionStep<?>> sourceSteps = buildContext.getPlanInfo().get().getSources();
        useOldExecutionStepVersion = sourceSteps.stream().anyMatch(executionStep -> executionStep instanceof TableSourceV1);
    }
    if (useOldExecutionStepVersion && pseudoColumnVersionToUse != SystemColumns.LEGACY_PSEUDOCOLUMN_VERSION_NUMBER) {
        throw new IllegalStateException("TableSourceV2 was released in conjunction with pseudocolumn" + "version 1. Something has gone very wrong");
    }
    if (buildContext.getKsqlConfig().getBoolean(KsqlConfig.KSQL_ROWPARTITION_ROWOFFSET_ENABLED) && !useOldExecutionStepVersion) {
        step = ExecutionStepFactory.tableSource(contextStacker, dataSource.getSchema(), dataSource.getKafkaTopicName(), Formats.from(dataSource.getKsqlTopic()), dataSource.getTimestampColumn(), InternalFormats.of(keyFormat, Formats.from(dataSource.getKsqlTopic()).getValueFormat()), pseudoColumnVersionToUse);
    } else {
        step = ExecutionStepFactory.tableSourceV1(contextStacker, dataSource.getSchema(), dataSource.getKafkaTopicName(), Formats.from(dataSource.getKsqlTopic()), dataSource.getTimestampColumn(), pseudoColumnVersionToUse);
    }
    return schemaKTable(buildContext, resolveSchema(buildContext, step, dataSource), dataSource.getKsqlTopic().getKeyFormat(), step);
}