Examples with StreamSource io.confluent.ksql.execution.plan.StreamSource used on opensource projects

Example 1 with StreamSource

use of io.confluent.ksql.execution.plan.StreamSource in project ksql by confluentinc.

the class SourceBuilderV1Test method givenMultiColumnSourceStream.

private void givenMultiColumnSourceStream(final int pseudoColumnVersion) {
    when(buildContext.buildKeySerde(any(), any(), any())).thenReturn(keySerde);
    givenConsumed(consumed, keySerde);
    streamSource = new StreamSource(new ExecutionStepPropertiesV1(ctx), TOPIC_NAME, Formats.of(keyFormatInfo, valueFormatInfo, KEY_FEATURES, VALUE_FEATURES), TIMESTAMP_COLUMN, MULTI_COL_SOURCE_SCHEMA, OptionalInt.of(pseudoColumnVersion));
}

Example 2 with StreamSource

use of io.confluent.ksql.execution.plan.StreamSource in project ksql by confluentinc.

the class StepSchemaResolverTest method shouldResolveSchemaForStreamSource.

@Test
public void shouldResolveSchemaForStreamSource() {
    final StreamSource step = new StreamSource(PROPERTIES, "foo", formats, Optional.empty(), SCHEMA, OptionalInt.of(SystemColumns.CURRENT_PSEUDOCOLUMN_VERSION_NUMBER));
    // When:
    final LogicalSchema result = resolver.resolve(step, SCHEMA);
    // Then:
    assertThat(result, is(SCHEMA.withPseudoAndKeyColsInValue(false)));
}

Example 3 with StreamSource

use of io.confluent.ksql.execution.plan.StreamSource in project ksql by confluentinc.

the class SchemaKSourceFactory method buildStream.

private static SchemaKStream<?> buildStream(final PlanBuildContext buildContext, final DataSource dataSource, final Stacker contextStacker) {
    if (dataSource.getKsqlTopic().getKeyFormat().getWindowInfo().isPresent()) {
        throw new IllegalArgumentException("windowed");
    }
    final int pseudoColumnVersionToUse = determinePseudoColumnVersionToUse(buildContext);
    final StreamSource step = ExecutionStepFactory.streamSource(contextStacker, dataSource.getSchema(), dataSource.getKafkaTopicName(), Formats.from(dataSource.getKsqlTopic()), dataSource.getTimestampColumn(), pseudoColumnVersionToUse);
    return schemaKStream(buildContext, resolveSchema(buildContext, step, dataSource), dataSource.getKsqlTopic().getKeyFormat(), step);
}

Example 4 with StreamSource

use of io.confluent.ksql.execution.plan.StreamSource in project ksql by confluentinc.

the class SourceBuilderV1 method buildKTable.

@Override
<K> KTable<K, GenericRow> buildKTable(final SourceStep<?> streamSource, final RuntimeBuildContext buildContext, final Consumed<K, GenericRow> consumed, final Function<K, Collection<?>> keyGenerator, final Materialized<K, GenericRow, KeyValueStore<Bytes, byte[]>> materialized, final Serde<GenericRow> valueSerde, final String stateStoreName, final PlanInfo planInfo) {
    validateNotUsingOldExecutionStepWithNewQueries(streamSource);
    final boolean forceChangelog = streamSource instanceof TableSourceV1 && ((TableSourceV1) streamSource).isForceChangelog();
    final KTable<K, GenericRow> table;
    if (!forceChangelog) {
        final String changelogTopic = changelogTopic(buildContext, stateStoreName);
        final Callback onFailure = getRegisterCallback(buildContext, streamSource.getFormats().getValueFormat());
        table = buildContext.getStreamsBuilder().table(streamSource.getTopicName(), consumed.withValueSerde(StaticTopicSerde.wrap(changelogTopic, valueSerde, onFailure)), materialized);
    } else {
        final KTable<K, GenericRow> source = buildContext.getStreamsBuilder().table(streamSource.getTopicName(), consumed);
        final boolean forceMaterialization = !planInfo.isRepartitionedInPlan(streamSource);
        if (forceMaterialization) {
            // add this identity mapValues call to prevent the source-changelog
            // optimization in kafka streams - we don't want this optimization to
            // be enabled because we cannot require symmetric serialization between
            // producer and KSQL (see https://issues.apache.org/jira/browse/KAFKA-10179
            // and https://github.com/confluentinc/ksql/issues/5673 for more details)
            table = source.mapValues(row -> row, materialized);
        } else {
            // if we know this table source is repartitioned later in the topology,
            // we do not need to force a materialization at this source step since the
            // re-partitioned topic will be used for any subsequent state stores, in lieu
            // of the original source topic, thus avoiding the issues above.
            // See https://github.com/confluentinc/ksql/issues/6650
            table = source.mapValues(row -> row);
        }
    }
    return table.transformValues(new AddKeyAndPseudoColumns<>(keyGenerator, streamSource.getPseudoColumnVersion(), streamSource.getSourceSchema().headers()));
}

Example 5 with StreamSource

use of io.confluent.ksql.execution.plan.StreamSource in project ksql by confluentinc.

the class SourceBuilderV1Test method givenUnwindowedSourceStream.

private void givenUnwindowedSourceStream(final int pseudoColumnVersion) {
    when(buildContext.buildKeySerde(any(), any(), any())).thenReturn(keySerde);
    givenConsumed(consumed, keySerde);
    streamSource = new StreamSource(new ExecutionStepPropertiesV1(ctx), TOPIC_NAME, Formats.of(keyFormatInfo, valueFormatInfo, KEY_FEATURES, VALUE_FEATURES), TIMESTAMP_COLUMN, SOURCE_SCHEMA, OptionalInt.of(pseudoColumnVersion));
}