Examples with PlanInfo io.confluent.ksql.execution.plan.PlanInfo used on opensource projects

Example 6 with PlanInfo

use of io.confluent.ksql.execution.plan.PlanInfo in project ksql by confluentinc.

the class SourceBuilderV1 method buildKTable.

@Override
<K> KTable<K, GenericRow> buildKTable(final SourceStep<?> streamSource, final RuntimeBuildContext buildContext, final Consumed<K, GenericRow> consumed, final Function<K, Collection<?>> keyGenerator, final Materialized<K, GenericRow, KeyValueStore<Bytes, byte[]>> materialized, final Serde<GenericRow> valueSerde, final String stateStoreName, final PlanInfo planInfo) {
    validateNotUsingOldExecutionStepWithNewQueries(streamSource);
    final boolean forceChangelog = streamSource instanceof TableSourceV1 && ((TableSourceV1) streamSource).isForceChangelog();
    final KTable<K, GenericRow> table;
    if (!forceChangelog) {
        final String changelogTopic = changelogTopic(buildContext, stateStoreName);
        final Callback onFailure = getRegisterCallback(buildContext, streamSource.getFormats().getValueFormat());
        table = buildContext.getStreamsBuilder().table(streamSource.getTopicName(), consumed.withValueSerde(StaticTopicSerde.wrap(changelogTopic, valueSerde, onFailure)), materialized);
    } else {
        final KTable<K, GenericRow> source = buildContext.getStreamsBuilder().table(streamSource.getTopicName(), consumed);
        final boolean forceMaterialization = !planInfo.isRepartitionedInPlan(streamSource);
        if (forceMaterialization) {
            // add this identity mapValues call to prevent the source-changelog
            // optimization in kafka streams - we don't want this optimization to
            // be enabled because we cannot require symmetric serialization between
            // producer and KSQL (see https://issues.apache.org/jira/browse/KAFKA-10179
            // and https://github.com/confluentinc/ksql/issues/5673 for more details)
            table = source.mapValues(row -> row, materialized);
        } else {
            // if we know this table source is repartitioned later in the topology,
            // we do not need to force a materialization at this source step since the
            // re-partitioned topic will be used for any subsequent state stores, in lieu
            // of the original source topic, thus avoiding the issues above.
            // See https://github.com/confluentinc/ksql/issues/6650
            table = source.mapValues(row -> row);
        }
    }
    return table.transformValues(new AddKeyAndPseudoColumns<>(keyGenerator, streamSource.getPseudoColumnVersion(), streamSource.getSourceSchema().headers()));
}

Example 7 with PlanInfo

use of io.confluent.ksql.execution.plan.PlanInfo in project ksql by confluentinc.

the class EngineExecutor method planQuery.

private ExecutorPlans planQuery(final ConfiguredStatement<?> statement, final Query query, final Optional<Sink> sink, final Optional<String> withQueryId, final MetaStore metaStore) {
    final QueryEngine queryEngine = engineContext.createQueryEngine(serviceContext);
    final KsqlConfig ksqlConfig = config.getConfig(true);
    final OutputNode outputNode = QueryEngine.buildQueryLogicalPlan(query, sink, metaStore, ksqlConfig, getRowpartitionRowoffsetEnabled(ksqlConfig, statement.getSessionConfig().getOverrides()), statement.getStatementText());
    final LogicalPlanNode logicalPlan = new LogicalPlanNode(statement.getStatementText(), Optional.of(outputNode));
    final QueryId queryId = QueryIdUtil.buildId(statement.getStatement(), engineContext, engineContext.idGenerator(), outputNode, ksqlConfig.getBoolean(KsqlConfig.KSQL_CREATE_OR_REPLACE_ENABLED), withQueryId);
    if (withQueryId.isPresent() && engineContext.getQueryRegistry().getPersistentQuery(queryId).isPresent()) {
        throw new KsqlException(String.format("Query ID '%s' already exists.", queryId));
    }
    final Optional<PersistentQueryMetadata> persistentQueryMetadata = engineContext.getQueryRegistry().getPersistentQuery(queryId);
    final Optional<PlanInfo> oldPlanInfo;
    if (persistentQueryMetadata.isPresent()) {
        final ExecutionStep<?> oldPlan = persistentQueryMetadata.get().getPhysicalPlan();
        oldPlanInfo = Optional.of(oldPlan.extractPlanInfo(new PlanInfoExtractor()));
    } else {
        oldPlanInfo = Optional.empty();
    }
    final PhysicalPlan physicalPlan = queryEngine.buildPhysicalPlan(logicalPlan, config, metaStore, queryId, oldPlanInfo);
    return new ExecutorPlans(logicalPlan, physicalPlan);
}

Example 8 with PlanInfo

use of io.confluent.ksql.execution.plan.PlanInfo in project ksql by confluentinc.

the class SourceBuilderV1Test method givenDownstreamRepartition.

private static PlanInfo givenDownstreamRepartition(final ExecutionStep<?> sourceStep) {
    final PlanInfo mockPlanInfo = mock(PlanInfo.class);
    when(mockPlanInfo.isRepartitionedInPlan(sourceStep)).thenReturn(true);
    return mockPlanInfo;
}

Example 9 with PlanInfo

use of io.confluent.ksql.execution.plan.PlanInfo in project ksql by confluentinc.

the class SourceBuilderTest method shouldApplyCorrectTransformationsToSourceTableWithDownstreamRepartition.

@Test
@SuppressWarnings("unchecked")
public void shouldApplyCorrectTransformationsToSourceTableWithDownstreamRepartition() {
    // Given:
    givenUnwindowedSourceTable();
    final PlanInfo planInfo = givenDownstreamRepartition(tableSource);
    // When:
    final KTableHolder<GenericKey> builtKTable = tableSource.build(planBuilder, planInfo);
    // Then:
    assertThat(builtKTable.getTable(), is(kTable));
    final InOrder validator = inOrder(streamsBuilder, kTable);
    validator.verify(streamsBuilder).table(eq(TOPIC_NAME), eq(consumed));
    validator.verify(kTable, times(2)).transformValues(any(ValueTransformerWithKeySupplier.class));
    verify(consumedFactory).create(keySerde, valueSerde);
    verify(consumed).withTimestampExtractor(any());
    verify(consumed).withOffsetResetPolicy(AutoOffsetReset.EARLIEST);
    verify(kTable, never()).mapValues(any(ValueMapper.class), any(Materialized.class));
}

Example 10 with PlanInfo

use of io.confluent.ksql.execution.plan.PlanInfo in project ksql by confluentinc.

the class PlanInfoExtractorTest method shouldExtractSourceWithRepartition.

@Test
public void shouldExtractSourceWithRepartition() {
    // When:
    final PlanInfo planInfo = streamSourceRepartitioned.extractPlanInfo(planInfoExtractor);
    // Then:
    assertThat(planInfo.isRepartitionedInPlan(streamSource), is(true));
}