Examples with Formats io.confluent.ksql.execution.plan.Formats used on opensource projects

Example 6 with Formats

use of io.confluent.ksql.execution.plan.Formats in project ksql by confluentinc.

the class SinkBuilder method build.

public static <K> void build(final LogicalSchema schema, final Formats formats, final Optional<TimestampColumn> timestampColumn, final String topicName, final KStream<K, GenericRow> stream, final ExecutionKeyFactory<K> executionKeyFactory, final QueryContext queryContext, final RuntimeBuildContext buildContext) {
    final PhysicalSchema physicalSchema = PhysicalSchema.from(schema, formats.getKeyFeatures(), formats.getValueFeatures());
    final Serde<K> keySerde = executionKeyFactory.buildKeySerde(formats.getKeyFormat(), physicalSchema, queryContext);
    final Serde<GenericRow> valueSerde = buildContext.buildValueSerde(formats.getValueFormat(), physicalSchema, queryContext);
    final Optional<TransformTimestamp<K>> tsTransformer = timestampTransformer(buildContext, queryContext, schema, timestampColumn);
    final KStream<K, GenericRow> transformed = tsTransformer.map(t -> stream.transform(t, Named.as(TIMESTAMP_TRANSFORM_NAME + StreamsUtil.buildOpName(queryContext)))).orElse(stream);
    transformed.to(topicName, Produced.with(keySerde, valueSerde));
}

Example 7 with Formats

use of io.confluent.ksql.execution.plan.Formats in project ksql by confluentinc.

the class SourceBuilder method getPhysicalSchemaWithPseudoColumnsToMaterialize.

private static PhysicalSchema getPhysicalSchemaWithPseudoColumnsToMaterialize(final SourceStep<?> streamSource) {
    final Formats format = ((TableSource) streamSource).getStateStoreFormats();
    final LogicalSchema withPseudoCols = streamSource.getSourceSchema().withPseudoColumnsToMaterialize(streamSource.getPseudoColumnVersion());
    return PhysicalSchema.from(withPseudoCols, format.getKeyFeatures(), format.getValueFeatures());
}

Example 8 with Formats

use of io.confluent.ksql.execution.plan.Formats in project ksql by confluentinc.

the class ForeignKeyTableTableJoinBuilder method build.

public static <KLeftT, KRightT> KTableHolder<KLeftT> build(final KTableHolder<KLeftT> left, final KTableHolder<KRightT> right, final ForeignKeyTableTableJoin<KLeftT, KRightT> join, final RuntimeBuildContext buildContext) {
    final LogicalSchema leftSchema = left.getSchema();
    final LogicalSchema rightSchema = right.getSchema();
    final ProcessingLogger logger = buildContext.getProcessingLogger(join.getProperties().getQueryContext());
    final ExpressionEvaluator expressionEvaluator;
    final CodeGenRunner codeGenRunner = new CodeGenRunner(leftSchema, buildContext.getKsqlConfig(), buildContext.getFunctionRegistry());
    final Optional<ColumnName> leftColumnName = join.getLeftJoinColumnName();
    final Optional<Expression> leftJoinExpression = join.getLeftJoinExpression();
    if (leftColumnName.isPresent()) {
        expressionEvaluator = codeGenRunner.buildCodeGenFromParseTree(new UnqualifiedColumnReferenceExp(leftColumnName.get()), "Left Join Expression");
    } else if (leftJoinExpression.isPresent()) {
        expressionEvaluator = codeGenRunner.buildCodeGenFromParseTree(leftJoinExpression.get(), "Left Join Expression");
    } else {
        throw new IllegalStateException("Both leftColumnName and leftJoinExpression are empty.");
    }
    final ForeignKeyJoinParams<KRightT> joinParams = ForeignKeyJoinParamsFactory.create(expressionEvaluator, leftSchema, rightSchema, logger);
    final Formats formats = join.getFormats();
    final PhysicalSchema physicalSchema = PhysicalSchema.from(joinParams.getSchema(), formats.getKeyFeatures(), formats.getValueFeatures());
    final Serde<KLeftT> keySerde = left.getExecutionKeyFactory().buildKeySerde(formats.getKeyFormat(), physicalSchema, join.getProperties().getQueryContext());
    final Serde<GenericRow> valSerde = buildContext.buildValueSerde(formats.getValueFormat(), physicalSchema, join.getProperties().getQueryContext());
    final KTable<KLeftT, GenericRow> result;
    switch(join.getJoinType()) {
        case INNER:
            result = left.getTable().join(right.getTable(), joinParams.getKeyExtractor(), joinParams.getJoiner(), Materialized.with(keySerde, valSerde));
            break;
        case LEFT:
            result = left.getTable().leftJoin(right.getTable(), joinParams.getKeyExtractor(), joinParams.getJoiner(), Materialized.with(keySerde, valSerde));
            break;
        default:
            throw new IllegalStateException("invalid join type: " + join.getJoinType());
    }
    return KTableHolder.unmaterialized(result, joinParams.getSchema(), left.getExecutionKeyFactory());
}

Example 9 with Formats

use of io.confluent.ksql.execution.plan.Formats in project ksql by confluentinc.

the class StreamTableJoinBuilder method build.

public static <K> KStreamHolder<K> build(final KStreamHolder<K> left, final KTableHolder<K> right, final StreamTableJoin<K> join, final RuntimeBuildContext buildContext, final JoinedFactory joinedFactory) {
    final Formats leftFormats = join.getInternalFormats();
    final QueryContext queryContext = join.getProperties().getQueryContext();
    final QueryContext.Stacker stacker = QueryContext.Stacker.of(queryContext);
    final LogicalSchema leftSchema = left.getSchema();
    final PhysicalSchema leftPhysicalSchema = PhysicalSchema.from(leftSchema, leftFormats.getKeyFeatures(), leftFormats.getValueFeatures());
    final Serde<GenericRow> leftSerde = buildContext.buildValueSerde(leftFormats.getValueFormat(), leftPhysicalSchema, stacker.push(SERDE_CTX).getQueryContext());
    final Serde<K> keySerde = left.getExecutionKeyFactory().buildKeySerde(leftFormats.getKeyFormat(), leftPhysicalSchema, queryContext);
    final Joined<K, GenericRow, GenericRow> joined = joinedFactory.create(keySerde, leftSerde, null, StreamsUtil.buildOpName(queryContext));
    final LogicalSchema rightSchema = right.getSchema();
    final JoinParams joinParams = JoinParamsFactory.create(join.getKeyColName(), leftSchema, rightSchema);
    final KStream<K, GenericRow> result;
    switch(join.getJoinType()) {
        case LEFT:
            result = left.getStream().leftJoin(right.getTable(), joinParams.getJoiner(), joined);
            break;
        case INNER:
            result = left.getStream().join(right.getTable(), joinParams.getJoiner(), joined);
            break;
        default:
            throw new IllegalStateException("invalid join type");
    }
    return left.withStream(result, joinParams.getSchema());
}

Example 10 with Formats

use of io.confluent.ksql.execution.plan.Formats in project ksql by confluentinc.

the class StreamGroupByBuilderBase method build.

public KGroupedStreamHolder build(final KStreamHolder<GenericKey> stream, final StreamGroupByKey step) {
    final LogicalSchema sourceSchema = stream.getSchema();
    final QueryContext queryContext = step.getProperties().getQueryContext();
    final Formats formats = step.getInternalFormats();
    final Grouped<GenericKey, GenericRow> grouped = buildGrouped(formats, sourceSchema, queryContext, buildContext, groupedFactory);
    return KGroupedStreamHolder.of(stream.getStream().groupByKey(grouped), stream.getSchema());
}