Examples with PhysicalSchema io.confluent.ksql.schema.ksql.PhysicalSchema used on opensource projects

Example 26 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class ForeignKeyTableTableJoinBuilder method build.

public static <KLeftT, KRightT> KTableHolder<KLeftT> build(final KTableHolder<KLeftT> left, final KTableHolder<KRightT> right, final ForeignKeyTableTableJoin<KLeftT, KRightT> join, final RuntimeBuildContext buildContext) {
    final LogicalSchema leftSchema = left.getSchema();
    final LogicalSchema rightSchema = right.getSchema();
    final ProcessingLogger logger = buildContext.getProcessingLogger(join.getProperties().getQueryContext());
    final ExpressionEvaluator expressionEvaluator;
    final CodeGenRunner codeGenRunner = new CodeGenRunner(leftSchema, buildContext.getKsqlConfig(), buildContext.getFunctionRegistry());
    final Optional<ColumnName> leftColumnName = join.getLeftJoinColumnName();
    final Optional<Expression> leftJoinExpression = join.getLeftJoinExpression();
    if (leftColumnName.isPresent()) {
        expressionEvaluator = codeGenRunner.buildCodeGenFromParseTree(new UnqualifiedColumnReferenceExp(leftColumnName.get()), "Left Join Expression");
    } else if (leftJoinExpression.isPresent()) {
        expressionEvaluator = codeGenRunner.buildCodeGenFromParseTree(leftJoinExpression.get(), "Left Join Expression");
    } else {
        throw new IllegalStateException("Both leftColumnName and leftJoinExpression are empty.");
    }
    final ForeignKeyJoinParams<KRightT> joinParams = ForeignKeyJoinParamsFactory.create(expressionEvaluator, leftSchema, rightSchema, logger);
    final Formats formats = join.getFormats();
    final PhysicalSchema physicalSchema = PhysicalSchema.from(joinParams.getSchema(), formats.getKeyFeatures(), formats.getValueFeatures());
    final Serde<KLeftT> keySerde = left.getExecutionKeyFactory().buildKeySerde(formats.getKeyFormat(), physicalSchema, join.getProperties().getQueryContext());
    final Serde<GenericRow> valSerde = buildContext.buildValueSerde(formats.getValueFormat(), physicalSchema, join.getProperties().getQueryContext());
    final KTable<KLeftT, GenericRow> result;
    switch(join.getJoinType()) {
        case INNER:
            result = left.getTable().join(right.getTable(), joinParams.getKeyExtractor(), joinParams.getJoiner(), Materialized.with(keySerde, valSerde));
            break;
        case LEFT:
            result = left.getTable().leftJoin(right.getTable(), joinParams.getKeyExtractor(), joinParams.getJoiner(), Materialized.with(keySerde, valSerde));
            break;
        default:
            throw new IllegalStateException("invalid join type: " + join.getJoinType());
    }
    return KTableHolder.unmaterialized(result, joinParams.getSchema(), left.getExecutionKeyFactory());
}

Example 27 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class StandaloneExecutorFunctionalTest method shouldHandleJsonWithSchemas.

@Test
public void shouldHandleJsonWithSchemas() {
    // Given:
    givenScript("" + "CREATE STREAM S (ROWKEY STRING KEY, ORDERTIME BIGINT)" + "    WITH (kafka_topic='" + JSON_TOPIC + "', value_format='json');\n" + "\n" + "CREATE TABLE T (ROWKEY STRING PRIMARY KEY, ORDERTIME BIGINT) " + "    WITH (kafka_topic='" + JSON_TOPIC + "', value_format='json');\n" + "\n" + "SET 'auto.offset.reset' = 'earliest';" + "\n" + "CREATE STREAM " + s1 + " AS SELECT * FROM S;\n" + "\n" + "INSERT INTO " + s1 + " SELECT * FROM S;\n" + "\n" + "CREATE TABLE " + t1 + " AS SELECT * FROM T;\n" + "\n" + "UNSET 'auto.offset.reset';" + "\n" + "CREATE STREAM " + s2 + " AS SELECT * FROM S;\n");
    final PhysicalSchema dataSchema = PhysicalSchema.from(LogicalSchema.builder().keyColumn(SystemColumns.ROWKEY_NAME, SqlTypes.STRING).valueColumn(ColumnName.of("ORDERTIME"), SqlTypes.BIGINT).build(), SerdeFeatures.of(), SerdeFeatures.of());
    // When:
    standalone.startAsync();
    // Then:
    // CSAS and INSERT INTO both input into S1:
    TEST_HARNESS.verifyAvailableRows(s1, DATA_SIZE * 2, KAFKA, JSON, dataSchema);
    // CTAS only into T1:
    TEST_HARNESS.verifyAvailableUniqueRows(t1, UNIQUE_DATA_SIZE, KAFKA, JSON, dataSchema);
    // S2 should be empty as 'auto.offset.reset' unset:
    TEST_HARNESS.verifyAvailableUniqueRows(s2, 0, KAFKA, JSON, dataSchema);
}

Example 28 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class QueryBuilder method buildPersistentQueryInDedicatedRuntime.

@SuppressWarnings("ParameterNumber")
PersistentQueryMetadata buildPersistentQueryInDedicatedRuntime(final KsqlConfig ksqlConfig, final KsqlConstants.PersistentQueryType persistentQueryType, final String statementText, final QueryId queryId, final Optional<DataSource> sinkDataSource, final Set<DataSource> sources, final ExecutionStep<?> physicalPlan, final String planSummary, final QueryMetadata.Listener listener, final Supplier<List<PersistentQueryMetadata>> allPersistentQueries, final StreamsBuilder streamsBuilder, final MetricCollectors metricCollectors) {
    final String applicationId = QueryApplicationId.build(ksqlConfig, true, queryId);
    final Map<String, Object> streamsProperties = buildStreamsProperties(applicationId, Optional.of(queryId), metricCollectors, config.getConfig(true), processingLogContext);
    final LogicalSchema logicalSchema;
    final KeyFormat keyFormat;
    final ValueFormat valueFormat;
    final KsqlTopic ksqlTopic;
    switch(persistentQueryType) {
        // CREATE_SOURCE does not have a sink, so the schema is obtained from the query source
        case CREATE_SOURCE:
            final DataSource dataSource = Iterables.getOnlyElement(sources);
            logicalSchema = dataSource.getSchema();
            keyFormat = dataSource.getKsqlTopic().getKeyFormat();
            valueFormat = dataSource.getKsqlTopic().getValueFormat();
            ksqlTopic = dataSource.getKsqlTopic();
            break;
        default:
            logicalSchema = sinkDataSource.get().getSchema();
            keyFormat = sinkDataSource.get().getKsqlTopic().getKeyFormat();
            valueFormat = sinkDataSource.get().getKsqlTopic().getValueFormat();
            ksqlTopic = sinkDataSource.get().getKsqlTopic();
            break;
    }
    final PhysicalSchema querySchema = PhysicalSchema.from(logicalSchema, keyFormat.getFeatures(), valueFormat.getFeatures());
    final RuntimeBuildContext runtimeBuildContext = buildContext(applicationId, queryId, streamsBuilder);
    final Object result = buildQueryImplementation(physicalPlan, runtimeBuildContext);
    final Topology topology = streamsBuilder.build(PropertiesUtil.asProperties(streamsProperties));
    final Optional<MaterializationProviderBuilderFactory.MaterializationProviderBuilder> materializationProviderBuilder = getMaterializationInfo(result).map(info -> materializationProviderBuilderFactory.materializationProviderBuilder(info, querySchema, keyFormat, streamsProperties, applicationId, queryId.toString()));
    final Optional<ScalablePushRegistry> scalablePushRegistry = applyScalablePushProcessor(querySchema.logicalSchema(), result, allPersistentQueries, streamsProperties, applicationId, ksqlConfig, ksqlTopic, serviceContext);
    return new PersistentQueryMetadataImpl(persistentQueryType, statementText, querySchema, sources.stream().map(DataSource::getName).collect(Collectors.toSet()), sinkDataSource, planSummary, queryId, materializationProviderBuilder, applicationId, topology, kafkaStreamsBuilder, runtimeBuildContext.getSchemas(), streamsProperties, config.getOverrides(), ksqlConfig.getLong(KSQL_SHUTDOWN_TIMEOUT_MS_CONFIG), getConfiguredQueryErrorClassifier(ksqlConfig, applicationId), physicalPlan, ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_ERROR_MAX_QUEUE_SIZE), getUncaughtExceptionProcessingLogger(queryId), ksqlConfig.getLong(KsqlConfig.KSQL_QUERY_RETRY_BACKOFF_INITIAL_MS), ksqlConfig.getLong(KsqlConfig.KSQL_QUERY_RETRY_BACKOFF_MAX_MS), listener, scalablePushRegistry);
}

Example 29 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class SourceBuilder method buildTableMaterialized.

@Override
Materialized<GenericKey, GenericRow, KeyValueStore<Bytes, byte[]>> buildTableMaterialized(final SourceStep<KTableHolder<GenericKey>> source, final RuntimeBuildContext buildContext, final MaterializedFactory materializedFactory, final Serde<GenericKey> sourceKeySerde, final Serde<GenericRow> sourceValueSerde, final String stateStoreName) {
    final PhysicalSchema physicalSchema = getPhysicalSchemaWithPseudoColumnsToMaterialize(source);
    final QueryContext queryContext = addMaterializedContext(source);
    final Serde<GenericRow> valueSerdeToMaterialize = getValueSerde(buildContext, source, physicalSchema, queryContext);
    final Serde<GenericKey> keySerdeToMaterialize = getKeySerde(source, physicalSchema, buildContext, queryContext);
    return materializedFactory.create(keySerdeToMaterialize, valueSerdeToMaterialize, stateStoreName);
}

Example 30 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class StreamTableJoinBuilder method build.

public static <K> KStreamHolder<K> build(final KStreamHolder<K> left, final KTableHolder<K> right, final StreamTableJoin<K> join, final RuntimeBuildContext buildContext, final JoinedFactory joinedFactory) {
    final Formats leftFormats = join.getInternalFormats();
    final QueryContext queryContext = join.getProperties().getQueryContext();
    final QueryContext.Stacker stacker = QueryContext.Stacker.of(queryContext);
    final LogicalSchema leftSchema = left.getSchema();
    final PhysicalSchema leftPhysicalSchema = PhysicalSchema.from(leftSchema, leftFormats.getKeyFeatures(), leftFormats.getValueFeatures());
    final Serde<GenericRow> leftSerde = buildContext.buildValueSerde(leftFormats.getValueFormat(), leftPhysicalSchema, stacker.push(SERDE_CTX).getQueryContext());
    final Serde<K> keySerde = left.getExecutionKeyFactory().buildKeySerde(leftFormats.getKeyFormat(), leftPhysicalSchema, queryContext);
    final Joined<K, GenericRow, GenericRow> joined = joinedFactory.create(keySerde, leftSerde, null, StreamsUtil.buildOpName(queryContext));
    final LogicalSchema rightSchema = right.getSchema();
    final JoinParams joinParams = JoinParamsFactory.create(join.getKeyColName(), leftSchema, rightSchema);
    final KStream<K, GenericRow> result;
    switch(join.getJoinType()) {
        case LEFT:
            result = left.getStream().leftJoin(right.getTable(), joinParams.getJoiner(), joined);
            break;
        case INNER:
            result = left.getStream().join(right.getTable(), joinParams.getJoiner(), joined);
            break;
        default:
            throw new IllegalStateException("invalid join type");
    }
    return left.withStream(result, joinParams.getSchema());
}