Examples with PhysicalSchema io.confluent.ksql.schema.ksql.PhysicalSchema used on opensource projects

Example 31 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class StreamStreamJoinBuilder method build.

// deprecation can be fixed after GRACE clause is mandatory
// (cf. `WithinExpression`)
@SuppressWarnings("deprecation")
public static <K> KStreamHolder<K> build(final KStreamHolder<K> left, final KStreamHolder<K> right, final StreamStreamJoin<K> join, final RuntimeBuildContext buildContext, final StreamJoinedFactory streamJoinedFactory) {
    final Formats leftFormats = join.getLeftInternalFormats();
    final QueryContext queryContext = join.getProperties().getQueryContext();
    final QueryContext.Stacker stacker = QueryContext.Stacker.of(queryContext);
    final LogicalSchema leftSchema = left.getSchema();
    final PhysicalSchema leftPhysicalSchema = PhysicalSchema.from(leftSchema, leftFormats.getKeyFeatures(), leftFormats.getValueFeatures());
    final Serde<GenericRow> leftSerde = buildContext.buildValueSerde(leftFormats.getValueFormat(), leftPhysicalSchema, stacker.push(LEFT_SERDE_CTX).getQueryContext());
    final Formats rightFormats = join.getRightInternalFormats();
    final LogicalSchema rightSchema = right.getSchema();
    final PhysicalSchema rightPhysicalSchema = PhysicalSchema.from(rightSchema, rightFormats.getKeyFeatures(), rightFormats.getValueFeatures());
    final Serde<GenericRow> rightSerde = buildContext.buildValueSerde(rightFormats.getValueFormat(), rightPhysicalSchema, stacker.push(RIGHT_SERDE_CTX).getQueryContext());
    final Serde<K> keySerde = left.getExecutionKeyFactory().buildKeySerde(leftFormats.getKeyFormat(), leftPhysicalSchema, queryContext);
    final StreamJoined<K, GenericRow, GenericRow> joined = streamJoinedFactory.create(keySerde, leftSerde, rightSerde, StreamsUtil.buildOpName(queryContext), StreamsUtil.buildOpName(queryContext));
    final JoinParams joinParams = JoinParamsFactory.create(join.getKeyColName(), leftSchema, rightSchema);
    JoinWindows joinWindows;
    // which enables the "spurious" results bugfix with left/outer joins (see KAFKA-10847).
    if (join.getGraceMillis().isPresent()) {
        joinWindows = JoinWindows.ofTimeDifferenceAndGrace(join.getBeforeMillis(), join.getGraceMillis().get());
    } else {
        joinWindows = JoinWindows.of(join.getBeforeMillis());
    }
    joinWindows = joinWindows.after(join.getAfterMillis());
    final KStream<K, GenericRow> result;
    switch(join.getJoinType()) {
        case LEFT:
            result = left.getStream().leftJoin(right.getStream(), joinParams.getJoiner(), joinWindows, joined);
            break;
        case OUTER:
            result = left.getStream().outerJoin(right.getStream(), joinParams.getJoiner(), joinWindows, joined);
            break;
        case INNER:
            result = left.getStream().join(right.getStream(), joinParams.getJoiner(), joinWindows, joined);
            break;
        default:
            throw new IllegalStateException("invalid join type");
    }
    return left.withStream(result, joinParams.getSchema());
}

Example 32 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class UdfIntTest method consumeOutputMessages.

private Map<GenericKey, GenericRow> consumeOutputMessages() {
    final DataSource source = ksqlContext.getMetaStore().getSource(SourceName.of(resultStreamName));
    final PhysicalSchema resultSchema = PhysicalSchema.from(source.getSchema(), source.getKsqlTopic().getKeyFormat().getFeatures(), source.getKsqlTopic().getValueFormat().getFeatures());
    return TEST_HARNESS.verifyAvailableUniqueRows(resultStreamName, 1, KAFKA, testData.format, resultSchema);
}

Example 33 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class InsertsSubscriber method createInsertsSubscriber.

public static InsertsSubscriber createInsertsSubscriber(final ServiceContext serviceContext, final JsonObject properties, final DataSource dataSource, final KsqlConfig ksqlConfig, final Context context, final Subscriber<InsertResult> acksSubscriber, final WorkerExecutor workerExecutor) {
    final KsqlConfig configCopy = ksqlConfig.cloneWithPropertyOverwrite(properties.getMap());
    final Producer<byte[], byte[]> producer = serviceContext.getKafkaClientSupplier().getProducer(configCopy.originals());
    final PhysicalSchema physicalSchema = PhysicalSchema.from(dataSource.getSchema(), dataSource.getKsqlTopic().getKeyFormat().getFeatures(), dataSource.getKsqlTopic().getValueFormat().getFeatures());
    final KeySerdeFactory keySerdeFactory = new GenericKeySerDe();
    final Serde<GenericKey> keySerde = keySerdeFactory.create(dataSource.getKsqlTopic().getKeyFormat().getFormatInfo(), physicalSchema.keySchema(), ksqlConfig, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
    final ValueSerdeFactory valueSerdeFactory = new GenericRowSerDe();
    final Serde<GenericRow> valueSerde = valueSerdeFactory.create(dataSource.getKsqlTopic().getValueFormat().getFormatInfo(), physicalSchema.valueSchema(), ksqlConfig, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
    final BufferedPublisher<InsertResult> acksPublisher = new BufferedPublisher<>(context);
    acksPublisher.subscribe(acksSubscriber);
    return new InsertsSubscriber(context, producer, dataSource, keySerde.serializer(), valueSerde.serializer(), acksPublisher, workerExecutor);
}

Example 34 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class StreamAggregateBuilderTest method shouldReturnCorrectSerdeForWindowedAggregate.

@Test
public void shouldReturnCorrectSerdeForWindowedAggregate() {
    for (final Runnable given : given()) {
        // Given:
        clearInvocations(groupedStream, timeWindowedStream, sessionWindowedStream, aggregated, buildContext);
        given.run();
        // When:
        final KTableHolder<Windowed<GenericKey>> tableHolder = windowedAggregate.build(planBuilder, planInfo);
        // Then:
        final ExecutionKeyFactory<Windowed<GenericKey>> serdeFactory = tableHolder.getExecutionKeyFactory();
        final FormatInfo mockFormat = mock(FormatInfo.class);
        final PhysicalSchema mockSchema = mock(PhysicalSchema.class);
        final QueryContext mockCtx = mock(QueryContext.class);
        serdeFactory.buildKeySerde(mockFormat, mockSchema, mockCtx);
        verify(buildContext).buildKeySerde(same(mockFormat), eq(windowedAggregate.getWindowExpression().getWindowInfo()), same(mockSchema), same(mockCtx));
    }
}