Examples with PhysicalSchema io.confluent.ksql.schema.ksql.PhysicalSchema used on opensource projects

Example 16 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class SourceBuilderV1 method buildWindowedTable.

KTableHolder<Windowed<GenericKey>> buildWindowedTable(final RuntimeBuildContext buildContext, final SourceStep<KTableHolder<Windowed<GenericKey>>> source, final ConsumedFactory consumedFactory, final MaterializedFactory materializedFactory, final PlanInfo planInfo) {
    final PhysicalSchema physicalSchema = getPhysicalSchema(source);
    final Serde<GenericRow> valueSerde = getValueSerde(buildContext, source, physicalSchema);
    final WindowInfo windowInfo;
    if (source instanceof WindowedTableSource) {
        windowInfo = ((WindowedTableSource) source).getWindowInfo();
    } else {
        throw new IllegalArgumentException("Expected a version of WindowedTableSource");
    }
    final Serde<Windowed<GenericKey>> keySerde = SourceBuilderUtils.getWindowedKeySerde(source, physicalSchema, buildContext, windowInfo);
    final Consumed<Windowed<GenericKey>, GenericRow> consumed = buildSourceConsumed(source, keySerde, valueSerde, AutoOffsetReset.EARLIEST, buildContext, consumedFactory);
    final String stateStoreName = tableChangeLogOpName(source.getProperties());
    final Materialized<Windowed<GenericKey>, GenericRow, KeyValueStore<Bytes, byte[]>> materialized = materializedFactory.create(keySerde, valueSerde, stateStoreName);
    final KTable<Windowed<GenericKey>, GenericRow> ktable = buildKTable(source, buildContext, consumed, windowedKeyGenerator(source.getSourceSchema()), materialized, valueSerde, stateStoreName, planInfo);
    return KTableHolder.materialized(ktable, buildSchema(source, true), ExecutionKeyFactory.windowed(buildContext, windowInfo), MaterializationInfo.builder(stateStoreName, physicalSchema.logicalSchema()));
}

Example 17 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class StreamGroupByBuilderBase method buildGrouped.

private static Grouped<GenericKey, GenericRow> buildGrouped(final Formats formats, final LogicalSchema schema, final QueryContext queryContext, final RuntimeBuildContext buildContext, final GroupedFactory groupedFactory) {
    final PhysicalSchema physicalSchema = PhysicalSchema.from(schema, formats.getKeyFeatures(), formats.getValueFeatures());
    final Serde<GenericKey> keySerde = buildContext.buildKeySerde(formats.getKeyFormat(), physicalSchema, queryContext);
    final Serde<GenericRow> valSerde = buildContext.buildValueSerde(formats.getValueFormat(), physicalSchema, queryContext);
    return groupedFactory.create(StreamsUtil.buildOpName(queryContext), keySerde, valSerde);
}

Example 18 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class CliTest method testSelectProject.

@Test
public void testSelectProject() {
    final Map<GenericKey, GenericRow> expectedResults = ImmutableMap.<GenericKey, GenericRow>builder().put(genericKey("ORDER_1"), genericRow("ITEM_1", 10.0, ImmutableList.of(100.0, 110.99, 90.0))).put(genericKey("ORDER_2"), genericRow("ITEM_2", 20.0, ImmutableList.of(10.0, 10.99, 9.0))).put(genericKey("ORDER_3"), genericRow("ITEM_3", 30.0, ImmutableList.of(10.0, 10.99, 91.0))).put(genericKey("ORDER_4"), genericRow("ITEM_4", 40.0, ImmutableList.of(10.0, 140.99, 94.0))).put(genericKey("ORDER_5"), genericRow("ITEM_5", 50.0, ImmutableList.of(160.0, 160.99, 98.0))).put(genericKey("ORDER_6"), genericRow("ITEM_8", 80.0, ImmutableList.of(1100.0, 1110.99, 970.0))).build();
    final PhysicalSchema resultSchema = PhysicalSchema.from(LogicalSchema.builder().keyColumn(ColumnName.of("ORDERID"), SqlTypes.STRING).valueColumn(ColumnName.of("ITEMID"), SqlTypes.STRING).valueColumn(ColumnName.of("ORDERUNITS"), SqlTypes.DOUBLE).valueColumn(ColumnName.of("PRICEARRAY"), SqlTypes.array(SqlTypes.DOUBLE)).build(), SerdeFeatures.of(), SerdeFeatures.of());
    testCreateStreamAsSelect("SELECT ORDERID, ITEMID, ORDERUNITS, PRICEARRAY " + "FROM " + ORDER_DATA_PROVIDER.sourceName() + ";", resultSchema, expectedResults);
}

Example 19 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class CliTest method testSelectUDFs.

@Test
public void testSelectUDFs() {
    final String queryString = String.format("SELECT " + "ORDERID, " + "ITEMID, " + "ORDERUNITS*10 AS Col1, " + "PRICEARRAY[1]+10 AS Col2, " + "KEYVALUEMAP['key1']*KEYVALUEMAP['key2']+10 AS Col3, " + "PRICEARRAY[2]>1000 AS Col4 " + "FROM %s " + "WHERE ORDERUNITS > 20 AND ITEMID LIKE '%%_8';", ORDER_DATA_PROVIDER.sourceName());
    final PhysicalSchema resultSchema = PhysicalSchema.from(LogicalSchema.builder().keyColumn(ColumnName.of("ORDERID"), SqlTypes.STRING).valueColumn(ColumnName.of("ITEMID"), SqlTypes.STRING).valueColumn(ColumnName.of("COL1"), SqlTypes.DOUBLE).valueColumn(ColumnName.of("COL2"), SqlTypes.DOUBLE).valueColumn(ColumnName.of("COL3"), SqlTypes.DOUBLE).valueColumn(ColumnName.of("COL4"), SqlTypes.BOOLEAN).build(), SerdeFeatures.of(), SerdeFeatures.of());
    final Map<GenericKey, GenericRow> expectedResults = ImmutableMap.of(genericKey("ORDER_6"), genericRow("ITEM_8", 800.0, 1110.0, 12.0, true));
    testCreateStreamAsSelect(queryString, resultSchema, expectedResults);
}

Example 20 with PhysicalSchema

use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.

the class QueryBuilder method buildPersistentQueryInSharedRuntime.

@SuppressWarnings("ParameterNumber")
PersistentQueryMetadata buildPersistentQueryInSharedRuntime(final KsqlConfig ksqlConfig, final KsqlConstants.PersistentQueryType persistentQueryType, final String statementText, final QueryId queryId, final Optional<DataSource> sinkDataSource, final Set<DataSource> sources, final ExecutionStep<?> physicalPlan, final String planSummary, final QueryMetadata.Listener listener, final Supplier<List<PersistentQueryMetadata>> allPersistentQueries, final String applicationId, final MetricCollectors metricCollectors) {
    final SharedKafkaStreamsRuntime sharedKafkaStreamsRuntime = getKafkaStreamsInstance(applicationId, sources.stream().map(DataSource::getName).collect(Collectors.toSet()), queryId, metricCollectors);
    final Map<String, Object> queryOverrides = sharedKafkaStreamsRuntime.getStreamProperties();
    final LogicalSchema logicalSchema;
    final KeyFormat keyFormat;
    final ValueFormat valueFormat;
    final KsqlTopic ksqlTopic;
    switch(persistentQueryType) {
        // CREATE_SOURCE does not have a sink, so the schema is obtained from the query source
        case CREATE_SOURCE:
            final DataSource dataSource = Iterables.getOnlyElement(sources);
            logicalSchema = dataSource.getSchema();
            keyFormat = dataSource.getKsqlTopic().getKeyFormat();
            valueFormat = dataSource.getKsqlTopic().getValueFormat();
            ksqlTopic = dataSource.getKsqlTopic();
            break;
        default:
            logicalSchema = sinkDataSource.get().getSchema();
            keyFormat = sinkDataSource.get().getKsqlTopic().getKeyFormat();
            valueFormat = sinkDataSource.get().getKsqlTopic().getValueFormat();
            ksqlTopic = sinkDataSource.get().getKsqlTopic();
            break;
    }
    final PhysicalSchema querySchema = PhysicalSchema.from(logicalSchema, keyFormat.getFeatures(), valueFormat.getFeatures());
    final NamedTopologyBuilder namedTopologyBuilder = sharedKafkaStreamsRuntime.getKafkaStreams().newNamedTopologyBuilder(queryId.toString(), PropertiesUtil.asProperties(queryOverrides));
    final RuntimeBuildContext runtimeBuildContext = buildContext(applicationId, queryId, namedTopologyBuilder);
    final Object result = buildQueryImplementation(physicalPlan, runtimeBuildContext);
    final NamedTopology topology = namedTopologyBuilder.build();
    final Optional<MaterializationProviderBuilderFactory.MaterializationProviderBuilder> materializationProviderBuilder = getMaterializationInfo(result).map(info -> materializationProviderBuilderFactory.materializationProviderBuilder(info, querySchema, keyFormat, queryOverrides, applicationId, queryId.toString()));
    final Optional<ScalablePushRegistry> scalablePushRegistry = applyScalablePushProcessor(querySchema.logicalSchema(), result, allPersistentQueries, queryOverrides, applicationId, ksqlConfig, ksqlTopic, serviceContext);
    final BinPackedPersistentQueryMetadataImpl binPackedPersistentQueryMetadata = new BinPackedPersistentQueryMetadataImpl(persistentQueryType, statementText, querySchema, sources.stream().map(DataSource::getName).collect(Collectors.toSet()), planSummary, applicationId, topology, sharedKafkaStreamsRuntime, runtimeBuildContext.getSchemas(), config.getOverrides(), queryId, materializationProviderBuilder, physicalPlan, getUncaughtExceptionProcessingLogger(queryId), sinkDataSource, listener, queryOverrides, scalablePushRegistry, (streamsRuntime) -> getNamedTopology(streamsRuntime, queryId, applicationId, queryOverrides, physicalPlan));
    if (real) {
        return binPackedPersistentQueryMetadata;
    } else {
        return SandboxedBinPackedPersistentQueryMetadataImpl.of(binPackedPersistentQueryMetadata, listener);
    }
}