Examples with SchemaKStream io.confluent.ksql.structured.SchemaKStream used on opensource projects

Example 6 with SchemaKStream

use of io.confluent.ksql.structured.SchemaKStream in project ksql by confluentinc.

the class JoinNode method tableForJoin.

// package private for test
SchemaKTable tableForJoin(final StreamsBuilder builder, final KsqlConfig ksqlConfig, final KafkaTopicClient kafkaTopicClient, final FunctionRegistry functionRegistry, final Map<String, Object> props, final SchemaRegistryClient schemaRegistryClient) {
    Map<String, Object> joinTableProps = new HashMap<>(props);
    joinTableProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    final SchemaKStream schemaKStream = right.buildStream(builder, ksqlConfig, kafkaTopicClient, functionRegistry, joinTableProps, schemaRegistryClient);
    if (!(schemaKStream instanceof SchemaKTable)) {
        throw new KsqlException("Unsupported Join. Only stream-table joins are supported, but was " + getLeft() + "-" + getRight());
    }
    return (SchemaKTable) schemaKStream;
}

Example 7 with SchemaKStream

use of io.confluent.ksql.structured.SchemaKStream in project ksql by confluentinc.

the class KsqlStructuredDataOutputNode method buildStream.

@Override
public SchemaKStream buildStream(final StreamsBuilder builder, final KsqlConfig ksqlConfig, final KafkaTopicClient kafkaTopicClient, final FunctionRegistry functionRegistry, final Map<String, Object> props, final SchemaRegistryClient schemaRegistryClient) {
    final SchemaKStream schemaKStream = getSource().buildStream(builder, ksqlConfig, kafkaTopicClient, functionRegistry, props, schemaRegistryClient);
    final Set<Integer> rowkeyIndexes = SchemaUtil.getRowTimeRowKeyIndexes(getSchema());
    final Builder outputNodeBuilder = Builder.from(this);
    final Schema schema = SchemaUtil.removeImplicitRowTimeRowKeyFromSchema(getSchema());
    outputNodeBuilder.withSchema(schema);
    if (getTopicSerde() instanceof KsqlAvroTopicSerDe) {
        addAvroSchemaToResultTopic(outputNodeBuilder);
    }
    final Map<String, Object> outputProperties = getOutputProperties();
    if (outputProperties.containsKey(KsqlConfig.SINK_NUMBER_OF_PARTITIONS_PROPERTY)) {
        ksqlConfig.put(KsqlConfig.SINK_NUMBER_OF_PARTITIONS_PROPERTY, outputProperties.get(KsqlConfig.SINK_NUMBER_OF_PARTITIONS_PROPERTY));
    }
    if (outputProperties.containsKey(KsqlConfig.SINK_NUMBER_OF_REPLICAS_PROPERTY)) {
        ksqlConfig.put(KsqlConfig.SINK_NUMBER_OF_REPLICAS_PROPERTY, outputProperties.get(KsqlConfig.SINK_NUMBER_OF_REPLICAS_PROPERTY));
    }
    final SchemaKStream result = createOutputStream(schemaKStream, outputNodeBuilder, functionRegistry, outputProperties, schemaRegistryClient);
    final KsqlStructuredDataOutputNode noRowKey = outputNodeBuilder.build();
    createSinkTopic(noRowKey.getKafkaTopicName(), ksqlConfig, kafkaTopicClient, shoulBeCompacted(result));
    result.into(noRowKey.getKafkaTopicName(), noRowKey.getKsqlTopic().getKsqlTopicSerDe().getGenericRowSerde(noRowKey.getSchema(), ksqlConfig, false, schemaRegistryClient), rowkeyIndexes);
    result.setOutputNode(outputNodeBuilder.withSchema(SchemaUtil.addImplicitRowTimeRowKeyToSchema(noRowKey.getSchema())).build());
    return result;
}

Example 8 with SchemaKStream

use of io.confluent.ksql.structured.SchemaKStream in project ksql by confluentinc.

the class KsqlStructuredDataOutputNode method createOutputStream.

private SchemaKStream createOutputStream(final SchemaKStream schemaKStream, final KsqlStructuredDataOutputNode.Builder outputNodeBuilder, final FunctionRegistry functionRegistry, final Map<String, Object> outputProperties, final SchemaRegistryClient schemaRegistryClient) {
    if (schemaKStream instanceof SchemaKTable) {
        return schemaKStream;
    }
    final SchemaKStream result = new SchemaKStream(getSchema(), schemaKStream.getKstream(), this.getKeyField(), Collections.singletonList(schemaKStream), SchemaKStream.Type.SINK, functionRegistry, schemaRegistryClient);
    if (outputProperties.containsKey(DdlConfig.PARTITION_BY_PROPERTY)) {
        String keyFieldName = outputProperties.get(DdlConfig.PARTITION_BY_PROPERTY).toString();
        Field keyField = SchemaUtil.getFieldByName(result.getSchema(), keyFieldName).orElseThrow(() -> new KsqlException(String.format("Column %s does not exist in the result schema." + " Error in Partition By clause.", keyFieldName)));
        outputNodeBuilder.withKeyField(keyField);
        return result.selectKey(keyField, false);
    }
    return result;
}

Example 9 with SchemaKStream

use of io.confluent.ksql.structured.SchemaKStream in project ksql by confluentinc.

the class StructuredDataSourceNode method buildStream.

@Override
public SchemaKStream buildStream(final StreamsBuilder builder, final KsqlConfig ksqlConfig, final KafkaTopicClient kafkaTopicClient, final FunctionRegistry functionRegistry, final Map<String, Object> props, final SchemaRegistryClient schemaRegistryClient) {
    if (getTimestampField() != null) {
        int timestampColumnIndex = getTimeStampColumnIndex();
        ksqlConfig.put(KsqlConfig.KSQL_TIMESTAMP_COLUMN_INDEX, timestampColumnIndex);
    }
    KsqlTopicSerDe ksqlTopicSerDe = getStructuredDataSource().getKsqlTopic().getKsqlTopicSerDe();
    Serde<GenericRow> genericRowSerde = ksqlTopicSerDe.getGenericRowSerde(SchemaUtil.removeImplicitRowTimeRowKeyFromSchema(getSchema()), ksqlConfig, false, schemaRegistryClient);
    if (getDataSourceType() == StructuredDataSource.DataSourceType.KTABLE) {
        final KsqlTable table = (KsqlTable) getStructuredDataSource();
        final KTable kTable = createKTable(builder, getAutoOffsetReset(props), table, genericRowSerde, table.getKsqlTopic().getKsqlTopicSerDe().getGenericRowSerde(getSchema(), ksqlConfig, true, schemaRegistryClient));
        return new SchemaKTable(getSchema(), kTable, getKeyField(), new ArrayList<>(), table.isWindowed(), SchemaKStream.Type.SOURCE, functionRegistry, schemaRegistryClient);
    }
    return new SchemaKStream(getSchema(), builder.stream(getStructuredDataSource().getKsqlTopic().getKafkaTopicName(), Consumed.with(Serdes.String(), genericRowSerde)).mapValues(nonWindowedValueMapper).transformValues(new AddTimestampColumn()), getKeyField(), new ArrayList<>(), SchemaKStream.Type.SOURCE, functionRegistry, schemaRegistryClient);
}

Example 10 with SchemaKStream

use of io.confluent.ksql.structured.SchemaKStream in project ksql by confluentinc.

the class PhysicalPlanBuilder method buildPhysicalPlan.

public QueryMetadata buildPhysicalPlan(final Pair<String, PlanNode> statementPlanPair) throws Exception {
    final SchemaKStream resultStream = statementPlanPair.getRight().buildStream(builder, ksqlConfig, kafkaTopicClient, functionRegistry, overriddenStreamsProperties, schemaRegistryClient);
    final OutputNode outputNode = resultStream.outputNode();
    boolean isBareQuery = outputNode instanceof KsqlBareOutputNode;
    // the corresponding Kafka Streams job
    if (isBareQuery && !(resultStream instanceof QueuedSchemaKStream)) {
        throw new Exception(String.format("Mismatch between logical and physical output; " + "expected a QueuedSchemaKStream based on logical " + "KsqlBareOutputNode, found a %s instead", resultStream.getClass().getCanonicalName()));
    }
    String serviceId = getServiceId();
    String persistanceQueryPrefix = ksqlConfig.get(KsqlConfig.KSQL_PERSISTENT_QUERY_NAME_PREFIX_CONFIG).toString();
    String transientQueryPrefix = ksqlConfig.get(KsqlConfig.KSQL_TRANSIENT_QUERY_NAME_PREFIX_CONFIG).toString();
    if (isBareQuery) {
        return buildPlanForBareQuery((QueuedSchemaKStream) resultStream, (KsqlBareOutputNode) outputNode, serviceId, transientQueryPrefix, statementPlanPair.getLeft());
    } else if (outputNode instanceof KsqlStructuredDataOutputNode) {
        return buildPlanForStructuredOutputNode(statementPlanPair.getLeft(), resultStream, (KsqlStructuredDataOutputNode) outputNode, serviceId, persistanceQueryPrefix, statementPlanPair.getLeft());
    } else {
        throw new KsqlException("Sink data source of type: " + outputNode.getClass() + " is not supported.");
    }
}