Examples with KsqlTable io.confluent.ksql.metastore.KsqlTable used on opensource projects

Example 1 with KsqlTable

use of io.confluent.ksql.metastore.KsqlTable in project ksql by confluentinc.

the class LogicalPlanner method buildSourceNode.

private StructuredDataSourceNode buildSourceNode() {
    Pair<StructuredDataSource, String> dataSource = analysis.getFromDataSource(0);
    Schema fromSchema = SchemaUtil.buildSchemaWithAlias(dataSource.left.getSchema(), dataSource.right);
    if (dataSource.left instanceof KsqlStream || dataSource.left instanceof KsqlTable) {
        return new StructuredDataSourceNode(new PlanNodeId("KsqlTopic"), dataSource.left, fromSchema);
    }
    throw new RuntimeException("Data source is not supported yet.");
}

Example 2 with KsqlTable

use of io.confluent.ksql.metastore.KsqlTable in project ksql by confluentinc.

the class StructuredDataSourceNode method buildStream.

@Override
public SchemaKStream buildStream(final StreamsBuilder builder, final KsqlConfig ksqlConfig, final KafkaTopicClient kafkaTopicClient, final FunctionRegistry functionRegistry, final Map<String, Object> props, final SchemaRegistryClient schemaRegistryClient) {
    if (getTimestampField() != null) {
        int timestampColumnIndex = getTimeStampColumnIndex();
        ksqlConfig.put(KsqlConfig.KSQL_TIMESTAMP_COLUMN_INDEX, timestampColumnIndex);
    }
    KsqlTopicSerDe ksqlTopicSerDe = getStructuredDataSource().getKsqlTopic().getKsqlTopicSerDe();
    Serde<GenericRow> genericRowSerde = ksqlTopicSerDe.getGenericRowSerde(SchemaUtil.removeImplicitRowTimeRowKeyFromSchema(getSchema()), ksqlConfig, false, schemaRegistryClient);
    if (getDataSourceType() == StructuredDataSource.DataSourceType.KTABLE) {
        final KsqlTable table = (KsqlTable) getStructuredDataSource();
        final KTable kTable = createKTable(builder, getAutoOffsetReset(props), table, genericRowSerde, table.getKsqlTopic().getKsqlTopicSerDe().getGenericRowSerde(getSchema(), ksqlConfig, true, schemaRegistryClient));
        return new SchemaKTable(getSchema(), kTable, getKeyField(), new ArrayList<>(), table.isWindowed(), SchemaKStream.Type.SOURCE, functionRegistry, schemaRegistryClient);
    }
    return new SchemaKStream(getSchema(), builder.stream(getStructuredDataSource().getKsqlTopic().getKafkaTopicName(), Consumed.with(Serdes.String(), genericRowSerde)).mapValues(nonWindowedValueMapper).transformValues(new AddTimestampColumn()), getKeyField(), new ArrayList<>(), SchemaKStream.Type.SOURCE, functionRegistry, schemaRegistryClient);
}

Example 3 with KsqlTable

use of io.confluent.ksql.metastore.KsqlTable in project ksql by confluentinc.

the class CreateTableCommand method run.

@Override
public DdlCommandResult run(MetaStore metaStore) {
    if (registerTopicCommand != null) {
        registerTopicCommand.run(metaStore);
    }
    checkMetaData(metaStore, sourceName, topicName);
    KsqlTable ksqlTable = new KsqlTable(sqlExpression, sourceName, schema, (keyColumnName.length() == 0) ? null : SchemaUtil.getFieldByName(schema, keyColumnName).orElse(null), (timestampColumnName.length() == 0) ? null : SchemaUtil.getFieldByName(schema, timestampColumnName).orElse(null), metaStore.getTopic(topicName), stateStoreName, isWindowed);
    // TODO: Need to check if the topic exists.
    // Add the topic to the metastore
    metaStore.putSource(ksqlTable.cloneWithTimeKeyColumns());
    return new DdlCommandResult(true, "Table created");
}

Example 4 with KsqlTable

use of io.confluent.ksql.metastore.KsqlTable in project ksql by confluentinc.

the class KsqlStructuredDataOutputNodeTest method getKsqlStructuredDataOutputNode.

private KsqlStructuredDataOutputNode getKsqlStructuredDataOutputNode(boolean isWindowed) {
    final Map<String, Object> props = new HashMap<>();
    props.put(KsqlConfig.SINK_NUMBER_OF_PARTITIONS_PROPERTY, 4);
    props.put(KsqlConfig.SINK_NUMBER_OF_REPLICAS_PROPERTY, (short) 3);
    StructuredDataSourceNode tableSourceNode = new StructuredDataSourceNode(new PlanNodeId("0"), new KsqlTable("sqlExpression", "datasource", schema, schema.field("key"), schema.field("timestamp"), new KsqlTopic("input", "input", new KsqlJsonTopicSerDe()), "TableStateStore", isWindowed), schema);
    return new KsqlStructuredDataOutputNode(new PlanNodeId("0"), tableSourceNode, schema, schema.field("timestamp"), schema.field("key"), new KsqlTopic("output", "output", new KsqlJsonTopicSerDe()), "output", props, Optional.empty());
}

Example 5 with KsqlTable

use of io.confluent.ksql.metastore.KsqlTable in project ksql by confluentinc.

the class StructuredDataSourceNodeTest method shouldBuildSchemaKTableWhenKTableSource.

@Test
public void shouldBuildSchemaKTableWhenKTableSource() {
    StructuredDataSourceNode node = new StructuredDataSourceNode(new PlanNodeId("0"), new KsqlTable("sqlExpression", "datasource", schema, schema.field("field"), schema.field("timestamp"), new KsqlTopic("topic2", "topic2", new KsqlJsonTopicSerDe()), "statestore", false), schema);
    final SchemaKStream result = build(node);
    assertThat(result.getClass(), equalTo(SchemaKTable.class));
}