Examples with KsqlStream io.confluent.ksql.metastore.KsqlStream used on opensource projects

Example 1 with KsqlStream

use of io.confluent.ksql.metastore.KsqlStream in project ksql by confluentinc.

the class CodeGenRunnerTest method init.

@Before
public void init() {
    metaStore = MetaStoreFixture.getNewMetaStore();
    functionRegistry = new FunctionRegistry();
    schema = SchemaBuilder.struct().field("CODEGEN_TEST.COL0", SchemaBuilder.INT64_SCHEMA).field("CODEGEN_TEST.COL1", SchemaBuilder.STRING_SCHEMA).field("CODEGEN_TEST.COL2", SchemaBuilder.STRING_SCHEMA).field("CODEGEN_TEST.COL3", SchemaBuilder.FLOAT64_SCHEMA).field("CODEGEN_TEST.COL4", SchemaBuilder.FLOAT64_SCHEMA).field("CODEGEN_TEST.COL5", SchemaBuilder.INT32_SCHEMA).field("CODEGEN_TEST.COL6", SchemaBuilder.BOOLEAN_SCHEMA).field("CODEGEN_TEST.COL7", SchemaBuilder.BOOLEAN_SCHEMA).field("CODEGEN_TEST.COL8", SchemaBuilder.INT64_SCHEMA).field("CODEGEN_TEST.COL9", SchemaBuilder.array(SchemaBuilder.INT32_SCHEMA)).field("CODEGEN_TEST.COL10", SchemaBuilder.array(SchemaBuilder.INT32_SCHEMA)).field("CODEGEN_TEST.COL11", SchemaBuilder.map(SchemaBuilder.INT32_SCHEMA, SchemaBuilder.INT32_SCHEMA)).field("CODEGEN_TEST.COL12", SchemaBuilder.map(SchemaBuilder.INT32_SCHEMA, SchemaBuilder.INT32_SCHEMA));
    Schema metaStoreSchema = SchemaBuilder.struct().field("COL0", SchemaBuilder.INT64_SCHEMA).field("COL1", SchemaBuilder.STRING_SCHEMA).field("COL2", SchemaBuilder.STRING_SCHEMA).field("COL3", SchemaBuilder.FLOAT64_SCHEMA).field("COL4", SchemaBuilder.FLOAT64_SCHEMA).field("COL5", SchemaBuilder.INT32_SCHEMA).field("COL6", SchemaBuilder.BOOLEAN_SCHEMA).field("COL7", SchemaBuilder.BOOLEAN_SCHEMA).field("COL8", SchemaBuilder.INT64_SCHEMA).field("COL9", SchemaBuilder.array(SchemaBuilder.INT32_SCHEMA)).field("COL10", SchemaBuilder.array(SchemaBuilder.INT32_SCHEMA)).field("COL11", SchemaBuilder.map(SchemaBuilder.INT32_SCHEMA, SchemaBuilder.INT32_SCHEMA)).field("COL12", SchemaBuilder.map(SchemaBuilder.INT32_SCHEMA, SchemaBuilder.INT32_SCHEMA));
    KsqlTopic ksqlTopic = new KsqlTopic("CODEGEN_TEST", "codegen_test", new KsqlJsonTopicSerDe());
    KsqlStream ksqlStream = new KsqlStream("sqlexpression", "CODEGEN_TEST", metaStoreSchema, metaStoreSchema.field("COL0"), null, ksqlTopic);
    metaStore.putTopic(ksqlTopic);
    metaStore.putSource(ksqlStream);
    codeGenRunner = new CodeGenRunner(schema, functionRegistry);
}

Example 2 with KsqlStream

use of io.confluent.ksql.metastore.KsqlStream in project ksql by confluentinc.

the class AstBuilder method getResultDatasource.

private StructuredDataSource getResultDatasource(Select select, Table into) {
    SchemaBuilder dataSource = SchemaBuilder.struct().name(into.toString());
    for (SelectItem selectItem : select.getSelectItems()) {
        if (selectItem instanceof SingleColumn) {
            SingleColumn singleColumn = (SingleColumn) selectItem;
            String fieldName = singleColumn.getAlias().get();
            dataSource = dataSource.field(fieldName, Schema.BOOLEAN_SCHEMA);
        }
    }
    KsqlTopic ksqlTopic = new KsqlTopic(into.getName().toString(), into.getName().toString(), null);
    StructuredDataSource resultStream = new KsqlStream("AstBuilder-Into", into.getName().toString(), dataSource.schema(), dataSource.fields().get(0), null, ksqlTopic);
    return resultStream;
}

Example 3 with KsqlStream

use of io.confluent.ksql.metastore.KsqlStream in project ksql by confluentinc.

the class LogicalPlanner method buildSourceNode.

private StructuredDataSourceNode buildSourceNode() {
    Pair<StructuredDataSource, String> dataSource = analysis.getFromDataSource(0);
    Schema fromSchema = SchemaUtil.buildSchemaWithAlias(dataSource.left.getSchema(), dataSource.right);
    if (dataSource.left instanceof KsqlStream || dataSource.left instanceof KsqlTable) {
        return new StructuredDataSourceNode(new PlanNodeId("KsqlTopic"), dataSource.left, fromSchema);
    }
    throw new RuntimeException("Data source is not supported yet.");
}

Example 4 with KsqlStream

use of io.confluent.ksql.metastore.KsqlStream in project ksql by confluentinc.

the class MetaStoreFixture method getNewMetaStore.

public static MetaStore getNewMetaStore() {
    MetaStore metaStore = new MetaStoreImpl();
    SchemaBuilder schemaBuilder1 = SchemaBuilder.struct().field("COL0", SchemaBuilder.INT64_SCHEMA).field("COL1", SchemaBuilder.STRING_SCHEMA).field("COL2", SchemaBuilder.STRING_SCHEMA).field("COL3", SchemaBuilder.FLOAT64_SCHEMA).field("COL4", SchemaBuilder.array(SchemaBuilder.FLOAT64_SCHEMA)).field("COL5", SchemaBuilder.map(SchemaBuilder.STRING_SCHEMA, SchemaBuilder.FLOAT64_SCHEMA));
    KsqlTopic ksqlTopic1 = new KsqlTopic("TEST1", "test1", new KsqlJsonTopicSerDe());
    KsqlStream ksqlStream = new KsqlStream("sqlexpression", "TEST1", schemaBuilder1, schemaBuilder1.field("COL0"), null, ksqlTopic1);
    metaStore.putTopic(ksqlTopic1);
    metaStore.putSource(ksqlStream);
    SchemaBuilder schemaBuilder2 = SchemaBuilder.struct().field("COL0", SchemaBuilder.INT64_SCHEMA).field("COL1", SchemaBuilder.STRING_SCHEMA).field("COL2", SchemaBuilder.STRING_SCHEMA).field("COL3", SchemaBuilder.FLOAT64_SCHEMA).field("COL4", SchemaBuilder.BOOLEAN_SCHEMA);
    KsqlTopic ksqlTopic2 = new KsqlTopic("TEST2", "test2", new KsqlJsonTopicSerDe());
    KsqlTable ksqlTable = new KsqlTable("sqlexpression", "TEST2", schemaBuilder2, schemaBuilder2.field("COL0"), null, ksqlTopic2, "TEST2", false);
    metaStore.putTopic(ksqlTopic2);
    metaStore.putSource(ksqlTable);
    SchemaBuilder schemaBuilderOrders = SchemaBuilder.struct().field("ORDERTIME", SchemaBuilder.INT64_SCHEMA).field("ORDERID", SchemaBuilder.STRING_SCHEMA).field("ITEMID", SchemaBuilder.STRING_SCHEMA).field("ORDERUNITS", SchemaBuilder.FLOAT64_SCHEMA);
    KsqlTopic ksqlTopicOrders = new KsqlTopic("ORDERS_TOPIC", "orders_topic", new KsqlJsonTopicSerDe());
    KsqlStream ksqlStreamOrders = new KsqlStream("sqlexpression", "ORDERS", schemaBuilderOrders, schemaBuilderOrders.field("ORDERTIME"), null, ksqlTopicOrders);
    metaStore.putTopic(ksqlTopicOrders);
    metaStore.putSource(ksqlStreamOrders);
    return metaStore;
}

Example 5 with KsqlStream

use of io.confluent.ksql.metastore.KsqlStream in project ksql by confluentinc.

the class QueryEngine method buildQueryLogicalPlan.

private PlanNode buildQueryLogicalPlan(String sqlExpression, final Query query, final MetaStore tempMetaStore) {
    final QueryAnalyzer queryAnalyzer = new QueryAnalyzer(tempMetaStore, ksqlEngine.getFunctionRegistry());
    final Analysis analysis = queryAnalyzer.analyze(sqlExpression, query);
    final AggregateAnalysis aggAnalysis = queryAnalyzer.analyzeAggregate(query, analysis);
    final PlanNode logicalPlan = new LogicalPlanner(analysis, aggAnalysis, ksqlEngine.getFunctionRegistry()).buildPlan();
    if (logicalPlan instanceof KsqlStructuredDataOutputNode) {
        KsqlStructuredDataOutputNode ksqlStructuredDataOutputNode = (KsqlStructuredDataOutputNode) logicalPlan;
        StructuredDataSource structuredDataSource = new KsqlStream(sqlExpression, ksqlStructuredDataOutputNode.getId().toString(), ksqlStructuredDataOutputNode.getSchema(), ksqlStructuredDataOutputNode.getKeyField(), ksqlStructuredDataOutputNode.getTimestampField() == null ? ksqlStructuredDataOutputNode.getTheSourceNode().getTimestampField() : ksqlStructuredDataOutputNode.getTimestampField(), ksqlStructuredDataOutputNode.getKsqlTopic());
        tempMetaStore.putTopic(ksqlStructuredDataOutputNode.getKsqlTopic());
        tempMetaStore.putSource(structuredDataSource.cloneWithTimeKeyColumns());
    }
    return logicalPlan;
}