Examples with StructuredDataSource io.confluent.ksql.metastore.StructuredDataSource used on opensource projects

Example 6 with StructuredDataSource

use of io.confluent.ksql.metastore.StructuredDataSource in project ksql by confluentinc.

the class LogicalPlanner method buildSourceNode.

private StructuredDataSourceNode buildSourceNode() {
    Pair<StructuredDataSource, String> dataSource = analysis.getFromDataSource(0);
    Schema fromSchema = SchemaUtil.buildSchemaWithAlias(dataSource.left.getSchema(), dataSource.right);
    if (dataSource.left instanceof KsqlStream || dataSource.left instanceof KsqlTable) {
        return new StructuredDataSourceNode(new PlanNodeId("KsqlTopic"), dataSource.left, fromSchema);
    }
    throw new RuntimeException("Data source is not supported yet.");
}

Example 7 with StructuredDataSource

use of io.confluent.ksql.metastore.StructuredDataSource in project ksql by confluentinc.

the class LogicalPlannerTest method shouldCreatePlanWithTableAsSource.

@Test
public void shouldCreatePlanWithTableAsSource() {
    PlanNode planNode = buildLogicalPlan("select col0 from TEST2 limit 5;");
    assertThat(planNode.getSources().size(), equalTo(1));
    StructuredDataSource structuredDataSource = ((StructuredDataSourceNode) planNode.getSources().get(0).getSources().get(0)).getStructuredDataSource();
    assertThat(structuredDataSource.getDataSourceType(), equalTo(DataSource.DataSourceType.KTABLE));
    assertThat(structuredDataSource.getName(), equalTo("TEST2"));
}

Example 8 with StructuredDataSource

use of io.confluent.ksql.metastore.StructuredDataSource in project ksql by confluentinc.

the class JoinNodeTest method shouldHaveAllFieldsFromJoinedInputs.

@Test
public void shouldHaveAllFieldsFromJoinedInputs() {
    setupTopicClientExpectations(1, 1);
    buildJoin();
    final MetaStore metaStore = MetaStoreFixture.getNewMetaStore();
    final StructuredDataSource source1 = metaStore.getSource("TEST1");
    final StructuredDataSource source2 = metaStore.getSource("TEST2");
    final Set<String> expected = source1.getSchema().fields().stream().map(field -> "T1." + field.name()).collect(Collectors.toSet());
    expected.addAll(source2.getSchema().fields().stream().map(field -> "T2." + field.name()).collect(Collectors.toSet()));
    final Set<String> fields = stream.getSchema().fields().stream().map(Field::name).collect(Collectors.toSet());
    assertThat(fields, equalTo(expected));
}

Example 9 with StructuredDataSource

use of io.confluent.ksql.metastore.StructuredDataSource in project ksql by confluentinc.

the class DataSourceExtractor method visitJoinRelation.

@Override
public Node visitJoinRelation(final SqlBaseParser.JoinRelationContext context) {
    this.isJoin = true;
    AliasedRelation left = (AliasedRelation) visit(context.left);
    AliasedRelation right;
    if (context.CROSS() != null) {
        right = (AliasedRelation) visit(context.right);
    } else {
        if (context.NATURAL() != null) {
            right = (AliasedRelation) visit(context.right);
        } else {
            right = (AliasedRelation) visit(context.rightRelation);
        }
    }
    this.leftAlias = left.getAlias();
    StructuredDataSource leftDataSource = metaStore.getSource(((Table) left.getRelation()).getName().getSuffix());
    if (leftDataSource == null) {
        throw new KsqlException(((Table) left.getRelation()).getName().getSuffix() + " does not " + "exist.");
    }
    this.joinLeftSchema = leftDataSource.getSchema();
    this.rightAlias = right.getAlias();
    StructuredDataSource rightDataSource = metaStore.getSource(((Table) right.getRelation()).getName().getSuffix());
    if (rightDataSource == null) {
        throw new KsqlException(((Table) right.getRelation()).getName().getSuffix() + " does not " + "exist.");
    }
    this.joinRightSchema = rightDataSource.getSchema();
    return null;
}

Example 10 with StructuredDataSource

use of io.confluent.ksql.metastore.StructuredDataSource in project ksql by confluentinc.

the class QueryEngine method buildQueryLogicalPlan.

private PlanNode buildQueryLogicalPlan(String sqlExpression, final Query query, final MetaStore tempMetaStore) {
    final QueryAnalyzer queryAnalyzer = new QueryAnalyzer(tempMetaStore, ksqlEngine.getFunctionRegistry());
    final Analysis analysis = queryAnalyzer.analyze(sqlExpression, query);
    final AggregateAnalysis aggAnalysis = queryAnalyzer.analyzeAggregate(query, analysis);
    final PlanNode logicalPlan = new LogicalPlanner(analysis, aggAnalysis, ksqlEngine.getFunctionRegistry()).buildPlan();
    if (logicalPlan instanceof KsqlStructuredDataOutputNode) {
        KsqlStructuredDataOutputNode ksqlStructuredDataOutputNode = (KsqlStructuredDataOutputNode) logicalPlan;
        StructuredDataSource structuredDataSource = new KsqlStream(sqlExpression, ksqlStructuredDataOutputNode.getId().toString(), ksqlStructuredDataOutputNode.getSchema(), ksqlStructuredDataOutputNode.getKeyField(), ksqlStructuredDataOutputNode.getTimestampField() == null ? ksqlStructuredDataOutputNode.getTheSourceNode().getTimestampField() : ksqlStructuredDataOutputNode.getTimestampField(), ksqlStructuredDataOutputNode.getKsqlTopic());
        tempMetaStore.putTopic(ksqlStructuredDataOutputNode.getKsqlTopic());
        tempMetaStore.putSource(structuredDataSource.cloneWithTimeKeyColumns());
    }
    return logicalPlan;
}