Examples with DataSourceNode io.confluent.ksql.planner.plan.DataSourceNode used on opensource projects

Example 1 with DataSourceNode

use of io.confluent.ksql.planner.plan.DataSourceNode in project ksql by confluentinc.

the class PullPhysicalPlanBuilder method buildPullPhysicalPlan.

/**
 * Visits the logical plan top-down to build the physical plan.
 * @param logicalPlanNode the logical plan root node
 * @return the root node of the tree of physical operators
 */
// CHECKSTYLE_RULES.OFF: CyclomaticComplexity
public PullPhysicalPlan buildPullPhysicalPlan(final LogicalPlanNode logicalPlanNode) {
    // CHECKSTYLE_RULES.ON: CyclomaticComplexity
    DataSourceOperator dataSourceOperator = null;
    final OutputNode outputNode = logicalPlanNode.getNode().orElseThrow(() -> new IllegalArgumentException("Need an output node to build a plan"));
    if (!(outputNode instanceof KsqlBareOutputNode)) {
        throw new KsqlException("Pull queries expect the root of the logical plan to be a " + "KsqlBareOutputNode.");
    }
    // We skip KsqlBareOutputNode in the translation since it only applies the LIMIT
    PlanNode currentLogicalNode = outputNode.getSource();
    AbstractPhysicalOperator prevPhysicalOp = null;
    AbstractPhysicalOperator rootPhysicalOp = null;
    while (true) {
        AbstractPhysicalOperator currentPhysicalOp = null;
        if (currentLogicalNode instanceof QueryProjectNode) {
            currentPhysicalOp = translateProjectNode((QueryProjectNode) currentLogicalNode);
        } else if (currentLogicalNode instanceof QueryFilterNode) {
            currentPhysicalOp = translateFilterNode((QueryFilterNode) currentLogicalNode);
            seenSelectOperator = true;
        } else if (currentLogicalNode instanceof QueryLimitNode) {
            currentPhysicalOp = new LimitOperator((QueryLimitNode) currentLogicalNode);
        } else if (currentLogicalNode instanceof DataSourceNode) {
            currentPhysicalOp = translateDataSourceNode((DataSourceNode) currentLogicalNode);
            dataSourceOperator = (DataSourceOperator) currentPhysicalOp;
        } else {
            throw new KsqlException(String.format("Error in translating logical to physical plan for pull queries: unrecognized logical" + " node %s.", currentLogicalNode));
        }
        if (prevPhysicalOp == null) {
            rootPhysicalOp = currentPhysicalOp;
        } else {
            prevPhysicalOp.addChild(currentPhysicalOp);
        }
        prevPhysicalOp = currentPhysicalOp;
        // Exit the loop when a leaf node is reached
        if (currentLogicalNode.getSources().isEmpty()) {
            break;
        }
        if (currentLogicalNode.getSources().size() > 1) {
            throw new KsqlException("Pull queries do not support joins or nested sub-queries yet.");
        }
        currentLogicalNode = currentLogicalNode.getSources().get(0);
    }
    if (dataSourceOperator == null) {
        throw new IllegalStateException("DataSourceOperator cannot be null in Pull physical plan");
    }
    return new PullPhysicalPlan(rootPhysicalOp, (rootPhysicalOp).getLogicalNode().getSchema(), queryId, lookupConstraints, pullPhysicalPlanType, querySourceType, mat, dataSourceOperator);
}

Example 2 with DataSourceNode

use of io.confluent.ksql.planner.plan.DataSourceNode in project ksql by confluentinc.

the class LogicalPlanner method verifyForeignKeyJoin.

private Optional<Expression> verifyForeignKeyJoin(final JoinInfo joinInfo, final PlanNode leftNode, final PlanNode rightNode) {
    final JoinType joinType = joinInfo.getType();
    final Expression leftExpression = joinInfo.getLeftJoinExpression();
    final Expression rightExpression = joinInfo.getRightJoinExpression();
    if (joinInfo.getType().equals(JoinType.OUTER)) {
        throw new KsqlException(String.format("Invalid join type:" + " full-outer join not supported for foreign-key table-table join." + " Got %s %s %s.", joinInfo.getLeftSource().getDataSource().getName().text(), joinType, joinInfo.getRightSource().getDataSource().getName().text()));
    }
    // because a FK-join output table has the same PK as its left input table
    if (!(leftNode instanceof DataSourceNode) || !(rightNode instanceof DataSourceNode)) {
        throw new KsqlException(String.format("Invalid join condition:" + " foreign-key table-table joins are not supported as part of n-way joins." + " Got %s = %s.", joinInfo.getFlippedLeftJoinExpression(), joinInfo.getFlippedRightJoinExpression()));
    }
    final CodeGenRunner codeGenRunner = new CodeGenRunner(leftNode.getSchema(), ksqlConfig, metaStore);
    final VisitParentExpressionVisitor<Optional<Expression>, Context<Void>> unqualifiedRewritter = new VisitParentExpressionVisitor<Optional<Expression>, Context<Void>>(Optional.empty()) {

        @Override
        public Optional<Expression> visitQualifiedColumnReference(final QualifiedColumnReferenceExp node, final Context<Void> ctx) {
            return Optional.of(new UnqualifiedColumnReferenceExp(node.getColumnName()));
        }
    };
    final Expression leftExpressionUnqualified = ExpressionTreeRewriter.rewriteWith(unqualifiedRewritter::process, leftExpression);
    final ExpressionEvaluator expressionEvaluator = codeGenRunner.buildCodeGenFromParseTree(leftExpressionUnqualified, "Left Join Expression");
    final SqlType fkType = expressionEvaluator.getExpressionType();
    final SqlType rightKeyType = Iterables.getOnlyElement(rightNode.getSchema().key()).type();
    verifyJoinConditionTypes(fkType, rightKeyType, leftExpression, rightExpression, joinInfo.hasFlippedJoinCondition());
    if (((DataSourceNode) rightNode).isWindowed()) {
        throw new KsqlException("Foreign-key table-table joins are not supported on windowed tables.");
    }
    return Optional.of(leftExpression);
}

Example 3 with DataSourceNode

use of io.confluent.ksql.planner.plan.DataSourceNode in project ksql by confluentinc.

the class LogicalPlanner method buildJoin.

/**
 * @param root    the root of the Join Tree
 * @param prefix  the prefix to uniquely identify the plan node
 * @return the PlanNode representing this Join Tree
 */
private JoinNode buildJoin(final Join root, final String prefix, final boolean isWindowed) {
    final PlanNode preRepartitionLeft;
    if (root.getLeft() instanceof JoinTree.Join) {
        preRepartitionLeft = buildJoin((Join) root.getLeft(), prefix + "L_", isWindowed);
    } else {
        final JoinTree.Leaf leaf = (Leaf) root.getLeft();
        preRepartitionLeft = new DataSourceNode(new PlanNodeId("KafkaTopic_" + prefix + "Left"), leaf.getSource().getDataSource(), leaf.getSource().getAlias(), isWindowed, ksqlConfig);
    }
    final PlanNode preRepartitionRight;
    if (root.getRight() instanceof JoinTree.Join) {
        preRepartitionRight = buildJoin((Join) root.getRight(), prefix + "R_", isWindowed);
    } else {
        final JoinTree.Leaf leaf = (Leaf) root.getRight();
        preRepartitionRight = new DataSourceNode(new PlanNodeId("KafkaTopic_" + prefix + "Right"), leaf.getSource().getDataSource(), leaf.getSource().getAlias(), isWindowed, ksqlConfig);
    }
    final Optional<Expression> fkExpression = verifyJoin(root.getInfo(), preRepartitionLeft, preRepartitionRight);
    final JoinKey joinKey = fkExpression.map(columnReferenceExp -> buildForeignJoinKey(root, fkExpression.get())).orElseGet(() -> buildJoinKey(root));
    final PlanNode left = prepareSourceForJoin(root.getLeft(), preRepartitionLeft, prefix + "Left", root.getInfo().getLeftJoinExpression(), fkExpression.isPresent());
    final PlanNode right = prepareSourceForJoin(root.getRight(), preRepartitionRight, prefix + "Right", root.getInfo().getRightJoinExpression(), fkExpression.isPresent());
    return new JoinNode(new PlanNodeId(prefix + "Join"), root.getInfo().getType(), joinKey.rewriteWith(refRewriter::process), prefix.isEmpty(), left, right, root.getInfo().getWithinExpression(), ksqlConfig.getString(KsqlConfig.KSQL_DEFAULT_KEY_FORMAT_CONFIG));
}

Example 4 with DataSourceNode

use of io.confluent.ksql.planner.plan.DataSourceNode in project ksql by confluentinc.

the class LogicalPlannerTest method shouldCreatePlanWithTableAsSource.

@Test
public void shouldCreatePlanWithTableAsSource() {
    final PlanNode planNode = buildLogicalPlan("select col0 from TEST2 EMIT CHANGES limit 5;");
    assertThat(planNode.getSources().size(), equalTo(1));
    final DataSource dataSource = ((DataSourceNode) planNode.getSources().get(0).getSources().get(0).getSources().get(0)).getDataSource();
    assertThat(dataSource.getDataSourceType(), equalTo(DataSourceType.KTABLE));
    assertThat(dataSource.getName(), equalTo(SourceName.of("TEST2")));
}

Example 5 with DataSourceNode

use of io.confluent.ksql.planner.plan.DataSourceNode in project ksql by confluentinc.

the class LogicalPlannerTest method testLimitStreamPullQueryLogicalPlan.

@Test
public void testLimitStreamPullQueryLogicalPlan() {
    final String simpleQuery = "SELECT * FROM test1 LIMIT 3;";
    final PlanNode logicalPlan = buildLogicalPlan(simpleQuery);
    assertThat(logicalPlan, instanceOf(KsqlBareOutputNode.class));
    assertThat(logicalPlan.getNodeOutputType(), equalTo(DataSourceType.KSTREAM));
    final PlanNode finalProjectNode = logicalPlan.getSources().get(0);
    assertThat(finalProjectNode, instanceOf(FinalProjectNode.class));
    assertThat(finalProjectNode.getNodeOutputType(), equalTo(DataSourceType.KSTREAM));
    final PlanNode queryLimitNode = finalProjectNode.getSources().get(0);
    assertThat(queryLimitNode, instanceOf(QueryLimitNode.class));
    assertThat(((QueryLimitNode) queryLimitNode).getLimit(), equalTo(3));
    assertThat(queryLimitNode.getNodeOutputType(), equalTo(DataSourceType.KSTREAM));
    final PlanNode dataSourceNode = queryLimitNode.getSources().get(0);
    assertThat(dataSourceNode, instanceOf(DataSourceNode.class));
    assertThat(dataSourceNode.getNodeOutputType(), equalTo(DataSourceType.KSTREAM));
}