Examples with DataSourceNode io.confluent.ksql.planner.plan.DataSourceNode used on opensource projects

Example 6 with DataSourceNode

use of io.confluent.ksql.planner.plan.DataSourceNode in project ksql by confluentinc.

the class LogicalPlannerTest method testLimitTableScanLogicalPlan.

@Test
public void testLimitTableScanLogicalPlan() {
    final String simpleQuery = "SELECT * FROM test2 LIMIT 3;";
    final PlanNode logicalPlan = buildLogicalPlan(simpleQuery);
    assertThat(logicalPlan, instanceOf(KsqlBareOutputNode.class));
    assertThat(logicalPlan.getNodeOutputType(), equalTo(DataSourceType.KTABLE));
    final PlanNode finalProjectNode = logicalPlan.getSources().get(0);
    assertThat(finalProjectNode, instanceOf(FinalProjectNode.class));
    assertThat(finalProjectNode.getNodeOutputType(), equalTo(DataSourceType.KTABLE));
    final PlanNode queryLimitNode = finalProjectNode.getSources().get(0);
    assertThat(queryLimitNode, instanceOf(QueryLimitNode.class));
    assertThat(((QueryLimitNode) queryLimitNode).getLimit(), equalTo(3));
    assertThat(queryLimitNode.getNodeOutputType(), equalTo(DataSourceType.KTABLE));
    final PlanNode dataSourceNode = queryLimitNode.getSources().get(0);
    assertThat(dataSourceNode, instanceOf(DataSourceNode.class));
    assertThat(dataSourceNode.getNodeOutputType(), equalTo(DataSourceType.KTABLE));
}

Example 7 with DataSourceNode

use of io.confluent.ksql.planner.plan.DataSourceNode in project ksql by confluentinc.

the class LogicalPlanner method joinOnNonKeyAttribute.

private static boolean joinOnNonKeyAttribute(final Expression joinExpression, final PlanNode node, final AliasedDataSource aliasedDataSource) {
    if (!(joinExpression instanceof ColumnReferenceExp)) {
        return true;
    }
    final ColumnReferenceExp simpleJoinExpression = (ColumnReferenceExp) joinExpression;
    final ColumnName joinAttributeName = simpleJoinExpression.getColumnName();
    final List<DataSourceNode> dataSourceNodes = node.getSourceNodes().collect(Collectors.toList());
    final List<Column> keyColumns;
    // n-way join sub-tree (ie, not a leaf)
    if (isInnerNode(node)) {
        final DataSourceNode qualifiedNode;
        if (simpleJoinExpression.maybeQualifier().isPresent()) {
            final SourceName qualifierOrAlias = simpleJoinExpression.maybeQualifier().get();
            final SourceName qualifier;
            if (aliasedDataSource.getAlias().equals(qualifierOrAlias)) {
                qualifier = aliasedDataSource.getDataSource().getName();
            } else {
                qualifier = qualifierOrAlias;
            }
            final List<DataSourceNode> allNodes = dataSourceNodes.stream().filter(n -> n.getDataSource().getName().equals(qualifier)).collect(Collectors.toList());
            if (allNodes.size() != 1) {
                throw new KsqlException(String.format("Join qualifier '%s' could not be resolved (either not found or not unique).", qualifier));
            }
            qualifiedNode = Iterables.getOnlyElement(allNodes);
        } else {
            final List<DataSourceNode> allNodes = dataSourceNodes.stream().filter(n -> n.getSchema().findColumn(simpleJoinExpression.getColumnName()).isPresent()).collect(Collectors.toList());
            if (allNodes.size() != 1) {
                throw new KsqlException(String.format("Join identifier '%s' could not be resolved (either not found or not unique).", joinAttributeName));
            }
            qualifiedNode = Iterables.getOnlyElement(allNodes);
        }
        keyColumns = qualifiedNode.getSchema().key();
    } else {
        // leaf node: we know we have single data source
        keyColumns = Iterables.getOnlyElement(dataSourceNodes).getSchema().key();
    }
    // - thus, if the key has more than one column, the join is not on the key
    if (keyColumns.size() > 1) {
        return true;
    }
    return !joinAttributeName.equals(Iterables.getOnlyElement(keyColumns).name());
}

Example 8 with DataSourceNode

use of io.confluent.ksql.planner.plan.DataSourceNode in project ksql by confluentinc.

the class PushPhysicalPlanBuilder method buildPushPhysicalPlan.

/**
 * Visits the logical plan top-down to build the physical plan.
 * @param logicalPlanNode the logical plan root node
 * @return the root node of the tree of physical operators
 */
public PushPhysicalPlan buildPushPhysicalPlan(final LogicalPlanNode logicalPlanNode, final Context context, final Optional<PushOffsetRange> offsetRange, final Optional<String> catchupConsumerGroup) {
    final String catchupConsumerGroupId = getConsumerGroupId(catchupConsumerGroup);
    PushDataSourceOperator dataSourceOperator = null;
    final OutputNode outputNode = logicalPlanNode.getNode().orElseThrow(() -> new IllegalArgumentException("Need an output node to build a plan"));
    if (!(outputNode instanceof KsqlBareOutputNode)) {
        throw new KsqlException("Push queries expect the root of the logical plan to be a " + "KsqlBareOutputNode.");
    }
    // We skip KsqlBareOutputNode in the translation since it only applies the LIMIT
    PlanNode currentLogicalNode = outputNode.getSource();
    AbstractPhysicalOperator prevPhysicalOp = null;
    AbstractPhysicalOperator rootPhysicalOp = null;
    while (true) {
        AbstractPhysicalOperator currentPhysicalOp = null;
        if (currentLogicalNode instanceof QueryProjectNode) {
            currentPhysicalOp = translateProjectNode((QueryProjectNode) currentLogicalNode);
        } else if (currentLogicalNode instanceof QueryFilterNode) {
            currentPhysicalOp = translateFilterNode((QueryFilterNode) currentLogicalNode);
        } else if (currentLogicalNode instanceof DataSourceNode) {
            currentPhysicalOp = translateDataSourceNode((DataSourceNode) currentLogicalNode, offsetRange, catchupConsumerGroupId);
            dataSourceOperator = (PushDataSourceOperator) currentPhysicalOp;
        } else {
            throw new KsqlException(String.format("Error in translating logical to physical plan for scalable push queries:" + " unrecognized logical node %s.", currentLogicalNode));
        }
        if (prevPhysicalOp == null) {
            rootPhysicalOp = currentPhysicalOp;
        } else {
            prevPhysicalOp.addChild(currentPhysicalOp);
        }
        prevPhysicalOp = currentPhysicalOp;
        // Exit the loop when a leaf node is reached
        if (currentLogicalNode.getSources().isEmpty()) {
            break;
        }
        if (currentLogicalNode.getSources().size() > 1) {
            throw new KsqlException("Push queries do not support joins or nested sub-queries yet.");
        }
        currentLogicalNode = currentLogicalNode.getSources().get(0);
    }
    if (dataSourceOperator == null) {
        throw new IllegalStateException("DataSourceOperator cannot be null in Push physical plan");
    }
    return new PushPhysicalPlan(rootPhysicalOp, (rootPhysicalOp).getLogicalNode().getSchema(), queryId, catchupConsumerGroupId, dataSourceOperator.getScalablePushRegistry(), dataSourceOperator, context, querySourceType);
}