Search in sources :

Example 1 with JoinKey

use of io.confluent.ksql.planner.plan.JoinNode.JoinKey in project ksql by confluentinc.

the class LogicalPlanner method buildJoin.

/**
 * @param root    the root of the Join Tree
 * @param prefix  the prefix to uniquely identify the plan node
 * @return the PlanNode representing this Join Tree
 */
private JoinNode buildJoin(final Join root, final String prefix, final boolean isWindowed) {
    final PlanNode preRepartitionLeft;
    if (root.getLeft() instanceof JoinTree.Join) {
        preRepartitionLeft = buildJoin((Join) root.getLeft(), prefix + "L_", isWindowed);
    } else {
        final JoinTree.Leaf leaf = (Leaf) root.getLeft();
        preRepartitionLeft = new DataSourceNode(new PlanNodeId("KafkaTopic_" + prefix + "Left"), leaf.getSource().getDataSource(), leaf.getSource().getAlias(), isWindowed, ksqlConfig);
    }
    final PlanNode preRepartitionRight;
    if (root.getRight() instanceof JoinTree.Join) {
        preRepartitionRight = buildJoin((Join) root.getRight(), prefix + "R_", isWindowed);
    } else {
        final JoinTree.Leaf leaf = (Leaf) root.getRight();
        preRepartitionRight = new DataSourceNode(new PlanNodeId("KafkaTopic_" + prefix + "Right"), leaf.getSource().getDataSource(), leaf.getSource().getAlias(), isWindowed, ksqlConfig);
    }
    final Optional<Expression> fkExpression = verifyJoin(root.getInfo(), preRepartitionLeft, preRepartitionRight);
    final JoinKey joinKey = fkExpression.map(columnReferenceExp -> buildForeignJoinKey(root, fkExpression.get())).orElseGet(() -> buildJoinKey(root));
    final PlanNode left = prepareSourceForJoin(root.getLeft(), preRepartitionLeft, prefix + "Left", root.getInfo().getLeftJoinExpression(), fkExpression.isPresent());
    final PlanNode right = prepareSourceForJoin(root.getRight(), preRepartitionRight, prefix + "Right", root.getInfo().getRightJoinExpression(), fkExpression.isPresent());
    return new JoinNode(new PlanNodeId(prefix + "Join"), root.getInfo().getType(), joinKey.rewriteWith(refRewriter::process), prefix.isEmpty(), left, right, root.getInfo().getWithinExpression(), ksqlConfig.getString(KsqlConfig.KSQL_DEFAULT_KEY_FORMAT_CONFIG));
}
Also used : PlanNodeId(io.confluent.ksql.planner.plan.PlanNodeId) JoinInfo(io.confluent.ksql.analyzer.Analysis.JoinInfo) DataSource(io.confluent.ksql.metastore.model.DataSource) Leaf(io.confluent.ksql.planner.JoinTree.Leaf) AggregateAnalysisResult(io.confluent.ksql.analyzer.AggregateAnalysisResult) Into(io.confluent.ksql.analyzer.Analysis.Into) ColumnName(io.confluent.ksql.name.ColumnName) SourceName(io.confluent.ksql.name.SourceName) BiFunction(java.util.function.BiFunction) AggregateAnalyzer(io.confluent.ksql.analyzer.AggregateAnalyzer) FilterNode(io.confluent.ksql.planner.plan.FilterNode) SerdeFeaturesFactory(io.confluent.ksql.serde.SerdeFeaturesFactory) JoinKey(io.confluent.ksql.planner.plan.JoinNode.JoinKey) CodeGenRunner(io.confluent.ksql.execution.codegen.CodeGenRunner) WindowInfo(io.confluent.ksql.serde.WindowInfo) RewrittenAnalysis(io.confluent.ksql.analyzer.RewrittenAnalysis) QueryLimitNode(io.confluent.ksql.planner.plan.QueryLimitNode) AggregateNode(io.confluent.ksql.planner.plan.AggregateNode) AliasedDataSource(io.confluent.ksql.analyzer.Analysis.AliasedDataSource) TimestampExtractionPolicyFactory(io.confluent.ksql.execution.streams.timestamp.TimestampExtractionPolicyFactory) ExpressionTypeManager(io.confluent.ksql.execution.util.ExpressionTypeManager) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) SelectionUtil(io.confluent.ksql.planner.plan.SelectionUtil) PartitionBy(io.confluent.ksql.parser.tree.PartitionBy) ColumnReferenceExp(io.confluent.ksql.execution.expression.tree.ColumnReferenceExp) PreJoinProjectNode(io.confluent.ksql.planner.plan.PreJoinProjectNode) VisitParentExpressionVisitor(io.confluent.ksql.execution.expression.tree.VisitParentExpressionVisitor) FinalProjectNode(io.confluent.ksql.planner.plan.FinalProjectNode) ColumnNames(io.confluent.ksql.schema.ksql.ColumnNames) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) ExpressionEvaluator(io.confluent.ksql.execution.transform.ExpressionEvaluator) Expression(io.confluent.ksql.execution.expression.tree.Expression) JoinType(io.confluent.ksql.planner.plan.JoinNode.JoinType) Set(java.util.Set) QueryFilterNode(io.confluent.ksql.planner.plan.QueryFilterNode) KsqlConfig(io.confluent.ksql.util.KsqlConfig) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) TimestampColumn(io.confluent.ksql.execution.timestamp.TimestampColumn) PlanNodeId(io.confluent.ksql.planner.plan.PlanNodeId) SingleSourcePlanNode(io.confluent.ksql.planner.plan.SingleSourcePlanNode) Builder(io.confluent.ksql.schema.ksql.LogicalSchema.Builder) Objects(java.util.Objects) Join(io.confluent.ksql.planner.JoinTree.Join) List(java.util.List) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) QueryProjectNode(io.confluent.ksql.planner.plan.QueryProjectNode) Column(io.confluent.ksql.schema.ksql.Column) FormatInfo(io.confluent.ksql.serde.FormatInfo) ProjectNode(io.confluent.ksql.planner.plan.ProjectNode) Iterables(com.google.common.collect.Iterables) FormatFactory(io.confluent.ksql.serde.FormatFactory) GrammaticalJoiner(io.confluent.ksql.util.GrammaticalJoiner) KeyFormat(io.confluent.ksql.serde.KeyFormat) JoinNode(io.confluent.ksql.planner.plan.JoinNode) SuppressNode(io.confluent.ksql.planner.plan.SuppressNode) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) PartitionByParamsFactory(io.confluent.ksql.execution.streams.PartitionByParamsFactory) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) Function(java.util.function.Function) NoneFormat(io.confluent.ksql.serde.none.NoneFormat) AsValue(io.confluent.ksql.function.udf.AsValue) QualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.QualifiedColumnReferenceExp) MetaStore(io.confluent.ksql.metastore.MetaStore) GroupBy(io.confluent.ksql.parser.tree.GroupBy) KsqlStructuredDataOutputNode(io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode) UserRepartitionNode(io.confluent.ksql.planner.plan.UserRepartitionNode) WindowExpression(io.confluent.ksql.parser.tree.WindowExpression) SqlType(io.confluent.ksql.schema.ksql.types.SqlType) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) DataSourceNode(io.confluent.ksql.planner.plan.DataSourceNode) NewTopic(io.confluent.ksql.analyzer.Analysis.Into.NewTopic) KsqlWindowExpression(io.confluent.ksql.execution.windows.KsqlWindowExpression) OutputNode(io.confluent.ksql.planner.plan.OutputNode) FilterTypeValidator(io.confluent.ksql.analyzer.FilterTypeValidator) FlatMapNode(io.confluent.ksql.planner.plan.FlatMapNode) ValueFormat(io.confluent.ksql.serde.ValueFormat) SelectExpression(io.confluent.ksql.execution.plan.SelectExpression) NodeLocation(io.confluent.ksql.parser.NodeLocation) PreJoinRepartitionNode(io.confluent.ksql.planner.plan.PreJoinRepartitionNode) FunctionCall(io.confluent.ksql.execution.expression.tree.FunctionCall) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) PlanNode(io.confluent.ksql.planner.plan.PlanNode) Context(io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter.Context) ExpressionTreeRewriter(io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter) OutputRefinement(io.confluent.ksql.parser.OutputRefinement) FilterType(io.confluent.ksql.analyzer.FilterTypeValidator.FilterType) Collections(java.util.Collections) DataSourceNode(io.confluent.ksql.planner.plan.DataSourceNode) SingleSourcePlanNode(io.confluent.ksql.planner.plan.SingleSourcePlanNode) PlanNode(io.confluent.ksql.planner.plan.PlanNode) JoinKey(io.confluent.ksql.planner.plan.JoinNode.JoinKey) Expression(io.confluent.ksql.execution.expression.tree.Expression) WindowExpression(io.confluent.ksql.parser.tree.WindowExpression) KsqlWindowExpression(io.confluent.ksql.execution.windows.KsqlWindowExpression) SelectExpression(io.confluent.ksql.execution.plan.SelectExpression) JoinNode(io.confluent.ksql.planner.plan.JoinNode) Join(io.confluent.ksql.planner.JoinTree.Join) Leaf(io.confluent.ksql.planner.JoinTree.Leaf) Leaf(io.confluent.ksql.planner.JoinTree.Leaf)

Example 2 with JoinKey

use of io.confluent.ksql.planner.plan.JoinNode.JoinKey in project ksql by confluentinc.

the class LogicalPlanner method buildForeignJoinKey.

private JoinKey buildForeignJoinKey(final Join join, final Expression foreignKeyExpression) {
    final AliasedDataSource leftSource = join.getInfo().getLeftSource();
    final SourceName alias = leftSource.getAlias();
    final List<QualifiedColumnReferenceExp> leftSourceKeys = leftSource.getDataSource().getSchema().key().stream().map(c -> new QualifiedColumnReferenceExp(alias, c.name())).collect(Collectors.toList());
    final VisitParentExpressionVisitor<Optional<Expression>, Context<Void>> aliasRewritter = new VisitParentExpressionVisitor<Optional<Expression>, Context<Void>>(Optional.empty()) {

        @Override
        public Optional<Expression> visitQualifiedColumnReference(final QualifiedColumnReferenceExp node, final Context<Void> ctx) {
            return Optional.of(new UnqualifiedColumnReferenceExp(ColumnNames.generatedJoinColumnAlias(node.getQualifier(), node.getColumnName())));
        }
    };
    final Expression aliasedForeignKeyExpression = ExpressionTreeRewriter.rewriteWith(aliasRewritter::process, foreignKeyExpression);
    return JoinKey.foreignKey(aliasedForeignKeyExpression, leftSourceKeys);
}
Also used : JoinInfo(io.confluent.ksql.analyzer.Analysis.JoinInfo) DataSource(io.confluent.ksql.metastore.model.DataSource) Leaf(io.confluent.ksql.planner.JoinTree.Leaf) AggregateAnalysisResult(io.confluent.ksql.analyzer.AggregateAnalysisResult) Into(io.confluent.ksql.analyzer.Analysis.Into) ColumnName(io.confluent.ksql.name.ColumnName) SourceName(io.confluent.ksql.name.SourceName) BiFunction(java.util.function.BiFunction) AggregateAnalyzer(io.confluent.ksql.analyzer.AggregateAnalyzer) FilterNode(io.confluent.ksql.planner.plan.FilterNode) SerdeFeaturesFactory(io.confluent.ksql.serde.SerdeFeaturesFactory) JoinKey(io.confluent.ksql.planner.plan.JoinNode.JoinKey) CodeGenRunner(io.confluent.ksql.execution.codegen.CodeGenRunner) WindowInfo(io.confluent.ksql.serde.WindowInfo) RewrittenAnalysis(io.confluent.ksql.analyzer.RewrittenAnalysis) QueryLimitNode(io.confluent.ksql.planner.plan.QueryLimitNode) AggregateNode(io.confluent.ksql.planner.plan.AggregateNode) AliasedDataSource(io.confluent.ksql.analyzer.Analysis.AliasedDataSource) TimestampExtractionPolicyFactory(io.confluent.ksql.execution.streams.timestamp.TimestampExtractionPolicyFactory) ExpressionTypeManager(io.confluent.ksql.execution.util.ExpressionTypeManager) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) SelectionUtil(io.confluent.ksql.planner.plan.SelectionUtil) PartitionBy(io.confluent.ksql.parser.tree.PartitionBy) ColumnReferenceExp(io.confluent.ksql.execution.expression.tree.ColumnReferenceExp) PreJoinProjectNode(io.confluent.ksql.planner.plan.PreJoinProjectNode) VisitParentExpressionVisitor(io.confluent.ksql.execution.expression.tree.VisitParentExpressionVisitor) FinalProjectNode(io.confluent.ksql.planner.plan.FinalProjectNode) ColumnNames(io.confluent.ksql.schema.ksql.ColumnNames) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) ExpressionEvaluator(io.confluent.ksql.execution.transform.ExpressionEvaluator) Expression(io.confluent.ksql.execution.expression.tree.Expression) JoinType(io.confluent.ksql.planner.plan.JoinNode.JoinType) Set(java.util.Set) QueryFilterNode(io.confluent.ksql.planner.plan.QueryFilterNode) KsqlConfig(io.confluent.ksql.util.KsqlConfig) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) TimestampColumn(io.confluent.ksql.execution.timestamp.TimestampColumn) PlanNodeId(io.confluent.ksql.planner.plan.PlanNodeId) SingleSourcePlanNode(io.confluent.ksql.planner.plan.SingleSourcePlanNode) Builder(io.confluent.ksql.schema.ksql.LogicalSchema.Builder) Objects(java.util.Objects) Join(io.confluent.ksql.planner.JoinTree.Join) List(java.util.List) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) QueryProjectNode(io.confluent.ksql.planner.plan.QueryProjectNode) Column(io.confluent.ksql.schema.ksql.Column) FormatInfo(io.confluent.ksql.serde.FormatInfo) ProjectNode(io.confluent.ksql.planner.plan.ProjectNode) Iterables(com.google.common.collect.Iterables) FormatFactory(io.confluent.ksql.serde.FormatFactory) GrammaticalJoiner(io.confluent.ksql.util.GrammaticalJoiner) KeyFormat(io.confluent.ksql.serde.KeyFormat) JoinNode(io.confluent.ksql.planner.plan.JoinNode) SuppressNode(io.confluent.ksql.planner.plan.SuppressNode) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) PartitionByParamsFactory(io.confluent.ksql.execution.streams.PartitionByParamsFactory) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) Function(java.util.function.Function) NoneFormat(io.confluent.ksql.serde.none.NoneFormat) AsValue(io.confluent.ksql.function.udf.AsValue) QualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.QualifiedColumnReferenceExp) MetaStore(io.confluent.ksql.metastore.MetaStore) GroupBy(io.confluent.ksql.parser.tree.GroupBy) KsqlStructuredDataOutputNode(io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode) UserRepartitionNode(io.confluent.ksql.planner.plan.UserRepartitionNode) WindowExpression(io.confluent.ksql.parser.tree.WindowExpression) SqlType(io.confluent.ksql.schema.ksql.types.SqlType) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) DataSourceNode(io.confluent.ksql.planner.plan.DataSourceNode) NewTopic(io.confluent.ksql.analyzer.Analysis.Into.NewTopic) KsqlWindowExpression(io.confluent.ksql.execution.windows.KsqlWindowExpression) OutputNode(io.confluent.ksql.planner.plan.OutputNode) FilterTypeValidator(io.confluent.ksql.analyzer.FilterTypeValidator) FlatMapNode(io.confluent.ksql.planner.plan.FlatMapNode) ValueFormat(io.confluent.ksql.serde.ValueFormat) SelectExpression(io.confluent.ksql.execution.plan.SelectExpression) NodeLocation(io.confluent.ksql.parser.NodeLocation) PreJoinRepartitionNode(io.confluent.ksql.planner.plan.PreJoinRepartitionNode) FunctionCall(io.confluent.ksql.execution.expression.tree.FunctionCall) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) PlanNode(io.confluent.ksql.planner.plan.PlanNode) Context(io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter.Context) ExpressionTreeRewriter(io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter) OutputRefinement(io.confluent.ksql.parser.OutputRefinement) FilterType(io.confluent.ksql.analyzer.FilterTypeValidator.FilterType) Collections(java.util.Collections) Context(io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter.Context) Optional(java.util.Optional) QualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.QualifiedColumnReferenceExp) SourceName(io.confluent.ksql.name.SourceName) VisitParentExpressionVisitor(io.confluent.ksql.execution.expression.tree.VisitParentExpressionVisitor) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) AliasedDataSource(io.confluent.ksql.analyzer.Analysis.AliasedDataSource) Expression(io.confluent.ksql.execution.expression.tree.Expression) WindowExpression(io.confluent.ksql.parser.tree.WindowExpression) KsqlWindowExpression(io.confluent.ksql.execution.windows.KsqlWindowExpression) SelectExpression(io.confluent.ksql.execution.plan.SelectExpression)

Aggregations

Iterables (com.google.common.collect.Iterables)2 AggregateAnalysisResult (io.confluent.ksql.analyzer.AggregateAnalysisResult)2 AggregateAnalyzer (io.confluent.ksql.analyzer.AggregateAnalyzer)2 AliasedDataSource (io.confluent.ksql.analyzer.Analysis.AliasedDataSource)2 Into (io.confluent.ksql.analyzer.Analysis.Into)2 NewTopic (io.confluent.ksql.analyzer.Analysis.Into.NewTopic)2 JoinInfo (io.confluent.ksql.analyzer.Analysis.JoinInfo)2 FilterTypeValidator (io.confluent.ksql.analyzer.FilterTypeValidator)2 FilterType (io.confluent.ksql.analyzer.FilterTypeValidator.FilterType)2 ImmutableAnalysis (io.confluent.ksql.analyzer.ImmutableAnalysis)2 RewrittenAnalysis (io.confluent.ksql.analyzer.RewrittenAnalysis)2 ExpressionTreeRewriter (io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter)2 Context (io.confluent.ksql.engine.rewrite.ExpressionTreeRewriter.Context)2 CodeGenRunner (io.confluent.ksql.execution.codegen.CodeGenRunner)2 KsqlTopic (io.confluent.ksql.execution.ddl.commands.KsqlTopic)2 ColumnReferenceExp (io.confluent.ksql.execution.expression.tree.ColumnReferenceExp)2 Expression (io.confluent.ksql.execution.expression.tree.Expression)2 FunctionCall (io.confluent.ksql.execution.expression.tree.FunctionCall)2 QualifiedColumnReferenceExp (io.confluent.ksql.execution.expression.tree.QualifiedColumnReferenceExp)2 UnqualifiedColumnReferenceExp (io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp)2