use of io.trino.sql.planner.plan.JoinNode.Type.INNER in project trino by trinodb.
the class ExtractSpatialJoins method tryCreateSpatialJoin.
private static Result tryCreateSpatialJoin(Context context, JoinNode joinNode, Expression filter, PlanNodeId nodeId, List<Symbol> outputSymbols, FunctionCall spatialFunction, Optional<Expression> radius, PlannerContext plannerContext, SplitManager splitManager, PageSourceManager pageSourceManager, TypeAnalyzer typeAnalyzer) {
// TODO Add support for distributed left spatial joins
Optional<String> spatialPartitioningTableName = joinNode.getType() == INNER ? getSpatialPartitioningTableName(context.getSession()) : Optional.empty();
Optional<KdbTree> kdbTree = spatialPartitioningTableName.map(tableName -> loadKdbTree(tableName, context.getSession(), plannerContext.getMetadata(), splitManager, pageSourceManager));
List<Expression> arguments = spatialFunction.getArguments();
verify(arguments.size() == 2);
Expression firstArgument = arguments.get(0);
Expression secondArgument = arguments.get(1);
Type sphericalGeographyType = plannerContext.getTypeManager().getType(SPHERICAL_GEOGRAPHY_TYPE_SIGNATURE);
if (typeAnalyzer.getType(context.getSession(), context.getSymbolAllocator().getTypes(), firstArgument).equals(sphericalGeographyType) || typeAnalyzer.getType(context.getSession(), context.getSymbolAllocator().getTypes(), secondArgument).equals(sphericalGeographyType)) {
return Result.empty();
}
Set<Symbol> firstSymbols = extractUnique(firstArgument);
Set<Symbol> secondSymbols = extractUnique(secondArgument);
if (firstSymbols.isEmpty() || secondSymbols.isEmpty()) {
return Result.empty();
}
Optional<Symbol> newFirstSymbol = newGeometrySymbol(context, firstArgument, plannerContext.getTypeManager());
Optional<Symbol> newSecondSymbol = newGeometrySymbol(context, secondArgument, plannerContext.getTypeManager());
PlanNode leftNode = joinNode.getLeft();
PlanNode rightNode = joinNode.getRight();
PlanNode newLeftNode;
PlanNode newRightNode;
// Check if the order of arguments of the spatial function matches the order of join sides
int alignment = checkAlignment(joinNode, firstSymbols, secondSymbols);
if (alignment > 0) {
newLeftNode = newFirstSymbol.map(symbol -> addProjection(context, leftNode, symbol, firstArgument)).orElse(leftNode);
newRightNode = newSecondSymbol.map(symbol -> addProjection(context, rightNode, symbol, secondArgument)).orElse(rightNode);
} else if (alignment < 0) {
newLeftNode = newSecondSymbol.map(symbol -> addProjection(context, leftNode, symbol, secondArgument)).orElse(leftNode);
newRightNode = newFirstSymbol.map(symbol -> addProjection(context, rightNode, symbol, firstArgument)).orElse(rightNode);
} else {
return Result.empty();
}
Expression newFirstArgument = toExpression(newFirstSymbol, firstArgument);
Expression newSecondArgument = toExpression(newSecondSymbol, secondArgument);
Optional<Symbol> leftPartitionSymbol = Optional.empty();
Optional<Symbol> rightPartitionSymbol = Optional.empty();
if (kdbTree.isPresent()) {
leftPartitionSymbol = Optional.of(context.getSymbolAllocator().newSymbol("pid", INTEGER));
rightPartitionSymbol = Optional.of(context.getSymbolAllocator().newSymbol("pid", INTEGER));
if (alignment > 0) {
newLeftNode = addPartitioningNodes(plannerContext, context, newLeftNode, leftPartitionSymbol.get(), kdbTree.get(), newFirstArgument, Optional.empty());
newRightNode = addPartitioningNodes(plannerContext, context, newRightNode, rightPartitionSymbol.get(), kdbTree.get(), newSecondArgument, radius);
} else {
newLeftNode = addPartitioningNodes(plannerContext, context, newLeftNode, leftPartitionSymbol.get(), kdbTree.get(), newSecondArgument, Optional.empty());
newRightNode = addPartitioningNodes(plannerContext, context, newRightNode, rightPartitionSymbol.get(), kdbTree.get(), newFirstArgument, radius);
}
}
Expression newSpatialFunction = FunctionCallBuilder.resolve(context.getSession(), plannerContext.getMetadata()).setName(spatialFunction.getName()).addArgument(GEOMETRY_TYPE_SIGNATURE, newFirstArgument).addArgument(GEOMETRY_TYPE_SIGNATURE, newSecondArgument).build();
Expression newFilter = replaceExpression(filter, ImmutableMap.of(spatialFunction, newSpatialFunction));
return Result.ofPlanNode(new SpatialJoinNode(nodeId, SpatialJoinNode.Type.fromJoinNodeType(joinNode.getType()), newLeftNode, newRightNode, outputSymbols, newFilter, leftPartitionSymbol, rightPartitionSymbol, kdbTree.map(KdbTreeUtils::toJson)));
}
use of io.trino.sql.planner.plan.JoinNode.Type.INNER in project trino by trinodb.
the class PushProjectionThroughJoin method pushProjectionThroughJoin.
public static Optional<PlanNode> pushProjectionThroughJoin(PlannerContext plannerContext, ProjectNode projectNode, Lookup lookup, PlanNodeIdAllocator planNodeIdAllocator, Session session, TypeAnalyzer typeAnalyzer, TypeProvider types) {
if (!projectNode.getAssignments().getExpressions().stream().allMatch(expression -> isDeterministic(expression, plannerContext.getMetadata()))) {
return Optional.empty();
}
PlanNode child = lookup.resolve(projectNode.getSource());
if (!(child instanceof JoinNode)) {
return Optional.empty();
}
JoinNode joinNode = (JoinNode) child;
PlanNode leftChild = joinNode.getLeft();
PlanNode rightChild = joinNode.getRight();
if (joinNode.getType() != INNER) {
return Optional.empty();
}
Assignments.Builder leftAssignmentsBuilder = Assignments.builder();
Assignments.Builder rightAssignmentsBuilder = Assignments.builder();
for (Map.Entry<Symbol, Expression> assignment : projectNode.getAssignments().entrySet()) {
Expression expression = assignment.getValue();
Set<Symbol> symbols = extractUnique(expression);
if (leftChild.getOutputSymbols().containsAll(symbols)) {
// expression is satisfied with left child symbols
leftAssignmentsBuilder.put(assignment.getKey(), expression);
} else if (rightChild.getOutputSymbols().containsAll(symbols)) {
// expression is satisfied with right child symbols
rightAssignmentsBuilder.put(assignment.getKey(), expression);
} else {
// expression is using symbols from both join sides
return Optional.empty();
}
}
// add projections for symbols required by the join itself
Set<Symbol> joinRequiredSymbols = getJoinRequiredSymbols(joinNode);
for (Symbol requiredSymbol : joinRequiredSymbols) {
if (leftChild.getOutputSymbols().contains(requiredSymbol)) {
leftAssignmentsBuilder.putIdentity(requiredSymbol);
} else {
checkState(rightChild.getOutputSymbols().contains(requiredSymbol));
rightAssignmentsBuilder.putIdentity(requiredSymbol);
}
}
Assignments leftAssignments = leftAssignmentsBuilder.build();
Assignments rightAssignments = rightAssignmentsBuilder.build();
List<Symbol> leftOutputSymbols = leftAssignments.getOutputs().stream().filter(ImmutableSet.copyOf(projectNode.getOutputSymbols())::contains).collect(toImmutableList());
List<Symbol> rightOutputSymbols = rightAssignments.getOutputs().stream().filter(ImmutableSet.copyOf(projectNode.getOutputSymbols())::contains).collect(toImmutableList());
return Optional.of(new JoinNode(joinNode.getId(), joinNode.getType(), inlineProjections(plannerContext, new ProjectNode(planNodeIdAllocator.getNextId(), leftChild, leftAssignments), lookup, session, typeAnalyzer, types), inlineProjections(plannerContext, new ProjectNode(planNodeIdAllocator.getNextId(), rightChild, rightAssignments), lookup, session, typeAnalyzer, types), joinNode.getCriteria(), leftOutputSymbols, rightOutputSymbols, joinNode.isMaySkipOutputDuplicates(), joinNode.getFilter(), joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType(), joinNode.isSpillable(), joinNode.getDynamicFilters(), joinNode.getReorderJoinStatsAndCost()));
}
use of io.trino.sql.planner.plan.JoinNode.Type.INNER in project trino by trinodb.
the class TransformFilteringSemiJoinToInnerJoin method apply.
@Override
public Result apply(FilterNode filterNode, Captures captures, Context context) {
SemiJoinNode semiJoin = captures.get(SEMI_JOIN);
// Do not transform semi-join in context of DELETE
if (PlanNodeSearcher.searchFrom(semiJoin.getSource(), context.getLookup()).where(node -> node instanceof TableScanNode && ((TableScanNode) node).isUpdateTarget()).matches()) {
return Result.empty();
}
Symbol semiJoinSymbol = semiJoin.getSemiJoinOutput();
Predicate<Expression> isSemiJoinSymbol = expression -> expression.equals(semiJoinSymbol.toSymbolReference());
List<Expression> conjuncts = extractConjuncts(filterNode.getPredicate());
if (conjuncts.stream().noneMatch(isSemiJoinSymbol)) {
return Result.empty();
}
Expression filteredPredicate = and(conjuncts.stream().filter(not(isSemiJoinSymbol)).collect(toImmutableList()));
Expression simplifiedPredicate = inlineSymbols(symbol -> {
if (symbol.equals(semiJoinSymbol)) {
return TRUE_LITERAL;
}
return symbol.toSymbolReference();
}, filteredPredicate);
Optional<Expression> joinFilter = simplifiedPredicate.equals(TRUE_LITERAL) ? Optional.empty() : Optional.of(simplifiedPredicate);
PlanNode filteringSourceDistinct = new AggregationNode(context.getIdAllocator().getNextId(), semiJoin.getFilteringSource(), ImmutableMap.of(), singleGroupingSet(ImmutableList.of(semiJoin.getFilteringSourceJoinSymbol())), ImmutableList.of(), SINGLE, Optional.empty(), Optional.empty());
JoinNode innerJoin = new JoinNode(semiJoin.getId(), INNER, semiJoin.getSource(), filteringSourceDistinct, ImmutableList.of(new EquiJoinClause(semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), semiJoin.getSource().getOutputSymbols(), ImmutableList.of(), false, joinFilter, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), semiJoin.getDynamicFilterId().map(id -> ImmutableMap.of(id, semiJoin.getFilteringSourceJoinSymbol())).orElse(ImmutableMap.of()), Optional.empty());
ProjectNode project = new ProjectNode(context.getIdAllocator().getNextId(), innerJoin, Assignments.builder().putIdentities(innerJoin.getOutputSymbols()).put(semiJoinSymbol, TRUE_LITERAL).build());
return Result.ofPlanNode(project);
}
use of io.trino.sql.planner.plan.JoinNode.Type.INNER in project trino by trinodb.
the class TestPushDownDereferencesRules method testPushdownDereferencesThroughUnnest.
@Test
public void testPushdownDereferencesThroughUnnest() {
ArrayType arrayType = new ArrayType(BIGINT);
tester().assertThat(new PushDownDereferenceThroughUnnest(tester().getTypeAnalyzer())).on(p -> p.project(Assignments.of(p.symbol("x"), expression("msg[1]")), p.unnest(ImmutableList.of(p.symbol("msg", ROW_TYPE)), ImmutableList.of(new UnnestNode.Mapping(p.symbol("arr", arrayType), ImmutableList.of(p.symbol("field")))), Optional.empty(), INNER, Optional.empty(), p.values(p.symbol("msg", ROW_TYPE), p.symbol("arr", arrayType))))).matches(strictProject(ImmutableMap.of("x", PlanMatchPattern.expression("msg_x")), unnest(strictProject(ImmutableMap.of("msg_x", PlanMatchPattern.expression("msg[1]"), "msg", PlanMatchPattern.expression("msg"), "arr", PlanMatchPattern.expression("arr")), values("msg", "arr")))));
// Test with dereferences on unnested column
RowType rowType = rowType(field("f1", BIGINT), field("f2", BIGINT));
ArrayType nestedColumnType = new ArrayType(rowType(field("f1", BIGINT), field("f2", rowType)));
tester().assertThat(new PushDownDereferenceThroughUnnest(tester().getTypeAnalyzer())).on(p -> p.project(Assignments.of(p.symbol("deref_replicate", BIGINT), expression("replicate[2]"), p.symbol("deref_unnest", BIGINT), expression("unnested_row[2]")), p.unnest(ImmutableList.of(p.symbol("replicate", rowType)), ImmutableList.of(new UnnestNode.Mapping(p.symbol("nested", nestedColumnType), ImmutableList.of(p.symbol("unnested_bigint", BIGINT), p.symbol("unnested_row", rowType)))), p.values(p.symbol("replicate", rowType), p.symbol("nested", nestedColumnType))))).matches(strictProject(ImmutableMap.of("deref_replicate", PlanMatchPattern.expression("symbol"), "deref_unnest", // not pushed down
PlanMatchPattern.expression("unnested_row[2]")), unnest(ImmutableList.of("replicate", "symbol"), ImmutableList.of(unnestMapping("nested", ImmutableList.of("unnested_bigint", "unnested_row"))), strictProject(ImmutableMap.of("symbol", PlanMatchPattern.expression("replicate[2]"), "replicate", PlanMatchPattern.expression("replicate"), "nested", PlanMatchPattern.expression("nested")), values("replicate", "nested")))));
}
use of io.trino.sql.planner.plan.JoinNode.Type.INNER in project trino by trinodb.
the class TestReorderJoins method testReplicatesWhenNotRestricted.
@Test
public void testReplicatesWhenNotRestricted() {
// variable width so that average row size is respected
Type symbolType = createUnboundedVarcharType();
int aRows = 10_000;
int bRows = 10;
PlanNodeStatsEstimate probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
PlanNodeStatsEstimate buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
// B table is small enough to be replicated in AUTOMATIC_RESTRICTED mode
assertReorderJoins().setSystemProperty(JOIN_DISTRIBUTION_TYPE, AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").on(p -> {
Symbol a1 = p.symbol("A1", symbolType);
Symbol b1 = p.symbol("B1", symbolType);
return p.join(INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1), ImmutableList.of(b1), Optional.empty());
}).overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).matches(join(INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
// B table exceeds AUTOMATIC_RESTRICTED limit therefore it is partitioned
assertReorderJoins().setSystemProperty(JOIN_DISTRIBUTION_TYPE, AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").on(p -> {
Symbol a1 = p.symbol("A1", symbolType);
Symbol b1 = p.symbol("B1", symbolType);
return p.join(INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1), ImmutableList.of(b1), Optional.empty());
}).overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).matches(join(INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(PARTITIONED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
Aggregations