Search in sources :

Example 1 with SpatialJoinNode

use of com.facebook.presto.sql.planner.plan.SpatialJoinNode in project presto by prestodb.

the class TestVerifyNoOriginalExpression method validateSpatialJoinWithFilter.

private void validateSpatialJoinWithFilter(RowExpression filter) {
    ImmutableList<VariableReferenceExpression> outputVariables = ImmutableList.of(VARIABLE_REFERENCE_EXPRESSION);
    Optional<VariableReferenceExpression> leftPartitionVariable = Optional.of(VARIABLE_REFERENCE_EXPRESSION);
    Optional<VariableReferenceExpression> rightPartitionVariable = Optional.of(VARIABLE_REFERENCE_EXPRESSION);
    Optional<String> kdbTree = Optional.of("");
    ImmutableMap<VariableReferenceExpression, RowExpression> map = ImmutableMap.of(VARIABLE_REFERENCE_EXPRESSION, VARIABLE_REFERENCE_EXPRESSION);
    ProjectNode projectNode = builder.project(new Assignments(map), valuesNode);
    SpatialJoinNode spatialJoinNode = new SpatialJoinNode(Optional.empty(), new PlanNodeId("1"), SpatialJoinNode.Type.INNER, projectNode, projectNode, outputVariables, filter, leftPartitionVariable, rightPartitionVariable, kdbTree);
    testValidation(spatialJoinNode);
}
Also used : PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) Assignments(com.facebook.presto.spi.plan.Assignments) RowExpression(com.facebook.presto.spi.relation.RowExpression) OriginalExpressionUtils.castToRowExpression(com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression) ProjectNode(com.facebook.presto.spi.plan.ProjectNode) SpatialJoinNode(com.facebook.presto.sql.planner.plan.SpatialJoinNode)

Example 2 with SpatialJoinNode

use of com.facebook.presto.sql.planner.plan.SpatialJoinNode in project presto by prestodb.

the class ExtractSpatialJoins method tryCreateSpatialJoin.

private static Result tryCreateSpatialJoin(Context context, JoinNode joinNode, RowExpression filter, PlanNodeId nodeId, List<VariableReferenceExpression> outputVariables, CallExpression spatialFunction, Optional<RowExpression> radius, Metadata metadata, SplitManager splitManager, PageSourceManager pageSourceManager) {
    FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager();
    List<RowExpression> arguments = spatialFunction.getArguments();
    verify(arguments.size() == 2);
    RowExpression firstArgument = arguments.get(0);
    RowExpression secondArgument = arguments.get(1);
    // Currently, only inner joins are supported for spherical geometries.
    if (joinNode.getType() != INNER && isSphericalJoin(metadata, firstArgument, secondArgument)) {
        return Result.empty();
    }
    Set<VariableReferenceExpression> firstVariables = extractUnique(firstArgument);
    Set<VariableReferenceExpression> secondVariables = extractUnique(secondArgument);
    if (firstVariables.isEmpty() || secondVariables.isEmpty()) {
        return Result.empty();
    }
    // If either firstArgument or secondArgument is not a
    // VariableReferenceExpression, will replace the left/right join node
    // with a projection that adds the argument as a variable.
    Optional<VariableReferenceExpression> newFirstVariable = newGeometryVariable(context, firstArgument);
    Optional<VariableReferenceExpression> newSecondVariable = newGeometryVariable(context, secondArgument);
    PlanNode leftNode = joinNode.getLeft();
    PlanNode rightNode = joinNode.getRight();
    PlanNode newLeftNode;
    PlanNode newRightNode;
    // Check if the order of arguments of the spatial function matches the order of join sides
    int alignment = checkAlignment(joinNode, firstVariables, secondVariables);
    if (alignment > 0) {
        newLeftNode = newFirstVariable.map(variable -> addProjection(context, leftNode, variable, firstArgument)).orElse(leftNode);
        newRightNode = newSecondVariable.map(variable -> addProjection(context, rightNode, variable, secondArgument)).orElse(rightNode);
    } else if (alignment < 0) {
        newLeftNode = newSecondVariable.map(variable -> addProjection(context, leftNode, variable, secondArgument)).orElse(leftNode);
        newRightNode = newFirstVariable.map(variable -> addProjection(context, rightNode, variable, firstArgument)).orElse(rightNode);
    } else {
        return Result.empty();
    }
    RowExpression newFirstArgument = mapToExpression(newFirstVariable, firstArgument);
    RowExpression newSecondArgument = mapToExpression(newSecondVariable, secondArgument);
    // Implement partitioned spatial joins:
    // If the session parameter points to a valid spatial partitioning, use
    // that to assign to each probe and build rows the partitions that the
    // geometry intersects.  This is a projection that adds an array of ints
    // which is subsequently unnested.
    Optional<String> spatialPartitioningTableName = canPartitionSpatialJoin(joinNode) ? getSpatialPartitioningTableName(context.getSession()) : Optional.empty();
    Optional<KdbTree> kdbTree = spatialPartitioningTableName.map(tableName -> loadKdbTree(tableName, context.getSession(), metadata, splitManager, pageSourceManager));
    Optional<VariableReferenceExpression> leftPartitionVariable = Optional.empty();
    Optional<VariableReferenceExpression> rightPartitionVariable = Optional.empty();
    if (kdbTree.isPresent()) {
        leftPartitionVariable = Optional.of(context.getVariableAllocator().newVariable(newFirstArgument.getSourceLocation(), "pid", INTEGER));
        rightPartitionVariable = Optional.of(context.getVariableAllocator().newVariable(newSecondArgument.getSourceLocation(), "pid", INTEGER));
        if (alignment > 0) {
            newLeftNode = addPartitioningNodes(context, functionAndTypeManager, newLeftNode, leftPartitionVariable.get(), kdbTree.get(), newFirstArgument, Optional.empty());
            newRightNode = addPartitioningNodes(context, functionAndTypeManager, newRightNode, rightPartitionVariable.get(), kdbTree.get(), newSecondArgument, radius);
        } else {
            newLeftNode = addPartitioningNodes(context, functionAndTypeManager, newLeftNode, leftPartitionVariable.get(), kdbTree.get(), newSecondArgument, Optional.empty());
            newRightNode = addPartitioningNodes(context, functionAndTypeManager, newRightNode, rightPartitionVariable.get(), kdbTree.get(), newFirstArgument, radius);
        }
    }
    CallExpression newSpatialFunction = new CallExpression(spatialFunction.getSourceLocation(), spatialFunction.getDisplayName(), spatialFunction.getFunctionHandle(), spatialFunction.getType(), ImmutableList.of(newFirstArgument, newSecondArgument));
    RowExpression newFilter = RowExpressionNodeInliner.replaceExpression(filter, ImmutableMap.of(spatialFunction, newSpatialFunction));
    return Result.ofPlanNode(new SpatialJoinNode(joinNode.getSourceLocation(), nodeId, SpatialJoinNode.Type.fromJoinNodeType(joinNode.getType()), newLeftNode, newRightNode, outputVariables, newFilter, leftPartitionVariable, rightPartitionVariable, kdbTree.map(KdbTreeUtils::toJson)));
}
Also used : FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) WarningCollector(com.facebook.presto.spi.WarningCollector) Page(com.facebook.presto.common.Page) SpatialJoinNode(com.facebook.presto.sql.planner.plan.SpatialJoinNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) TypeSignature(com.facebook.presto.common.type.TypeSignature) MoreFutures.getFutureValue(com.facebook.airlift.concurrent.MoreFutures.getFutureValue) NOT_PARTITIONED(com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) SpatialJoinUtils.getFlippedFunctionHandle(com.facebook.presto.util.SpatialJoinUtils.getFlippedFunctionHandle) Pattern(com.facebook.presto.matching.Pattern) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Capture(com.facebook.presto.matching.Capture) SpatialJoinUtils.flip(com.facebook.presto.util.SpatialJoinUtils.flip) Map(java.util.Map) UNGROUPED_SCHEDULING(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) LOCAL(com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL) QualifiedObjectName(com.facebook.presto.common.QualifiedObjectName) SystemSessionProperties.isSpatialJoinEnabled(com.facebook.presto.SystemSessionProperties.isSpatialJoinEnabled) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) CallExpression(com.facebook.presto.spi.relation.CallExpression) Splitter(com.google.common.base.Splitter) SplitSource(com.facebook.presto.split.SplitSource) Lifespan(com.facebook.presto.execution.Lifespan) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) SplitManager(com.facebook.presto.split.SplitManager) String.format(java.lang.String.format) KDB_TREE(com.facebook.presto.common.type.KdbTreeType.KDB_TREE) UncheckedIOException(java.io.UncheckedIOException) FunctionMetadata(com.facebook.presto.spi.function.FunctionMetadata) List(java.util.List) SpatialJoinUtils.extractSupportedSpatialFunctions(com.facebook.presto.util.SpatialJoinUtils.extractSupportedSpatialFunctions) KdbTreeUtils(com.facebook.presto.geospatial.KdbTreeUtils) ProjectNode(com.facebook.presto.spi.plan.ProjectNode) Capture.newCapture(com.facebook.presto.matching.Capture.newCapture) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) Optional(java.util.Optional) CAST(com.facebook.presto.metadata.CastType.CAST) VariablesExtractor.extractUnique(com.facebook.presto.sql.planner.VariablesExtractor.extractUnique) INNER(com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) Iterables(com.google.common.collect.Iterables) TableLayoutResult(com.facebook.presto.metadata.TableLayoutResult) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) Captures(com.facebook.presto.matching.Captures) Assignments(com.facebook.presto.spi.plan.Assignments) Result(com.facebook.presto.sql.planner.iterative.Rule.Result) PrestoException(com.facebook.presto.spi.PrestoException) Patterns.join(com.facebook.presto.sql.planner.plan.Patterns.join) TypeSignatureProvider.fromTypes(com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes) Patterns.filter(com.facebook.presto.sql.planner.plan.Patterns.filter) FilterNode(com.facebook.presto.spi.plan.FilterNode) SplitBatch(com.facebook.presto.split.SplitSource.SplitBatch) RowExpressionNodeInliner(com.facebook.presto.expressions.RowExpressionNodeInliner) ImmutableList(com.google.common.collect.ImmutableList) PageSourceManager(com.facebook.presto.split.PageSourceManager) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) SpatialJoinUtils.extractSupportedSpatialComparisons(com.facebook.presto.util.SpatialJoinUtils.extractSupportedSpatialComparisons) ArrayType(com.facebook.presto.common.type.ArrayType) TableHandle(com.facebook.presto.spi.TableHandle) Expressions(com.facebook.presto.sql.relational.Expressions) KdbTree(com.facebook.presto.geospatial.KdbTree) UnnestNode(com.facebook.presto.sql.planner.plan.UnnestNode) Type(com.facebook.presto.common.type.Type) RowExpression(com.facebook.presto.spi.relation.RowExpression) JoinNode(com.facebook.presto.sql.planner.plan.JoinNode) INVALID_SPATIAL_PARTITIONING(com.facebook.presto.spi.StandardErrorCode.INVALID_SPATIAL_PARTITIONING) SystemSessionProperties.getSpatialPartitioningTableName(com.facebook.presto.SystemSessionProperties.getSpatialPartitioningTableName) Session(com.facebook.presto.Session) Rule(com.facebook.presto.sql.planner.iterative.Rule) Constraint(com.facebook.presto.spi.Constraint) IOException(java.io.IOException) OperatorType(com.facebook.presto.common.function.OperatorType) Patterns.source(com.facebook.presto.sql.planner.plan.Patterns.source) PlanNode(com.facebook.presto.spi.plan.PlanNode) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TypeSignature.parseTypeSignature(com.facebook.presto.common.type.TypeSignature.parseTypeSignature) LEFT(com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT) ColumnHandle(com.facebook.presto.spi.ColumnHandle) FunctionHandle(com.facebook.presto.spi.function.FunctionHandle) Split(com.facebook.presto.metadata.Split) Context(com.facebook.presto.sql.planner.iterative.Rule.Context) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Metadata(com.facebook.presto.metadata.Metadata) RowExpressionNodeInliner.replaceExpression(com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression) KdbTreeUtils(com.facebook.presto.geospatial.KdbTreeUtils) KdbTree(com.facebook.presto.geospatial.KdbTree) RowExpression(com.facebook.presto.spi.relation.RowExpression) SpatialJoinNode(com.facebook.presto.sql.planner.plan.SpatialJoinNode) Constraint(com.facebook.presto.spi.Constraint) PlanNode(com.facebook.presto.spi.plan.PlanNode) FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) CallExpression(com.facebook.presto.spi.relation.CallExpression)

Example 3 with SpatialJoinNode

use of com.facebook.presto.sql.planner.plan.SpatialJoinNode in project presto by prestodb.

the class SpatialJoinMatcher method detailMatches.

@Override
public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) {
    checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName());
    SpatialJoinNode joinNode = (SpatialJoinNode) node;
    if (isExpression(joinNode.getFilter())) {
        if (!new ExpressionVerifier(symbolAliases).process(castToExpression(joinNode.getFilter()), filter)) {
            return NO_MATCH;
        }
    } else {
        if (!new RowExpressionVerifier(symbolAliases, metadata, session).process(filter, joinNode.getFilter())) {
            return NO_MATCH;
        }
    }
    if (!joinNode.getKdbTree().equals(kdbTree)) {
        return NO_MATCH;
    }
    return match();
}
Also used : SpatialJoinNode(com.facebook.presto.sql.planner.plan.SpatialJoinNode)

Aggregations

SpatialJoinNode (com.facebook.presto.sql.planner.plan.SpatialJoinNode)3 MoreFutures.getFutureValue (com.facebook.airlift.concurrent.MoreFutures.getFutureValue)1 Session (com.facebook.presto.Session)1 SystemSessionProperties.getSpatialPartitioningTableName (com.facebook.presto.SystemSessionProperties.getSpatialPartitioningTableName)1 SystemSessionProperties.isSpatialJoinEnabled (com.facebook.presto.SystemSessionProperties.isSpatialJoinEnabled)1 Page (com.facebook.presto.common.Page)1 QualifiedObjectName (com.facebook.presto.common.QualifiedObjectName)1 OperatorType (com.facebook.presto.common.function.OperatorType)1 ArrayType (com.facebook.presto.common.type.ArrayType)1 INTEGER (com.facebook.presto.common.type.IntegerType.INTEGER)1 KDB_TREE (com.facebook.presto.common.type.KdbTreeType.KDB_TREE)1 Type (com.facebook.presto.common.type.Type)1 TypeSignature (com.facebook.presto.common.type.TypeSignature)1 TypeSignature.parseTypeSignature (com.facebook.presto.common.type.TypeSignature.parseTypeSignature)1 VARCHAR (com.facebook.presto.common.type.VarcharType.VARCHAR)1 Lifespan (com.facebook.presto.execution.Lifespan)1 RowExpressionNodeInliner (com.facebook.presto.expressions.RowExpressionNodeInliner)1 RowExpressionNodeInliner.replaceExpression (com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression)1 KdbTree (com.facebook.presto.geospatial.KdbTree)1 KdbTreeUtils (com.facebook.presto.geospatial.KdbTreeUtils)1