Examples with ResultDescription io.crate.planner.ResultDescription used on opensource projects

Example 1 with ResultDescription

use of io.crate.planner.ResultDescription in project crate by crate.

the class HashJoin method build.

@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> hints, ProjectionBuilder projectionBuilder, int limit, int offset, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
    ExecutionPlan leftExecutionPlan = lhs.build(plannerContext, hints, projectionBuilder, NO_LIMIT, 0, null, null, params, subQueryResults);
    ExecutionPlan rightExecutionPlan = rhs.build(plannerContext, hints, projectionBuilder, NO_LIMIT, 0, null, null, params, subQueryResults);
    LogicalPlan leftLogicalPlan = lhs;
    LogicalPlan rightLogicalPlan = rhs;
    boolean tablesSwitched = false;
    // revealed that this improves performance in most cases.
    if (lhs.numExpectedRows() < rhs.numExpectedRows()) {
        tablesSwitched = true;
        leftLogicalPlan = rhs;
        rightLogicalPlan = lhs;
        ExecutionPlan tmp = leftExecutionPlan;
        leftExecutionPlan = rightExecutionPlan;
        rightExecutionPlan = tmp;
    }
    SubQueryAndParamBinder paramBinder = new SubQueryAndParamBinder(params, subQueryResults);
    Tuple<List<Symbol>, List<Symbol>> hashSymbols = extractHashJoinSymbolsFromJoinSymbolsAndSplitPerSide(tablesSwitched);
    ResultDescription leftResultDesc = leftExecutionPlan.resultDescription();
    ResultDescription rightResultDesc = rightExecutionPlan.resultDescription();
    Collection<String> joinExecutionNodes = leftResultDesc.nodeIds();
    List<Symbol> leftOutputs = leftLogicalPlan.outputs();
    List<Symbol> rightOutputs = rightLogicalPlan.outputs();
    MergePhase leftMerge = null;
    MergePhase rightMerge = null;
    // We can only run the join distributed if no remaining limit or offset must be applied on the source relations.
    // Because on distributed joins, every join is running on a slice (modulo) set of the data and so no limit/offset
    // could be applied. Limit/offset can only be applied on the whole data set after all partial rows from the
    // shards are merged
    boolean isDistributed = leftResultDesc.hasRemainingLimitOrOffset() == false && rightResultDesc.hasRemainingLimitOrOffset() == false;
    if (joinExecutionNodes.isEmpty()) {
        // The left source might have zero execution nodes, for example in the case of `sys.shards` without any tables
        // If the join then also uses zero execution nodes, a distributed plan no longer works because
        // the source operators wouldn't have a downstream node where they can send the results to.
        // → we switch to non-distributed which results in the join running on the handlerNode.
        isDistributed = false;
    }
    if (joinExecutionNodes.size() == 1 && joinExecutionNodes.equals(rightResultDesc.nodeIds()) && !rightResultDesc.hasRemainingLimitOrOffset()) {
        // If the left and the right plan are executed on the same single node the mergePhase
        // should be omitted. This is the case if the left and right table have only one shards which
        // are on the same node
        leftExecutionPlan.setDistributionInfo(DistributionInfo.DEFAULT_SAME_NODE);
        rightExecutionPlan.setDistributionInfo(DistributionInfo.DEFAULT_SAME_NODE);
    } else {
        if (isDistributed) {
            // Run the join distributed by modulo distribution algorithm
            leftOutputs = setModuloDistribution(Lists2.map(hashSymbols.v1(), paramBinder), leftLogicalPlan.outputs(), leftExecutionPlan);
            rightOutputs = setModuloDistribution(Lists2.map(hashSymbols.v2(), paramBinder), rightLogicalPlan.outputs(), rightExecutionPlan);
        } else {
            // Run the join non-distributed on the handler node
            joinExecutionNodes = Collections.singletonList(plannerContext.handlerNode());
            leftExecutionPlan.setDistributionInfo(DistributionInfo.DEFAULT_BROADCAST);
            rightExecutionPlan.setDistributionInfo(DistributionInfo.DEFAULT_BROADCAST);
        }
        leftMerge = JoinOperations.buildMergePhaseForJoin(plannerContext, leftResultDesc, joinExecutionNodes);
        rightMerge = JoinOperations.buildMergePhaseForJoin(plannerContext, rightResultDesc, joinExecutionNodes);
    }
    List<Symbol> joinOutputs = Lists2.concat(leftOutputs, rightOutputs);
    HashJoinPhase joinPhase = new HashJoinPhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), "hash-join", Collections.singletonList(JoinOperations.createJoinProjection(outputs, joinOutputs)), leftMerge, rightMerge, leftOutputs.size(), rightOutputs.size(), joinExecutionNodes, InputColumns.create(paramBinder.apply(joinCondition), joinOutputs), InputColumns.create(Lists2.map(hashSymbols.v1(), paramBinder), new InputColumns.SourceSymbols(leftOutputs)), InputColumns.create(Lists2.map(hashSymbols.v2(), paramBinder), new InputColumns.SourceSymbols(rightOutputs)), Symbols.typeView(leftOutputs), leftLogicalPlan.estimatedRowSize(), leftLogicalPlan.numExpectedRows());
    return new Join(joinPhase, leftExecutionPlan, rightExecutionPlan, TopN.NO_LIMIT, 0, TopN.NO_LIMIT, outputs.size(), null);
}

Example 2 with ResultDescription

use of io.crate.planner.ResultDescription in project crate by crate.

the class WindowAgg method build.

@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> planHints, ProjectionBuilder projectionBuilder, int limit, int offset, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
    InputColumns.SourceSymbols sourceSymbols = new InputColumns.SourceSymbols(source.outputs());
    SubQueryAndParamBinder binder = new SubQueryAndParamBinder(params, subQueryResults);
    Function<Symbol, Symbol> toInputCols = binder.andThen(s -> InputColumns.create(s, sourceSymbols));
    List<WindowFunction> boundWindowFunctions = (List<WindowFunction>) (List) Lists2.map(windowFunctions, toInputCols);
    List<Projection> projections = new ArrayList<>();
    WindowAggProjection windowAggProjection = new WindowAggProjection(windowDefinition.map(toInputCols), boundWindowFunctions, InputColumns.create(this.standalone, sourceSymbols));
    projections.add(windowAggProjection);
    ExecutionPlan sourcePlan = source.build(plannerContext, planHints, projectionBuilder, TopN.NO_LIMIT, TopN.NO_OFFSET, null, pageSizeHint, params, subQueryResults);
    ResultDescription resultDescription = sourcePlan.resultDescription();
    boolean executesOnHandler = executesOnHandler(plannerContext.handlerNode(), resultDescription.nodeIds());
    boolean nonDistExecution = windowDefinition.partitions().isEmpty() || resultDescription.hasRemainingLimitOrOffset() || executesOnHandler;
    if (nonDistExecution) {
        sourcePlan = Merge.ensureOnHandler(sourcePlan, plannerContext);
        for (Projection projection : projections) {
            sourcePlan.addProjection(projection);
        }
    } else {
        sourcePlan.setDistributionInfo(new DistributionInfo(DistributionType.MODULO, source.outputs().indexOf(windowDefinition.partitions().iterator().next())));
        MergePhase distWindowAgg = new MergePhase(UUIDs.dirtyUUID(), plannerContext.nextExecutionPhaseId(), "distWindowAgg", resultDescription.nodeIds().size(), resultDescription.numOutputs(), resultDescription.nodeIds(), resultDescription.streamOutputs(), projections, DistributionInfo.DEFAULT_BROADCAST, null);
        return new Merge(sourcePlan, distWindowAgg, TopN.NO_LIMIT, TopN.NO_OFFSET, windowAggProjection.outputs().size(), resultDescription.maxRowsPerNode(), null);
    }
    return sourcePlan;
}

Example 3 with ResultDescription

use of io.crate.planner.ResultDescription in project crate by crate.

the class Limit method build.

@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> planHints, ProjectionBuilder projectionBuilder, int limitHint, int offsetHint, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
    int limit = Objects.requireNonNullElse(DataTypes.INTEGER.sanitizeValue(evaluate(plannerContext.transactionContext(), plannerContext.nodeContext(), this.limit, params, subQueryResults)), NO_LIMIT);
    int offset = Objects.requireNonNullElse(DataTypes.INTEGER.sanitizeValue(evaluate(plannerContext.transactionContext(), plannerContext.nodeContext(), this.offset, params, subQueryResults)), 0);
    ExecutionPlan executionPlan = source.build(plannerContext, planHints, projectionBuilder, limit, offset, order, pageSizeHint, params, subQueryResults);
    List<DataType<?>> sourceTypes = Symbols.typeView(source.outputs());
    ResultDescription resultDescription = executionPlan.resultDescription();
    if (resultDescription.hasRemainingLimitOrOffset() && (resultDescription.limit() != limit || resultDescription.offset() != offset)) {
        executionPlan = Merge.ensureOnHandler(executionPlan, plannerContext);
        resultDescription = executionPlan.resultDescription();
    }
    if (ExecutionPhases.executesOnHandler(plannerContext.handlerNode(), resultDescription.nodeIds())) {
        executionPlan.addProjection(new TopNProjection(limit, offset, sourceTypes), TopN.NO_LIMIT, 0, resultDescription.orderBy());
    } else if (resultDescription.limit() != limit || resultDescription.offset() != 0) {
        executionPlan.addProjection(new TopNProjection(limit + offset, 0, sourceTypes), limit, offset, resultDescription.orderBy());
    }
    return executionPlan;
}

Example 4 with ResultDescription

use of io.crate.planner.ResultDescription in project crate by crate.

the class Union method build.

@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> hints, ProjectionBuilder projectionBuilder, int limit, int offset, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
    Integer childPageSizeHint = limit != TopN.NO_LIMIT ? limitAndOffset(limit, offset) : null;
    ExecutionPlan left = lhs.build(plannerContext, hints, projectionBuilder, limit + offset, TopN.NO_OFFSET, null, childPageSizeHint, params, subQueryResults);
    ExecutionPlan right = rhs.build(plannerContext, hints, projectionBuilder, limit + offset, TopN.NO_OFFSET, null, childPageSizeHint, params, subQueryResults);
    addCastsForIncompatibleObjects(right);
    if (left.resultDescription().hasRemainingLimitOrOffset()) {
        left = Merge.ensureOnHandler(left, plannerContext);
    }
    if (right.resultDescription().hasRemainingLimitOrOffset()) {
        right = Merge.ensureOnHandler(right, plannerContext);
    }
    ResultDescription leftResultDesc = left.resultDescription();
    ResultDescription rightResultDesc = right.resultDescription();
    assert DataTypes.isCompatibleType(leftResultDesc.streamOutputs(), rightResultDesc.streamOutputs()) : "Left and right must output the same types, got " + "lhs=" + leftResultDesc.streamOutputs() + ", rhs=" + rightResultDesc.streamOutputs();
    MergePhase mergePhase = new MergePhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), "union", leftResultDesc.nodeIds().size() + rightResultDesc.nodeIds().size(), 2, Collections.singletonList(plannerContext.handlerNode()), leftResultDesc.streamOutputs(), Collections.emptyList(), DistributionInfo.DEFAULT_BROADCAST, leftResultDesc.orderBy());
    return new UnionExecutionPlan(left, right, mergePhase, limit, offset, lhs.outputs().size(), TopN.NO_LIMIT, leftResultDesc.orderBy());
}

Example 5 with ResultDescription

use of io.crate.planner.ResultDescription in project crate by crate.

the class NestedLoopJoin method configureExecution.

private Tuple<Collection<String>, List<MergePhase>> configureExecution(ExecutionPlan left, ExecutionPlan right, PlannerContext plannerContext, boolean isDistributed) {
    Collection<String> nlExecutionNodes = Set.of(plannerContext.handlerNode());
    ResultDescription leftResultDesc = left.resultDescription();
    ResultDescription rightResultDesc = right.resultDescription();
    MergePhase leftMerge = null;
    MergePhase rightMerge = null;
    if (leftResultDesc.nodeIds().size() == 1 && leftResultDesc.nodeIds().equals(rightResultDesc.nodeIds()) && !rightResultDesc.hasRemainingLimitOrOffset()) {
        // if the left and the right plan are executed on the same single node the mergePhase
        // should be omitted. This is the case if the left and right table have only one shards which
        // are on the same node
        nlExecutionNodes = leftResultDesc.nodeIds();
        left.setDistributionInfo(DistributionInfo.DEFAULT_SAME_NODE);
        right.setDistributionInfo(DistributionInfo.DEFAULT_SAME_NODE);
    } else if (isDistributed && !leftResultDesc.hasRemainingLimitOrOffset()) {
        // run join phase distributed on all nodes of the left relation
        nlExecutionNodes = leftResultDesc.nodeIds();
        left.setDistributionInfo(DistributionInfo.DEFAULT_SAME_NODE);
        right.setDistributionInfo(DistributionInfo.DEFAULT_BROADCAST);
        rightMerge = JoinOperations.buildMergePhaseForJoin(plannerContext, rightResultDesc, nlExecutionNodes);
    } else {
        // run join phase non-distributed on the handler
        left.setDistributionInfo(DistributionInfo.DEFAULT_BROADCAST);
        right.setDistributionInfo(DistributionInfo.DEFAULT_BROADCAST);
        if (JoinOperations.isMergePhaseNeeded(nlExecutionNodes, leftResultDesc, false)) {
            leftMerge = JoinOperations.buildMergePhaseForJoin(plannerContext, leftResultDesc, nlExecutionNodes);
        }
        if (JoinOperations.isMergePhaseNeeded(nlExecutionNodes, rightResultDesc, false)) {
            rightMerge = JoinOperations.buildMergePhaseForJoin(plannerContext, rightResultDesc, nlExecutionNodes);
        }
    }
    return new Tuple<>(nlExecutionNodes, Arrays.asList(leftMerge, rightMerge));
}