use of io.crate.planner.ExecutionPlan in project crate by crate.
the class HashAggregate method build.
@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> planHints, ProjectionBuilder projectionBuilder, int limit, int offset, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
ExecutionPlan executionPlan = source.build(plannerContext, planHints, projectionBuilder, LogicalPlanner.NO_LIMIT, 0, null, null, params, subQueryResults);
AggregationOutputValidator.validateOutputs(aggregates);
var paramBinder = new SubQueryAndParamBinder(params, subQueryResults);
var sourceOutputs = source.outputs();
if (executionPlan.resultDescription().hasRemainingLimitOrOffset()) {
executionPlan = Merge.ensureOnHandler(executionPlan, plannerContext);
}
if (ExecutionPhases.executesOnHandler(plannerContext.handlerNode(), executionPlan.resultDescription().nodeIds())) {
if (source.preferShardProjections()) {
executionPlan.addProjection(projectionBuilder.aggregationProjection(sourceOutputs, aggregates, paramBinder, AggregateMode.ITER_PARTIAL, RowGranularity.SHARD, plannerContext.transactionContext().sessionContext().searchPath()));
executionPlan.addProjection(projectionBuilder.aggregationProjection(aggregates, aggregates, paramBinder, AggregateMode.PARTIAL_FINAL, RowGranularity.CLUSTER, plannerContext.transactionContext().sessionContext().searchPath()));
return executionPlan;
}
AggregationProjection fullAggregation = projectionBuilder.aggregationProjection(sourceOutputs, aggregates, paramBinder, AggregateMode.ITER_FINAL, RowGranularity.CLUSTER, plannerContext.transactionContext().sessionContext().searchPath());
executionPlan.addProjection(fullAggregation);
return executionPlan;
}
AggregationProjection toPartial = projectionBuilder.aggregationProjection(sourceOutputs, aggregates, paramBinder, AggregateMode.ITER_PARTIAL, source.preferShardProjections() ? RowGranularity.SHARD : RowGranularity.NODE, plannerContext.transactionContext().sessionContext().searchPath());
executionPlan.addProjection(toPartial);
AggregationProjection toFinal = projectionBuilder.aggregationProjection(aggregates, aggregates, paramBinder, AggregateMode.PARTIAL_FINAL, RowGranularity.CLUSTER, plannerContext.transactionContext().sessionContext().searchPath());
return new Merge(executionPlan, new MergePhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), MERGE_PHASE_NAME, executionPlan.resultDescription().nodeIds().size(), 1, Collections.singletonList(plannerContext.handlerNode()), executionPlan.resultDescription().streamOutputs(), Collections.singletonList(toFinal), DistributionInfo.DEFAULT_BROADCAST, null), LogicalPlanner.NO_LIMIT, 0, aggregates.size(), 1, null);
}
use of io.crate.planner.ExecutionPlan in project crate by crate.
the class GroupHashAggregate method build.
@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> hints, ProjectionBuilder projectionBuilder, int limit, int offset, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
if (hints.contains(PlanHint.PREFER_SOURCE_LOOKUP)) {
hints = new HashSet<>(hints);
hints.remove(PlanHint.PREFER_SOURCE_LOOKUP);
}
ExecutionPlan executionPlan = source.build(plannerContext, hints, projectionBuilder, NO_LIMIT, 0, null, null, params, subQueryResults);
if (executionPlan.resultDescription().hasRemainingLimitOrOffset()) {
executionPlan = Merge.ensureOnHandler(executionPlan, plannerContext);
}
SubQueryAndParamBinder paramBinder = new SubQueryAndParamBinder(params, subQueryResults);
List<Symbol> sourceOutputs = source.outputs();
if (shardsContainAllGroupKeyValues()) {
GroupProjection groupProjection = projectionBuilder.groupProjection(sourceOutputs, groupKeys, aggregates, paramBinder, AggregateMode.ITER_FINAL, source.preferShardProjections() ? RowGranularity.SHARD : RowGranularity.CLUSTER, plannerContext.transactionContext().sessionContext().searchPath());
executionPlan.addProjection(groupProjection, TopN.NO_LIMIT, 0, null);
return executionPlan;
}
if (ExecutionPhases.executesOnHandler(plannerContext.handlerNode(), executionPlan.resultDescription().nodeIds())) {
if (source.preferShardProjections()) {
executionPlan.addProjection(projectionBuilder.groupProjection(sourceOutputs, groupKeys, aggregates, paramBinder, AggregateMode.ITER_PARTIAL, RowGranularity.SHARD, plannerContext.transactionContext().sessionContext().searchPath()));
executionPlan.addProjection(projectionBuilder.groupProjection(outputs, groupKeys, aggregates, paramBinder, AggregateMode.PARTIAL_FINAL, RowGranularity.NODE, plannerContext.transactionContext().sessionContext().searchPath()), TopN.NO_LIMIT, 0, null);
return executionPlan;
} else {
executionPlan.addProjection(projectionBuilder.groupProjection(sourceOutputs, groupKeys, aggregates, paramBinder, AggregateMode.ITER_FINAL, RowGranularity.NODE, plannerContext.transactionContext().sessionContext().searchPath()), TopN.NO_LIMIT, 0, null);
return executionPlan;
}
}
GroupProjection toPartial = projectionBuilder.groupProjection(sourceOutputs, groupKeys, aggregates, paramBinder, AggregateMode.ITER_PARTIAL, source.preferShardProjections() ? RowGranularity.SHARD : RowGranularity.NODE, plannerContext.transactionContext().sessionContext().searchPath());
executionPlan.addProjection(toPartial);
executionPlan.setDistributionInfo(DistributionInfo.DEFAULT_MODULO);
GroupProjection toFinal = projectionBuilder.groupProjection(outputs, groupKeys, aggregates, paramBinder, AggregateMode.PARTIAL_FINAL, RowGranularity.CLUSTER, plannerContext.transactionContext().sessionContext().searchPath());
return createMerge(plannerContext, executionPlan, Collections.singletonList(toFinal), executionPlan.resultDescription().nodeIds());
}
use of io.crate.planner.ExecutionPlan in project crate by crate.
the class HashJoin method build.
@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> hints, ProjectionBuilder projectionBuilder, int limit, int offset, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
ExecutionPlan leftExecutionPlan = lhs.build(plannerContext, hints, projectionBuilder, NO_LIMIT, 0, null, null, params, subQueryResults);
ExecutionPlan rightExecutionPlan = rhs.build(plannerContext, hints, projectionBuilder, NO_LIMIT, 0, null, null, params, subQueryResults);
LogicalPlan leftLogicalPlan = lhs;
LogicalPlan rightLogicalPlan = rhs;
boolean tablesSwitched = false;
// revealed that this improves performance in most cases.
if (lhs.numExpectedRows() < rhs.numExpectedRows()) {
tablesSwitched = true;
leftLogicalPlan = rhs;
rightLogicalPlan = lhs;
ExecutionPlan tmp = leftExecutionPlan;
leftExecutionPlan = rightExecutionPlan;
rightExecutionPlan = tmp;
}
SubQueryAndParamBinder paramBinder = new SubQueryAndParamBinder(params, subQueryResults);
Tuple<List<Symbol>, List<Symbol>> hashSymbols = extractHashJoinSymbolsFromJoinSymbolsAndSplitPerSide(tablesSwitched);
ResultDescription leftResultDesc = leftExecutionPlan.resultDescription();
ResultDescription rightResultDesc = rightExecutionPlan.resultDescription();
Collection<String> joinExecutionNodes = leftResultDesc.nodeIds();
List<Symbol> leftOutputs = leftLogicalPlan.outputs();
List<Symbol> rightOutputs = rightLogicalPlan.outputs();
MergePhase leftMerge = null;
MergePhase rightMerge = null;
// We can only run the join distributed if no remaining limit or offset must be applied on the source relations.
// Because on distributed joins, every join is running on a slice (modulo) set of the data and so no limit/offset
// could be applied. Limit/offset can only be applied on the whole data set after all partial rows from the
// shards are merged
boolean isDistributed = leftResultDesc.hasRemainingLimitOrOffset() == false && rightResultDesc.hasRemainingLimitOrOffset() == false;
if (joinExecutionNodes.isEmpty()) {
// The left source might have zero execution nodes, for example in the case of `sys.shards` without any tables
// If the join then also uses zero execution nodes, a distributed plan no longer works because
// the source operators wouldn't have a downstream node where they can send the results to.
// → we switch to non-distributed which results in the join running on the handlerNode.
isDistributed = false;
}
if (joinExecutionNodes.size() == 1 && joinExecutionNodes.equals(rightResultDesc.nodeIds()) && !rightResultDesc.hasRemainingLimitOrOffset()) {
// If the left and the right plan are executed on the same single node the mergePhase
// should be omitted. This is the case if the left and right table have only one shards which
// are on the same node
leftExecutionPlan.setDistributionInfo(DistributionInfo.DEFAULT_SAME_NODE);
rightExecutionPlan.setDistributionInfo(DistributionInfo.DEFAULT_SAME_NODE);
} else {
if (isDistributed) {
// Run the join distributed by modulo distribution algorithm
leftOutputs = setModuloDistribution(Lists2.map(hashSymbols.v1(), paramBinder), leftLogicalPlan.outputs(), leftExecutionPlan);
rightOutputs = setModuloDistribution(Lists2.map(hashSymbols.v2(), paramBinder), rightLogicalPlan.outputs(), rightExecutionPlan);
} else {
// Run the join non-distributed on the handler node
joinExecutionNodes = Collections.singletonList(plannerContext.handlerNode());
leftExecutionPlan.setDistributionInfo(DistributionInfo.DEFAULT_BROADCAST);
rightExecutionPlan.setDistributionInfo(DistributionInfo.DEFAULT_BROADCAST);
}
leftMerge = JoinOperations.buildMergePhaseForJoin(plannerContext, leftResultDesc, joinExecutionNodes);
rightMerge = JoinOperations.buildMergePhaseForJoin(plannerContext, rightResultDesc, joinExecutionNodes);
}
List<Symbol> joinOutputs = Lists2.concat(leftOutputs, rightOutputs);
HashJoinPhase joinPhase = new HashJoinPhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), "hash-join", Collections.singletonList(JoinOperations.createJoinProjection(outputs, joinOutputs)), leftMerge, rightMerge, leftOutputs.size(), rightOutputs.size(), joinExecutionNodes, InputColumns.create(paramBinder.apply(joinCondition), joinOutputs), InputColumns.create(Lists2.map(hashSymbols.v1(), paramBinder), new InputColumns.SourceSymbols(leftOutputs)), InputColumns.create(Lists2.map(hashSymbols.v2(), paramBinder), new InputColumns.SourceSymbols(rightOutputs)), Symbols.typeView(leftOutputs), leftLogicalPlan.estimatedRowSize(), leftLogicalPlan.numExpectedRows());
return new Join(joinPhase, leftExecutionPlan, rightExecutionPlan, TopN.NO_LIMIT, 0, TopN.NO_LIMIT, outputs.size(), null);
}
use of io.crate.planner.ExecutionPlan in project crate by crate.
the class NestedLoopJoin method build.
@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> hints, ProjectionBuilder projectionBuilder, int limit, int offset, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
/*
* Benchmarks reveal that if rows are filtered out distributed execution gives better performance.
* Therefore if `filterNeeded` is true (there is joinCondition or a filtering after the join operation)
* then it's a good indication that distributed execution will be faster.
*
* We may at some point add some kind of session-settings to override this behaviour
* or otherwise come up with a better heuristic.
*/
Integer childPageSizeHint = !isFiltered && limit != TopN.NO_LIMIT ? limitAndOffset(limit, offset) : null;
ExecutionPlan left = lhs.build(plannerContext, hints, projectionBuilder, NO_LIMIT, 0, null, childPageSizeHint, params, subQueryResults);
ExecutionPlan right = rhs.build(plannerContext, hints, projectionBuilder, NO_LIMIT, 0, null, childPageSizeHint, params, subQueryResults);
PositionalOrderBy orderByFromLeft = left.resultDescription().orderBy();
boolean hasDocTables = baseTables.stream().anyMatch(r -> r instanceof DocTableRelation);
boolean isDistributed = hasDocTables && isFiltered && !joinType.isOuter();
LogicalPlan leftLogicalPlan = lhs;
LogicalPlan rightLogicalPlan = rhs;
isDistributed = isDistributed && (!left.resultDescription().nodeIds().isEmpty() && !right.resultDescription().nodeIds().isEmpty());
boolean blockNlPossible = !isDistributed && isBlockNlPossible(left, right);
JoinType joinType = this.joinType;
if (!orderByWasPushedDown && joinType.supportsInversion() && (isDistributed && lhs.numExpectedRows() < rhs.numExpectedRows() && orderByFromLeft == null) || (blockNlPossible && lhs.numExpectedRows() > rhs.numExpectedRows())) {
// 1) The right side is always broadcast-ed, so for performance reasons we switch the tables so that
// the right table is the smaller (numOfRows). If left relation has a pushed-down OrderBy that needs
// to be preserved, then the switch is not possible.
// 2) For block nested loop, the left side should always be smaller. Benchmarks have shown that the
// performance decreases if the left side is much larger and no limit is applied.
ExecutionPlan tmpExecutionPlan = left;
left = right;
right = tmpExecutionPlan;
leftLogicalPlan = rhs;
rightLogicalPlan = lhs;
joinType = joinType.invert();
}
Tuple<Collection<String>, List<MergePhase>> joinExecutionNodesAndMergePhases = configureExecution(left, right, plannerContext, isDistributed);
List<Symbol> joinOutputs = Lists2.concat(leftLogicalPlan.outputs(), rightLogicalPlan.outputs());
SubQueryAndParamBinder paramBinder = new SubQueryAndParamBinder(params, subQueryResults);
Symbol joinInput = null;
if (joinCondition != null) {
joinInput = InputColumns.create(paramBinder.apply(joinCondition), joinOutputs);
}
NestedLoopPhase nlPhase = new NestedLoopPhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), isDistributed ? "distributed-nested-loop" : "nested-loop", Collections.singletonList(JoinOperations.createJoinProjection(outputs, joinOutputs)), joinExecutionNodesAndMergePhases.v2().get(0), joinExecutionNodesAndMergePhases.v2().get(1), leftLogicalPlan.outputs().size(), rightLogicalPlan.outputs().size(), joinExecutionNodesAndMergePhases.v1(), joinType, joinInput, Symbols.typeView(leftLogicalPlan.outputs()), leftLogicalPlan.estimatedRowSize(), leftLogicalPlan.numExpectedRows(), blockNlPossible);
return new Join(nlPhase, left, right, TopN.NO_LIMIT, 0, TopN.NO_LIMIT, outputs.size(), orderByFromLeft);
}
use of io.crate.planner.ExecutionPlan in project crate by crate.
the class Get method build.
@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> hints, ProjectionBuilder projectionBuilder, int limitHint, int offsetHint, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
HashMap<String, Map<ShardId, List<PKAndVersion>>> idsByShardByNode = new HashMap<>();
DocTableInfo docTableInfo = tableRelation.tableInfo();
for (DocKeys.DocKey docKey : docKeys) {
String id = docKey.getId(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
if (id == null) {
continue;
}
List<String> partitionValues = docKey.getPartitionValues(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
String indexName = indexName(docTableInfo, partitionValues);
String routing = docKey.getRouting(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
ShardRouting shardRouting;
try {
shardRouting = plannerContext.resolveShard(indexName, id, routing);
} catch (IndexNotFoundException e) {
if (docTableInfo.isPartitioned()) {
continue;
}
throw e;
}
String currentNodeId = shardRouting.currentNodeId();
if (currentNodeId == null) {
// If relocating is fast enough this will work, otherwise it will result in a shard failure which
// will cause a statement retry
currentNodeId = shardRouting.relocatingNodeId();
if (currentNodeId == null) {
throw new ShardNotFoundException(shardRouting.shardId());
}
}
Map<ShardId, List<PKAndVersion>> idsByShard = idsByShardByNode.get(currentNodeId);
if (idsByShard == null) {
idsByShard = new HashMap<>();
idsByShardByNode.put(currentNodeId, idsByShard);
}
List<PKAndVersion> pkAndVersions = idsByShard.get(shardRouting.shardId());
if (pkAndVersions == null) {
pkAndVersions = new ArrayList<>();
idsByShard.put(shardRouting.shardId(), pkAndVersions);
}
long version = docKey.version(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(Versions.MATCH_ANY);
long sequenceNumber = docKey.sequenceNo(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(SequenceNumbers.UNASSIGNED_SEQ_NO);
long primaryTerm = docKey.primaryTerm(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(SequenceNumbers.UNASSIGNED_PRIMARY_TERM);
pkAndVersions.add(new PKAndVersion(id, version, sequenceNumber, primaryTerm));
}
var docKeyColumns = new ArrayList<>(docTableInfo.primaryKey());
docKeyColumns.addAll(docTableInfo.partitionedBy());
docKeyColumns.add(docTableInfo.clusteredBy());
docKeyColumns.add(DocSysColumns.VERSION);
docKeyColumns.add(DocSysColumns.SEQ_NO);
docKeyColumns.add(DocSysColumns.PRIMARY_TERM);
var binder = new SubQueryAndParamBinder(params, subQueryResults);
List<Symbol> boundOutputs = Lists2.map(outputs, binder);
var boundQuery = binder.apply(query);
// Collect all columns which are used inside the query
// If the query contains only DocKeys, no filter is needed as all DocKeys are handled by the PKLookupOperation
AtomicBoolean requiresAdditionalFilteringOnNonDocKeyColumns = new AtomicBoolean(false);
var toCollectSet = new LinkedHashSet<>(boundOutputs);
Consumer<Reference> addRefIfMatch = ref -> {
toCollectSet.add(ref);
if (docKeyColumns.contains(ref.column()) == false) {
requiresAdditionalFilteringOnNonDocKeyColumns.set(true);
}
};
RefVisitor.visitRefs(boundQuery, addRefIfMatch);
var toCollect = boundOutputs;
ArrayList<Projection> projections = new ArrayList<>();
if (requiresAdditionalFilteringOnNonDocKeyColumns.get()) {
toCollect = List.copyOf(toCollectSet);
var filterProjection = ProjectionBuilder.filterProjection(toCollect, boundQuery);
filterProjection.requiredGranularity(RowGranularity.SHARD);
projections.add(filterProjection);
// reduce outputs which have been added for the filter projection
var evalProjection = new EvalProjection(InputColumn.mapToInputColumns(boundOutputs), RowGranularity.SHARD);
projections.add(evalProjection);
}
var collect = new Collect(new PKLookupPhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), docTableInfo.partitionedBy(), toCollect, idsByShardByNode), TopN.NO_LIMIT, 0, toCollect.size(), docKeys.size(), null);
for (var projection : projections) {
collect.addProjection(projection);
}
return collect;
}
Aggregations