use of io.prestosql.spi.plan.CTEScanNode in project hetu-core by openlookeng.
the class RelationPlanner method visitTable.
@Override
protected RelationPlan visitTable(Table node, Void context) {
Scope scope = analysis.getScope(node);
Query namedQuery = analysis.getNamedQuery(node);
if (namedQuery != null) {
RelationPlan subPlan = process(namedQuery, null);
// Add implicit coercions if view query produces types that don't match the declared output types
// of the view (e.g., if the underlying tables referenced by the view changed)
Type[] types = scope.getRelationType().getAllFields().stream().map(Field::getType).toArray(Type[]::new);
RelationPlan withCoercions = addCoercions(subPlan, types);
if ((!isCTEReuseEnabled(session) || getExecutionPolicy(session).equals("phased")) || (getCteMaxQueueSize(session) < getTaskConcurrency(session) * 2) || !(analysis.getStatement() instanceof Query) || !((Query) analysis.getStatement()).getWith().isPresent()) {
if (getCteMaxQueueSize(session) < getTaskConcurrency(session) * 2) {
LOG.info("Main queue size " + getCteMaxQueueSize(session) + "should be more than 2 times of concurrent task " + getTaskConcurrency(session));
}
return new RelationPlan(withCoercions.getRoot(), scope, withCoercions.getFieldMappings());
}
Integer commonCTERefNum;
if (namedSubPlan.containsKey(node.getName())) {
commonCTERefNum = namedSubPlan.get(node.getName());
} else {
commonCTERefNum = uniqueIdAllocator.getNextId();
namedSubPlan.put(node.getName(), commonCTERefNum);
}
PlanNode cteNode = new CTEScanNode(idAllocator.getNextId(), withCoercions.getRoot(), withCoercions.getFieldMappings(), Optional.empty(), node.getName().toString(), new HashSet<>(), commonCTERefNum);
subPlan = new RelationPlan(cteNode, scope, withCoercions.getFieldMappings());
return subPlan;
}
TableHandle handle = analysis.getTableHandle(node);
ImmutableList.Builder<Symbol> outputSymbolsBuilder = ImmutableList.builder();
ImmutableMap.Builder<Symbol, ColumnHandle> columns = ImmutableMap.builder();
for (Field field : scope.getRelationType().getAllFields()) {
Symbol symbol = planSymbolAllocator.newSymbol(field.getName().get(), field.getType());
outputSymbolsBuilder.add(symbol);
columns.put(symbol, analysis.getColumn(field));
}
List<Symbol> outputSymbols = outputSymbolsBuilder.build();
boolean isDeleteTarget = analysis.isDeleteTarget(createQualifiedObjectName(session, node, node.getName()));
PlanNode root = TableScanNode.newInstance(idAllocator.getNextId(), handle, outputSymbols, columns.build(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, isDeleteTarget);
RelationPlan tableScan = new RelationPlan(root, scope, outputSymbols);
tableScan = addRowFilters(node, tableScan);
tableScan = addColumnMasks(node, tableScan);
return tableScan;
}
use of io.prestosql.spi.plan.CTEScanNode in project hetu-core by openlookeng.
the class TransformUnCorrelatedInPredicateSubQuerySelfJoinToAggregate method transformProjectNode.
private Optional<ProjectNode> transformProjectNode(Context context, ProjectNode projectNode) {
// IN predicate requires only one projection
if (projectNode.getOutputSymbols().size() > 1) {
return Optional.empty();
}
PlanNode source = context.getLookup().resolve(projectNode.getSource());
if (source instanceof CTEScanNode) {
source = getChildFilterNode(context, context.getLookup().resolve(((CTEScanNode) source).getSource()));
}
if (!(source instanceof FilterNode && context.getLookup().resolve(((FilterNode) source).getSource()) instanceof JoinNode)) {
return Optional.empty();
}
FilterNode filter = (FilterNode) source;
Expression predicate = OriginalExpressionUtils.castToExpression(filter.getPredicate());
List<SymbolReference> allPredicateSymbols = new ArrayList<>();
getAllSymbols(predicate, allPredicateSymbols);
JoinNode joinNode = (JoinNode) context.getLookup().resolve(((FilterNode) source).getSource());
if (!isSelfJoin(projectNode, predicate, joinNode, context.getLookup())) {
// Check next level for Self Join
PlanNode left = context.getLookup().resolve(joinNode.getLeft());
boolean changed = false;
if (left instanceof ProjectNode) {
Optional<ProjectNode> transformResult = transformProjectNode(context, (ProjectNode) left);
if (transformResult.isPresent()) {
joinNode = new JoinNode(joinNode.getId(), joinNode.getType(), transformResult.get(), joinNode.getRight(), joinNode.getCriteria(), joinNode.getOutputSymbols(), joinNode.getFilter(), joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType(), joinNode.isSpillable(), joinNode.getDynamicFilters());
changed = true;
}
}
PlanNode right = context.getLookup().resolve(joinNode.getRight());
if (right instanceof ProjectNode) {
Optional<ProjectNode> transformResult = transformProjectNode(context, (ProjectNode) right);
if (transformResult.isPresent()) {
joinNode = new JoinNode(joinNode.getId(), joinNode.getType(), joinNode.getLeft(), transformResult.get(), joinNode.getCriteria(), joinNode.getOutputSymbols(), joinNode.getFilter(), joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType(), joinNode.isSpillable(), joinNode.getDynamicFilters());
changed = true;
}
}
if (changed) {
FilterNode transformedFilter = new FilterNode(filter.getId(), joinNode, filter.getPredicate());
ProjectNode transformedProject = new ProjectNode(projectNode.getId(), transformedFilter, projectNode.getAssignments());
return Optional.of(transformedProject);
}
return Optional.empty();
}
// Choose the table to use based on projected output.
TableScanNode leftTable = (TableScanNode) context.getLookup().resolve(joinNode.getLeft());
TableScanNode rightTable = (TableScanNode) context.getLookup().resolve(joinNode.getRight());
TableScanNode tableToUse;
List<RowExpression> aggregationSymbols;
AggregationNode.GroupingSetDescriptor groupingSetDescriptor;
Assignments projectionsForCTE = null;
Assignments projectionsFromFilter = null;
// Use non-projected column for aggregation
if (context.getLookup().resolve(projectNode.getSource()) instanceof CTEScanNode) {
CTEScanNode cteScanNode = (CTEScanNode) context.getLookup().resolve(projectNode.getSource());
ProjectNode childProjectOfCte = (ProjectNode) context.getLookup().resolve(cteScanNode.getSource());
List<Symbol> completeOutputSymbols = new ArrayList<>();
rightTable.getOutputSymbols().forEach((s) -> completeOutputSymbols.add(s));
leftTable.getOutputSymbols().forEach((s) -> completeOutputSymbols.add(s));
List<Symbol> outputSymbols = new ArrayList<>();
for (int i = 0; i < completeOutputSymbols.size(); i++) {
Symbol outputSymbol = completeOutputSymbols.get(i);
for (Symbol symbol : projectNode.getOutputSymbols()) {
if (childProjectOfCte.getAssignments().getMap().containsKey(symbol)) {
if (((SymbolReference) OriginalExpressionUtils.castToExpression(childProjectOfCte.getAssignments().getMap().get(symbol))).getName().equals(outputSymbol.getName())) {
outputSymbols.add(outputSymbol);
}
}
}
}
Map<Symbol, RowExpression> projectionsForCTEMap = new HashMap<>();
Map<Symbol, RowExpression> projectionsFromFilterMap = new HashMap<>();
for (Map.Entry entry : childProjectOfCte.getAssignments().getMap().entrySet()) {
if (entry.getKey().equals(getOnlyElement(projectNode.getOutputSymbols()))) {
projectionsForCTEMap.put((Symbol) entry.getKey(), (RowExpression) entry.getValue());
}
if (entry.getKey().equals(getOnlyElement(projectNode.getOutputSymbols()))) {
projectionsFromFilterMap.put(getOnlyElement(outputSymbols), (RowExpression) entry.getValue());
}
}
projectionsForCTE = new Assignments(projectionsForCTEMap);
projectionsFromFilter = new Assignments(projectionsFromFilterMap);
tableToUse = leftTable.getOutputSymbols().contains(getOnlyElement(outputSymbols)) ? leftTable : rightTable;
aggregationSymbols = allPredicateSymbols.stream().filter(s -> tableToUse.getOutputSymbols().contains(SymbolUtils.from(s))).filter(s -> !outputSymbols.contains(SymbolUtils.from(s))).map(OriginalExpressionUtils::castToRowExpression).collect(Collectors.toList());
// Create aggregation
groupingSetDescriptor = new AggregationNode.GroupingSetDescriptor(ImmutableList.copyOf(outputSymbols), 1, ImmutableSet.of());
} else {
tableToUse = leftTable.getOutputSymbols().contains(getOnlyElement(projectNode.getOutputSymbols())) ? leftTable : rightTable;
aggregationSymbols = allPredicateSymbols.stream().filter(s -> tableToUse.getOutputSymbols().contains(SymbolUtils.from(s))).filter(s -> !projectNode.getOutputSymbols().contains(SymbolUtils.from(s))).map(OriginalExpressionUtils::castToRowExpression).collect(Collectors.toList());
// Create aggregation
groupingSetDescriptor = new AggregationNode.GroupingSetDescriptor(ImmutableList.copyOf(projectNode.getOutputSymbols()), 1, ImmutableSet.of());
}
AggregationNode.Aggregation aggregation = new AggregationNode.Aggregation(new CallExpression("count", functionResolution.countFunction(), BIGINT, aggregationSymbols), aggregationSymbols, // mark DISTINCT since NOT_EQUALS predicate
true, Optional.empty(), Optional.empty(), Optional.empty());
ImmutableMap.Builder<Symbol, AggregationNode.Aggregation> aggregationsBuilder = ImmutableMap.builder();
Symbol countSymbol = context.getSymbolAllocator().newSymbol(aggregation.getFunctionCall().getDisplayName(), BIGINT);
aggregationsBuilder.put(countSymbol, aggregation);
AggregationNode aggregationNode = new AggregationNode(context.getIdAllocator().getNextId(), tableToUse, aggregationsBuilder.build(), groupingSetDescriptor, ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty(), AggregationNode.AggregationType.HASH, Optional.empty());
// Filter rows with count < 1 from aggregation results to match the NOT_EQUALS clause in original query.
FilterNode filterNode = new FilterNode(context.getIdAllocator().getNextId(), aggregationNode, OriginalExpressionUtils.castToRowExpression(new ComparisonExpression(ComparisonExpression.Operator.GREATER_THAN, SymbolUtils.toSymbolReference(countSymbol), new GenericLiteral("BIGINT", "1"))));
// Project the aggregated+filtered rows.
ProjectNode transformedSubquery = new ProjectNode(projectNode.getId(), filterNode, projectNode.getAssignments());
if (context.getLookup().resolve(projectNode.getSource()) instanceof CTEScanNode) {
CTEScanNode cteScanNode = (CTEScanNode) context.getLookup().resolve(projectNode.getSource());
PlanNode projectNodeForCTE = context.getLookup().resolve(cteScanNode.getSource());
PlanNode projectNodeFromFilter = context.getLookup().resolve(projectNodeForCTE);
projectNodeFromFilter = new ProjectNode(projectNodeFromFilter.getId(), filterNode, projectionsFromFilter);
projectNodeForCTE = new ProjectNode(projectNodeForCTE.getId(), projectNodeFromFilter, projectionsForCTE);
cteScanNode = (CTEScanNode) cteScanNode.replaceChildren(ImmutableList.of(projectNodeForCTE));
cteScanNode.setOutputSymbols(projectNode.getOutputSymbols());
transformedSubquery = new ProjectNode(projectNode.getId(), cteScanNode, projectNode.getAssignments());
}
return Optional.of(transformedSubquery);
}
use of io.prestosql.spi.plan.CTEScanNode in project hetu-core by openlookeng.
the class TransformUnCorrelatedInPredicateSubQuerySelfJoinToAggregate method isSelfJoin.
private static boolean isSelfJoin(ProjectNode projectNode, Expression predicate, JoinNode joinNode, Lookup lookup) {
PlanNode left = lookup.resolve(joinNode.getLeft());
PlanNode right = lookup.resolve(joinNode.getRight());
// TODO FIX FOR framenwork changes
if (joinNode.getType() == JoinNode.Type.INNER && left instanceof TableScanNode && right instanceof TableScanNode && ((TableScanNode) left).getTable().getFullyQualifiedName().equals(((TableScanNode) right).getTable().getFullyQualifiedName())) {
if (!(predicate instanceof LogicalBinaryExpression && ((LogicalBinaryExpression) predicate).getLeft() instanceof ComparisonExpression && ((LogicalBinaryExpression) predicate).getRight() instanceof ComparisonExpression)) {
return false;
}
SymbolReference projected = SymbolUtils.toSymbolReference(getOnlyElement(projectNode.getOutputSymbols()));
ComparisonExpression leftPredicate = (ComparisonExpression) ((LogicalBinaryExpression) predicate).getLeft();
ComparisonExpression rightPredicate = (ComparisonExpression) ((LogicalBinaryExpression) predicate).getRight();
if (lookup.resolve(projectNode.getSource()) instanceof CTEScanNode) {
CTEScanNode cteScanNode = (CTEScanNode) lookup.resolve(projectNode.getSource());
ProjectNode childProjectOfCte = (ProjectNode) lookup.resolve(cteScanNode.getSource());
RowExpression projectedExpresssion = (childProjectOfCte.getAssignments().get(getOnlyElement(projectNode.getOutputSymbols())));
for (Symbol symbol : lookup.resolve(childProjectOfCte.getSource()).getOutputSymbols()) {
if (symbol.getName().equals(((SymbolReference) OriginalExpressionUtils.castToExpression(projectedExpresssion)).getName())) {
projected = SymbolUtils.toSymbolReference(symbol);
}
}
}
if (leftPredicate.getChildren().contains(projected) && leftPredicate.getOperator() == ComparisonExpression.Operator.EQUAL && rightPredicate.getOperator() == ComparisonExpression.Operator.NOT_EQUAL) {
return true;
} else if (rightPredicate.getChildren().contains(projected) && rightPredicate.getOperator() == ComparisonExpression.Operator.EQUAL && leftPredicate.getOperator() == ComparisonExpression.Operator.NOT_EQUAL) {
return true;
}
}
return false;
}
use of io.prestosql.spi.plan.CTEScanNode in project hetu-core by openlookeng.
the class PushPartialAggregationThroughExchange method apply.
@Override
public Result apply(AggregationNode aggregationNode, Captures captures, Context context) {
ExchangeNode exchangeNode = captures.get(EXCHANGE_NODE);
// Do not push down aggregation since CTEScanNode requires an exchange node above it
if (exchangeNode.getSources().size() == 1 && context.getLookup().resolve(exchangeNode.getSources().get(0)) instanceof CTEScanNode) {
return Result.empty();
}
boolean decomposable = isDecomposable(aggregationNode, metadata);
if (aggregationNode.getStep().equals(SINGLE) && aggregationNode.hasEmptyGroupingSet() && aggregationNode.hasNonEmptyGroupingSet() && exchangeNode.getType() == REPARTITION) {
// single-step aggregation w/ empty grouping sets in a partitioned stage, so we need a partial that will produce
// the default intermediates for the empty grouping set that will be routed to the appropriate final aggregation.
// TODO: technically, AddExchanges generates a broken plan that this rule "fixes"
checkState(decomposable, "Distributed aggregation with empty grouping set requires partial but functions are not decomposable");
return Result.ofPlanNode(split(aggregationNode, context));
}
if (!decomposable || !preferPartialAggregation(context.getSession())) {
return Result.empty();
}
// the cardinality of the stream (i.e., gather or repartition)
if ((exchangeNode.getType() != GATHER && exchangeNode.getType() != REPARTITION) || exchangeNode.getPartitioningScheme().isReplicateNullsAndAny()) {
return Result.empty();
}
if (exchangeNode.getType() == REPARTITION) {
// if partitioning columns are not a subset of grouping keys,
// we can't push this through
List<Symbol> partitioningColumns = exchangeNode.getPartitioningScheme().getPartitioning().getArguments().stream().filter(Partitioning.ArgumentBinding::isVariable).map(Partitioning.ArgumentBinding::getColumn).collect(Collectors.toList());
if (!aggregationNode.getGroupingKeys().containsAll(partitioningColumns)) {
return Result.empty();
}
}
// currently, we only support plans that don't use pre-computed hash functions
if (aggregationNode.getHashSymbol().isPresent() || exchangeNode.getPartitioningScheme().getHashColumn().isPresent()) {
return Result.empty();
}
switch(aggregationNode.getStep()) {
case SINGLE:
// Split it into a FINAL on top of a PARTIAL and
return Result.ofPlanNode(split(aggregationNode, context));
case PARTIAL:
// Push it underneath each branch of the exchange
return Result.ofPlanNode(pushPartial(aggregationNode, exchangeNode, context));
default:
return Result.empty();
}
}
use of io.prestosql.spi.plan.CTEScanNode in project hetu-core by openlookeng.
the class ExchangeNode method mergingExchange.
public static ExchangeNode mergingExchange(PlanNodeId id, Scope scope, PlanNode child, OrderingScheme orderingScheme) {
// CTEScanNode adds one exchange node on top of it,
// so if upper node going to have another ExchangeNode then we should omit previous one.
PlanNode childNode = child;
if (scope == REMOTE && childNode instanceof ExchangeNode && childNode.getSources().size() == 1 && childNode.getSources().get(0) instanceof CTEScanNode) {
childNode = childNode.getSources().get(0);
}
PartitioningHandle partitioningHandle = scope == LOCAL ? FIXED_PASSTHROUGH_DISTRIBUTION : SINGLE_DISTRIBUTION;
return new ExchangeNode(id, Type.GATHER, scope, new PartitioningScheme(Partitioning.create(partitioningHandle, ImmutableList.of()), childNode.getOutputSymbols()), ImmutableList.of(childNode), ImmutableList.of(childNode.getOutputSymbols()), Optional.of(orderingScheme), AggregationNode.AggregationType.HASH);
}
Aggregations