use of io.prestosql.spi.plan.ProjectNode in project hetu-core by openlookeng.
the class QueryPlanner method aggregate.
private PlanBuilder aggregate(PlanBuilder inputSubPlan, QuerySpecification node) {
PlanBuilder subPlan = inputSubPlan;
if (!analysis.isAggregation(node)) {
return subPlan;
}
// 1. Pre-project all scalar inputs (arguments and non-trivial group by expressions)
Set<Expression> groupByExpressions = ImmutableSet.copyOf(analysis.getGroupByExpressions(node));
ImmutableList.Builder<Expression> arguments = ImmutableList.builder();
analysis.getAggregates(node).stream().map(FunctionCall::getArguments).flatMap(List::stream).filter(// lambda expression is generated at execution time
exp -> !(exp instanceof LambdaExpression)).forEach(arguments::add);
analysis.getAggregates(node).stream().map(FunctionCall::getOrderBy).filter(Optional::isPresent).map(Optional::get).map(OrderBy::getSortItems).flatMap(List::stream).map(SortItem::getSortKey).forEach(arguments::add);
// filter expressions need to be projected first
analysis.getAggregates(node).stream().map(FunctionCall::getFilter).filter(Optional::isPresent).map(Optional::get).forEach(arguments::add);
Iterable<Expression> inputs = Iterables.concat(groupByExpressions, arguments.build());
subPlan = handleSubqueries(subPlan, node, inputs);
if (!Iterables.isEmpty(inputs)) {
// avoid an empty projection if the only aggregation is COUNT (which has no arguments)
subPlan = project(subPlan, inputs);
}
// 2. Aggregate
// 2.a. Rewrite aggregate arguments
TranslationMap argumentTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToSymbolMap);
ImmutableList.Builder<Symbol> aggregationArgumentsBuilder = ImmutableList.builder();
for (Expression argument : arguments.build()) {
Symbol symbol = subPlan.translate(argument);
argumentTranslations.put(argument, symbol);
aggregationArgumentsBuilder.add(symbol);
}
List<Symbol> aggregationArguments = aggregationArgumentsBuilder.build();
// 2.b. Rewrite grouping columns
TranslationMap groupingTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToSymbolMap);
Map<Symbol, Symbol> groupingSetMappings = new LinkedHashMap<>();
for (Expression expression : groupByExpressions) {
Symbol input = subPlan.translate(expression);
Symbol output = planSymbolAllocator.newSymbol(expression, analysis.getTypeWithCoercions(expression), "gid");
groupingTranslations.put(expression, output);
groupingSetMappings.put(output, input);
}
// This tracks the grouping sets before complex expressions are considered (see comments below)
// It's also used to compute the descriptors needed to implement grouping()
List<Set<FieldId>> columnOnlyGroupingSets = ImmutableList.of(ImmutableSet.of());
List<List<Symbol>> groupingSets = ImmutableList.of(ImmutableList.of());
if (node.getGroupBy().isPresent()) {
// For the purpose of "distinct", we need to canonicalize column references that may have varying
// syntactic forms (e.g., "t.a" vs "a"). Thus we need to enumerate grouping sets based on the underlying
// fieldId associated with each column reference expression.
// The catch is that simple group-by expressions can be arbitrary expressions (this is a departure from the SQL specification).
// But, they don't affect the number of grouping sets or the behavior of "distinct" . We can compute all the candidate
// grouping sets in terms of fieldId, dedup as appropriate and then cross-join them with the complex expressions.
Analysis.GroupingSetAnalysis groupingSetAnalysis = analysis.getGroupingSets(node);
columnOnlyGroupingSets = enumerateGroupingSets(groupingSetAnalysis);
if (node.getGroupBy().get().isDistinct()) {
columnOnlyGroupingSets = columnOnlyGroupingSets.stream().distinct().collect(toImmutableList());
}
// add in the complex expressions an turn materialize the grouping sets in terms of plan columns
ImmutableList.Builder<List<Symbol>> groupingSetBuilder = ImmutableList.builder();
for (Set<FieldId> groupingSet : columnOnlyGroupingSets) {
ImmutableList.Builder<Symbol> columns = ImmutableList.builder();
groupingSetAnalysis.getComplexExpressions().stream().map(groupingTranslations::get).forEach(columns::add);
groupingSet.stream().map(field -> groupingTranslations.get(new FieldReference(field.getFieldIndex()))).forEach(columns::add);
groupingSetBuilder.add(columns.build());
}
groupingSets = groupingSetBuilder.build();
}
// 2.c. Generate GroupIdNode (multiple grouping sets) or ProjectNode (single grouping set)
Optional<Symbol> groupIdSymbol = Optional.empty();
if (groupingSets.size() > 1) {
groupIdSymbol = Optional.of(planSymbolAllocator.newSymbol("groupId", BIGINT));
GroupIdNode groupId = new GroupIdNode(idAllocator.getNextId(), subPlan.getRoot(), groupingSets, groupingSetMappings, aggregationArguments, groupIdSymbol.get());
subPlan = new PlanBuilder(groupingTranslations, groupId);
} else {
Assignments.Builder assignments = Assignments.builder();
aggregationArguments.forEach(symbol -> assignments.put(symbol, castToRowExpression(toSymbolReference(symbol))));
groupingSetMappings.forEach((key, value) -> assignments.put(key, castToRowExpression(toSymbolReference(value))));
ProjectNode project = new ProjectNode(idAllocator.getNextId(), subPlan.getRoot(), assignments.build());
subPlan = new PlanBuilder(groupingTranslations, project);
}
TranslationMap aggregationTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToSymbolMap);
aggregationTranslations.copyMappingsFrom(groupingTranslations);
// 2.d. Rewrite aggregates
ImmutableMap.Builder<Symbol, Aggregation> aggregationsBuilder = ImmutableMap.builder();
boolean needPostProjectionCoercion = false;
for (FunctionCall aggregate : analysis.getAggregates(node)) {
Expression rewritten = argumentTranslations.rewrite(aggregate);
Symbol newSymbol = planSymbolAllocator.newSymbol(rewritten, analysis.getType(aggregate));
// Therefore we can end up with this implicit cast, and have to move it into a post-projection
if (rewritten instanceof Cast) {
rewritten = ((Cast) rewritten).getExpression();
needPostProjectionCoercion = true;
}
aggregationTranslations.put(aggregate, newSymbol);
FunctionCall functionCall = (FunctionCall) rewritten;
aggregationsBuilder.put(newSymbol, new Aggregation(call(aggregate.getName().getSuffix(), analysis.getFunctionHandle(aggregate), analysis.getType(aggregate), functionCall.getArguments().stream().map(OriginalExpressionUtils::castToRowExpression).collect(toImmutableList())), functionCall.getArguments().stream().map(OriginalExpressionUtils::castToRowExpression).collect(toImmutableList()), functionCall.isDistinct(), functionCall.getFilter().map(SymbolUtils::from), functionCall.getOrderBy().map(OrderingSchemeUtils::fromOrderBy), Optional.empty()));
}
Map<Symbol, Aggregation> aggregations = aggregationsBuilder.build();
ImmutableSet.Builder<Integer> globalGroupingSets = ImmutableSet.builder();
for (int i = 0; i < groupingSets.size(); i++) {
if (groupingSets.get(i).isEmpty()) {
globalGroupingSets.add(i);
}
}
ImmutableList.Builder<Symbol> groupingKeys = ImmutableList.builder();
groupingSets.stream().flatMap(List::stream).distinct().forEach(groupingKeys::add);
groupIdSymbol.ifPresent(groupingKeys::add);
AggregationNode aggregationNode = new AggregationNode(idAllocator.getNextId(), subPlan.getRoot(), aggregations, groupingSets(groupingKeys.build(), groupingSets.size(), globalGroupingSets.build()), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), groupIdSymbol, AggregationNode.AggregationType.HASH, Optional.empty());
subPlan = new PlanBuilder(aggregationTranslations, aggregationNode);
// TODO: this is a hack, we should change type coercions to coerce the inputs to functions/operators instead of coercing the output
if (needPostProjectionCoercion) {
ImmutableList.Builder<Expression> alreadyCoerced = ImmutableList.builder();
alreadyCoerced.addAll(groupByExpressions);
groupIdSymbol.map(SymbolUtils::toSymbolReference).ifPresent(alreadyCoerced::add);
subPlan = explicitCoercionFields(subPlan, alreadyCoerced.build(), analysis.getAggregates(node));
}
// 4. Project and re-write all grouping functions
return handleGroupingOperations(subPlan, node, groupIdSymbol, columnOnlyGroupingSets);
}
use of io.prestosql.spi.plan.ProjectNode in project hetu-core by openlookeng.
the class QueryPlanner method coerce.
/**
* Creates a projection with any additional coercions by identity of the provided expressions.
*
* @return the new subplan and a mapping of each expression to the symbol representing the coercion or an existing symbol if a coercion wasn't needed
*/
public static PlanAndMappings coerce(PlanBuilder inputSubPlan, List<Expression> expressions, Analysis analysis, PlanNodeIdAllocator idAllocator, PlanSymbolAllocator planSymbolAllocator, TypeCoercion typeCoercion) {
PlanBuilder subPlan = inputSubPlan;
Assignments.Builder assignments = Assignments.builder();
assignments.putAll(AssignmentUtils.identityAsSymbolReferences(subPlan.getRoot().getOutputSymbols()));
ImmutableMap.Builder<NodeRef<Expression>, Symbol> mappings = ImmutableMap.builder();
for (Expression expression : expressions) {
Type coercion = analysis.getCoercion(expression);
if (coercion != null) {
Type type = analysis.getType(expression);
Symbol symbol = planSymbolAllocator.newSymbol(expression, coercion);
assignments.put(symbol, castToRowExpression(new Cast(subPlan.rewrite(expression), coercion.getTypeSignature().toString(), false, typeCoercion.isTypeOnlyCoercion(type, coercion))));
mappings.put(NodeRef.of(expression), symbol);
} else {
mappings.put(NodeRef.of(expression), subPlan.translate(expression));
}
}
subPlan = subPlan.withNewRoot(new ProjectNode(idAllocator.getNextId(), subPlan.getRoot(), assignments.build()));
return new PlanAndMappings(subPlan, mappings.build());
}
use of io.prestosql.spi.plan.ProjectNode in project hetu-core by openlookeng.
the class LogicalPlanner method createInsertPlan.
private RelationPlan createInsertPlan(Analysis analysis, Insert insertStatement) {
Analysis.Insert insert = analysis.getInsert().get();
TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget());
List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
RelationPlan plan = createRelationPlan(analysis, insertStatement.getQuery());
Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget());
Assignments.Builder assignments = Assignments.builder();
for (ColumnMetadata column : tableMetadata.getColumns()) {
if (column.isHidden()) {
continue;
}
Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
int index = insert.getColumns().indexOf(columns.get(column.getName()));
if (index < 0) {
Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
assignments.put(output, castToRowExpression(cast));
} else {
Symbol input = plan.getSymbol(index);
Type tableType = column.getType();
Type queryType = planSymbolAllocator.getTypes().get(input);
if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
assignments.put(output, castToRowExpression(toSymbolReference(input)));
} else {
Expression cast = noTruncationCast(toSymbolReference(input), queryType, tableType);
assignments.put(output, castToRowExpression(cast));
}
}
}
ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols());
Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, insert.getTarget());
String catalogName = insert.getTarget().getCatalogName().getCatalogName();
TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
return createTableWriterPlan(analysis, plan, new InsertReference(insert.getTarget(), analysis.isInsertOverwrite()), visibleTableColumnNames, newTableLayout, statisticsMetadata);
}
use of io.prestosql.spi.plan.ProjectNode in project hetu-core by openlookeng.
the class LogicalPlanner method createInsertCubePlan.
private RelationPlan createInsertCubePlan(Analysis analysis, InsertCube insertCubeStatement) {
Analysis.CubeInsert insert = analysis.getCubeInsert().get();
TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget());
List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
RelationPlan plan = createRelationPlan(analysis, insertCubeStatement.getQuery());
Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget());
Assignments.Builder assignments = Assignments.builder();
for (ColumnMetadata column : tableMetadata.getColumns()) {
if (column.isHidden()) {
continue;
}
Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
int index = insert.getColumns().indexOf(columns.get(column.getName()));
if (index < 0) {
Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
assignments.put(output, castToRowExpression(cast));
} else {
Symbol input = plan.getSymbol(index);
Type tableType = column.getType();
Type queryType = planSymbolAllocator.getTypes().get(input);
if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
assignments.put(output, castToRowExpression(toSymbolReference(input)));
} else {
Expression cast = noTruncationCast(toSymbolReference(input), queryType, tableType);
assignments.put(output, castToRowExpression(cast));
}
}
}
ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols());
Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, insert.getTarget());
String catalogName = insert.getTarget().getCatalogName().getCatalogName();
TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
RelationPlan tableWriterPlan = createTableWriterPlan(analysis, plan, new InsertReference(insert.getTarget(), analysis.isCubeOverwrite()), visibleTableColumnNames, newTableLayout, statisticsMetadata);
Expression rewritten = null;
Set<Identifier> predicateColumns = new HashSet<>();
if (insertCubeStatement.getWhere().isPresent()) {
rewritten = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).rewriteExpression(tableWriterPlan, insertCubeStatement.getWhere().get(), analysis, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator));
predicateColumns.addAll(ExpressionUtils.getIdentifiers(rewritten));
}
CubeMetadata cubeMetadata = insert.getMetadata();
if (!insertCubeStatement.isOverwrite() && !insertCubeStatement.getWhere().isPresent() && cubeMetadata.getCubeStatus() != CubeStatus.INACTIVE) {
// Means data some data was inserted before, but trying to insert entire dataset
throw new PrestoException(QUERY_REJECTED, "Cannot allow insert. Inserting entire dataset but cube already has partial data");
} else if (insertCubeStatement.getWhere().isPresent()) {
if (!canSupportPredicate(rewritten)) {
throw new PrestoException(QUERY_REJECTED, String.format("Cannot support predicate '%s'", ExpressionFormatter.formatExpression(rewritten, Optional.empty())));
}
if (!insertCubeStatement.isOverwrite() && arePredicatesOverlapping(rewritten, cubeMetadata)) {
throw new PrestoException(QUERY_REJECTED, String.format("Cannot allow insert. Cube already contains data for the given predicate '%s'", ExpressionFormatter.formatExpression(insertCubeStatement.getWhere().get(), Optional.empty())));
}
}
TableHandle sourceTableHandle = insert.getSourceTable();
// At this point it has been verified that source table has not been updated
// so insert into cube should be allowed
LongSupplier tableLastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, sourceTableHandle);
checkState(tableLastModifiedTimeSupplier != null, "Table last modified time is null");
Map<Symbol, Type> predicateColumnsType = predicateColumns.stream().map(identifier -> new Symbol(identifier.getValue())).collect(Collectors.toMap(Function.identity(), symbol -> planSymbolAllocator.getTypes().get(symbol), (key1, ignored) -> key1));
CubeFinishNode cubeFinishNode = new CubeFinishNode(idAllocator.getNextId(), tableWriterPlan.getRoot(), planSymbolAllocator.newSymbol("rows", BIGINT), new CubeUpdateMetadata(tableMetadata.getQualifiedName().toString(), tableLastModifiedTimeSupplier.getAsLong(), rewritten != null ? ExpressionFormatter.formatExpression(rewritten, Optional.empty()) : null, insertCubeStatement.isOverwrite()), predicateColumnsType);
return new RelationPlan(cubeFinishNode, analysis.getScope(insertCubeStatement), cubeFinishNode.getOutputSymbols());
}
use of io.prestosql.spi.plan.ProjectNode in project hetu-core by openlookeng.
the class QueryPlanner method explicitCoercionFields.
private PlanBuilder explicitCoercionFields(PlanBuilder subPlan, Iterable<Expression> alreadyCoerced, Iterable<? extends Expression> uncoerced) {
TranslationMap translations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToSymbolMap);
Assignments.Builder projections = Assignments.builder();
projections.putAll(coerce(uncoerced, subPlan, translations));
for (Expression expression : alreadyCoerced) {
if (expression instanceof SymbolReference) {
// If this is an identity projection, no need to rewrite it
// This is needed because certain synthetic identity expressions such as "group id" introduced when planning GROUPING
// don't have a corresponding analysis, so the code below doesn't work for them
projections.put(from(expression), castToRowExpression(expression));
continue;
}
Symbol symbol = planSymbolAllocator.newSymbol(expression, analysis.getType(expression));
Expression rewritten = subPlan.rewrite(expression);
projections.put(symbol, castToRowExpression(rewritten));
translations.put(expression, symbol);
}
return new PlanBuilder(translations, new ProjectNode(idAllocator.getNextId(), subPlan.getRoot(), projections.build()));
}
Aggregations