use of com.facebook.presto.sql.tree.Node in project presto by prestodb.
the class QueryPlanner method handleGroupingOperations.
private PlanBuilder handleGroupingOperations(PlanBuilder subPlan, QuerySpecification node, Optional<VariableReferenceExpression> groupIdVariable, List<Set<FieldId>> groupingSets) {
if (analysis.getGroupingOperations(node).isEmpty()) {
return subPlan;
}
TranslationMap newTranslations = subPlan.copyTranslations();
Assignments.Builder projections = Assignments.builder();
projections.putAll(identitiesAsSymbolReferences(subPlan.getRoot().getOutputVariables()));
List<Set<Integer>> descriptor = groupingSets.stream().map(set -> set.stream().map(FieldId::getFieldIndex).collect(toImmutableSet())).collect(toImmutableList());
for (GroupingOperation groupingOperation : analysis.getGroupingOperations(node)) {
Expression rewritten = GroupingOperationRewriter.rewriteGroupingOperation(groupingOperation, descriptor, analysis.getColumnReferenceFields(), groupIdVariable);
Type coercion = analysis.getCoercion(groupingOperation);
VariableReferenceExpression variable = variableAllocator.newVariable(rewritten, analysis.getTypeWithCoercions(groupingOperation));
if (coercion != null) {
rewritten = new Cast(rewritten, coercion.getTypeSignature().toString(), false, metadata.getFunctionAndTypeManager().isTypeOnlyCoercion(analysis.getType(groupingOperation), coercion));
}
projections.put(variable, castToRowExpression(rewritten));
newTranslations.put(groupingOperation, variable);
}
return new PlanBuilder(newTranslations, new ProjectNode(subPlan.getRoot().getSourceLocation(), idAllocator.getNextId(), subPlan.getRoot(), projections.build(), LOCAL));
}
use of com.facebook.presto.sql.tree.Node in project presto by prestodb.
the class QueryPlanner method aggregate.
private PlanBuilder aggregate(PlanBuilder subPlan, QuerySpecification node) {
if (!analysis.isAggregation(node)) {
return subPlan;
}
// 1. Pre-project all scalar inputs (arguments and non-trivial group by expressions)
Set<Expression> groupByExpressions = ImmutableSet.copyOf(analysis.getGroupByExpressions(node));
ImmutableList.Builder<Expression> arguments = ImmutableList.builder();
analysis.getAggregates(node).stream().map(FunctionCall::getArguments).flatMap(List::stream).filter(// lambda expression is generated at execution time
exp -> !(exp instanceof LambdaExpression)).forEach(arguments::add);
analysis.getAggregates(node).stream().map(FunctionCall::getOrderBy).filter(Optional::isPresent).map(Optional::get).map(OrderBy::getSortItems).flatMap(List::stream).map(SortItem::getSortKey).forEach(arguments::add);
// filter expressions need to be projected first
analysis.getAggregates(node).stream().map(FunctionCall::getFilter).filter(Optional::isPresent).map(Optional::get).forEach(arguments::add);
Iterable<Expression> inputs = Iterables.concat(groupByExpressions, arguments.build());
subPlan = handleSubqueries(subPlan, node, inputs);
if (!Iterables.isEmpty(inputs)) {
// avoid an empty projection if the only aggregation is COUNT (which has no arguments)
subPlan = project(subPlan, inputs);
}
// 2. Aggregate
// 2.a. Rewrite aggregate arguments
TranslationMap argumentTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToVariableMap);
ImmutableList.Builder<VariableReferenceExpression> aggregationArgumentsBuilder = ImmutableList.builder();
for (Expression argument : arguments.build()) {
VariableReferenceExpression variable = subPlan.translate(argument);
argumentTranslations.put(argument, variable);
aggregationArgumentsBuilder.add(variable);
}
List<VariableReferenceExpression> aggregationArguments = aggregationArgumentsBuilder.build();
// 2.b. Rewrite grouping columns
TranslationMap groupingTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToVariableMap);
Map<VariableReferenceExpression, VariableReferenceExpression> groupingSetMappings = new LinkedHashMap<>();
for (Expression expression : groupByExpressions) {
VariableReferenceExpression input = subPlan.translate(expression);
VariableReferenceExpression output = variableAllocator.newVariable(expression, analysis.getTypeWithCoercions(expression), "gid");
groupingTranslations.put(expression, output);
groupingSetMappings.put(output, input);
}
// This tracks the grouping sets before complex expressions are considered (see comments below)
// It's also used to compute the descriptors needed to implement grouping()
List<Set<FieldId>> columnOnlyGroupingSets = ImmutableList.of(ImmutableSet.of());
List<List<VariableReferenceExpression>> groupingSets = ImmutableList.of(ImmutableList.of());
if (node.getGroupBy().isPresent()) {
// For the purpose of "distinct", we need to canonicalize column references that may have varying
// syntactic forms (e.g., "t.a" vs "a"). Thus we need to enumerate grouping sets based on the underlying
// fieldId associated with each column reference expression.
// The catch is that simple group-by expressions can be arbitrary expressions (this is a departure from the SQL specification).
// But, they don't affect the number of grouping sets or the behavior of "distinct" . We can compute all the candidate
// grouping sets in terms of fieldId, dedup as appropriate and then cross-join them with the complex expressions.
Analysis.GroupingSetAnalysis groupingSetAnalysis = analysis.getGroupingSets(node);
columnOnlyGroupingSets = enumerateGroupingSets(groupingSetAnalysis);
if (node.getGroupBy().get().isDistinct()) {
columnOnlyGroupingSets = columnOnlyGroupingSets.stream().distinct().collect(toImmutableList());
}
// add in the complex expressions an turn materialize the grouping sets in terms of plan columns
ImmutableList.Builder<List<VariableReferenceExpression>> groupingSetBuilder = ImmutableList.builder();
for (Set<FieldId> groupingSet : columnOnlyGroupingSets) {
ImmutableList.Builder<VariableReferenceExpression> columns = ImmutableList.builder();
groupingSetAnalysis.getComplexExpressions().stream().map(groupingTranslations::get).forEach(columns::add);
groupingSet.stream().map(field -> groupingTranslations.get(new FieldReference(field.getFieldIndex()))).forEach(columns::add);
groupingSetBuilder.add(columns.build());
}
groupingSets = groupingSetBuilder.build();
}
// 2.c. Generate GroupIdNode (multiple grouping sets) or ProjectNode (single grouping set)
Optional<VariableReferenceExpression> groupIdVariable = Optional.empty();
if (groupingSets.size() > 1) {
groupIdVariable = Optional.of(variableAllocator.newVariable("groupId", BIGINT));
GroupIdNode groupId = new GroupIdNode(subPlan.getRoot().getSourceLocation(), idAllocator.getNextId(), subPlan.getRoot(), groupingSets, groupingSetMappings, aggregationArguments, groupIdVariable.get());
subPlan = new PlanBuilder(groupingTranslations, groupId);
} else {
Assignments.Builder assignments = Assignments.builder();
aggregationArguments.stream().map(AssignmentUtils::identityAsSymbolReference).forEach(assignments::put);
groupingSetMappings.forEach((key, value) -> assignments.put(key, castToRowExpression(asSymbolReference(value))));
ProjectNode project = new ProjectNode(subPlan.getRoot().getSourceLocation(), idAllocator.getNextId(), subPlan.getRoot(), assignments.build(), LOCAL);
subPlan = new PlanBuilder(groupingTranslations, project);
}
TranslationMap aggregationTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToVariableMap);
aggregationTranslations.copyMappingsFrom(groupingTranslations);
// 2.d. Rewrite aggregates
ImmutableMap.Builder<VariableReferenceExpression, Aggregation> aggregationsBuilder = ImmutableMap.builder();
boolean needPostProjectionCoercion = false;
for (FunctionCall aggregate : analysis.getAggregates(node)) {
Expression rewritten = argumentTranslations.rewrite(aggregate);
VariableReferenceExpression newVariable = variableAllocator.newVariable(rewritten, analysis.getType(aggregate));
// Therefore we can end up with this implicit cast, and have to move it into a post-projection
if (rewritten instanceof Cast) {
rewritten = ((Cast) rewritten).getExpression();
needPostProjectionCoercion = true;
}
aggregationTranslations.put(aggregate, newVariable);
FunctionCall rewrittenFunction = (FunctionCall) rewritten;
aggregationsBuilder.put(newVariable, new Aggregation(new CallExpression(getSourceLocation(rewrittenFunction), aggregate.getName().getSuffix(), analysis.getFunctionHandle(aggregate), analysis.getType(aggregate), rewrittenFunction.getArguments().stream().map(OriginalExpressionUtils::castToRowExpression).collect(toImmutableList())), rewrittenFunction.getFilter().map(OriginalExpressionUtils::castToRowExpression), rewrittenFunction.getOrderBy().map(orderBy -> toOrderingScheme(orderBy, variableAllocator.getTypes())), rewrittenFunction.isDistinct(), Optional.empty()));
}
Map<VariableReferenceExpression, Aggregation> aggregations = aggregationsBuilder.build();
ImmutableSet.Builder<Integer> globalGroupingSets = ImmutableSet.builder();
for (int i = 0; i < groupingSets.size(); i++) {
if (groupingSets.get(i).isEmpty()) {
globalGroupingSets.add(i);
}
}
ImmutableList.Builder<VariableReferenceExpression> groupingKeys = ImmutableList.builder();
groupingSets.stream().flatMap(List::stream).distinct().forEach(groupingKeys::add);
groupIdVariable.ifPresent(groupingKeys::add);
AggregationNode aggregationNode = new AggregationNode(subPlan.getRoot().getSourceLocation(), idAllocator.getNextId(), subPlan.getRoot(), aggregations, groupingSets(groupingKeys.build(), groupingSets.size(), globalGroupingSets.build()), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), groupIdVariable);
subPlan = new PlanBuilder(aggregationTranslations, aggregationNode);
// TODO: this is a hack, we should change type coercions to coerce the inputs to functions/operators instead of coercing the output
if (needPostProjectionCoercion) {
ImmutableList.Builder<Expression> alreadyCoerced = ImmutableList.builder();
alreadyCoerced.addAll(groupByExpressions);
groupIdVariable.map(ExpressionTreeUtils::createSymbolReference).ifPresent(alreadyCoerced::add);
subPlan = explicitCoercionFields(subPlan, alreadyCoerced.build(), analysis.getAggregates(node));
}
// 4. Project and re-write all grouping functions
return handleGroupingOperations(subPlan, node, groupIdVariable, columnOnlyGroupingSets);
}
use of com.facebook.presto.sql.tree.Node in project presto by prestodb.
the class TreePrinter method print.
public void print(Node root) {
AstVisitor<Void, Integer> printer = new DefaultTraversalVisitor<Void, Integer>() {
@Override
protected Void visitNode(Node node, Integer indentLevel) {
throw new UnsupportedOperationException("not yet implemented: " + node);
}
@Override
protected Void visitQuery(Query node, Integer indentLevel) {
print(indentLevel, "Query ");
indentLevel++;
print(indentLevel, "QueryBody");
process(node.getQueryBody(), indentLevel);
if (node.getOrderBy().isPresent()) {
print(indentLevel, "OrderBy");
process(node.getOrderBy().get(), indentLevel + 1);
}
if (node.getLimit().isPresent()) {
print(indentLevel, "Limit: " + node.getLimit().get());
}
return null;
}
@Override
protected Void visitQuerySpecification(QuerySpecification node, Integer indentLevel) {
print(indentLevel, "QuerySpecification ");
indentLevel++;
process(node.getSelect(), indentLevel);
if (node.getFrom().isPresent()) {
print(indentLevel, "From");
process(node.getFrom().get(), indentLevel + 1);
}
if (node.getWhere().isPresent()) {
print(indentLevel, "Where");
process(node.getWhere().get(), indentLevel + 1);
}
if (node.getGroupBy().isPresent()) {
String distinct = "";
if (node.getGroupBy().get().isDistinct()) {
distinct = "[DISTINCT]";
}
print(indentLevel, "GroupBy" + distinct);
for (GroupingElement groupingElement : node.getGroupBy().get().getGroupingElements()) {
print(indentLevel, "SimpleGroupBy");
if (groupingElement instanceof SimpleGroupBy) {
for (Expression column : groupingElement.getExpressions()) {
process(column, indentLevel + 1);
}
} else if (groupingElement instanceof GroupingSets) {
print(indentLevel + 1, "GroupingSets");
for (List<Expression> set : ((GroupingSets) groupingElement).getSets()) {
print(indentLevel + 2, "GroupingSet[");
for (Expression expression : set) {
process(expression, indentLevel + 3);
}
print(indentLevel + 2, "]");
}
} else if (groupingElement instanceof Cube) {
print(indentLevel + 1, "Cube");
for (Expression column : groupingElement.getExpressions()) {
process(column, indentLevel + 1);
}
} else if (groupingElement instanceof Rollup) {
print(indentLevel + 1, "Rollup");
for (Expression column : groupingElement.getExpressions()) {
process(column, indentLevel + 1);
}
}
}
}
if (node.getHaving().isPresent()) {
print(indentLevel, "Having");
process(node.getHaving().get(), indentLevel + 1);
}
if (node.getOrderBy().isPresent()) {
print(indentLevel, "OrderBy");
process(node.getOrderBy().get(), indentLevel + 1);
}
if (node.getLimit().isPresent()) {
print(indentLevel, "Limit: " + node.getLimit().get());
}
return null;
}
protected Void visitOrderBy(OrderBy node, Integer indentLevel) {
for (SortItem sortItem : node.getSortItems()) {
process(sortItem, indentLevel);
}
return null;
}
@Override
protected Void visitSelect(Select node, Integer indentLevel) {
String distinct = "";
if (node.isDistinct()) {
distinct = "[DISTINCT]";
}
print(indentLevel, "Select" + distinct);
// visit children
super.visitSelect(node, indentLevel + 1);
return null;
}
@Override
protected Void visitAllColumns(AllColumns node, Integer indent) {
if (node.getPrefix().isPresent()) {
print(indent, node.getPrefix() + ".*");
} else {
print(indent, "*");
}
return null;
}
@Override
protected Void visitSingleColumn(SingleColumn node, Integer indent) {
if (node.getAlias().isPresent()) {
print(indent, "Alias: " + node.getAlias().get());
}
// visit children
super.visitSingleColumn(node, indent + 1);
return null;
}
@Override
protected Void visitComparisonExpression(ComparisonExpression node, Integer indentLevel) {
print(indentLevel, node.getOperator().toString());
super.visitComparisonExpression(node, indentLevel + 1);
return null;
}
@Override
protected Void visitArithmeticBinary(ArithmeticBinaryExpression node, Integer indentLevel) {
print(indentLevel, node.getOperator().toString());
super.visitArithmeticBinary(node, indentLevel + 1);
return null;
}
@Override
protected Void visitLogicalBinaryExpression(LogicalBinaryExpression node, Integer indentLevel) {
print(indentLevel, node.getOperator().toString());
super.visitLogicalBinaryExpression(node, indentLevel + 1);
return null;
}
@Override
protected Void visitStringLiteral(StringLiteral node, Integer indentLevel) {
print(indentLevel, "String[" + node.getValue() + "]");
return null;
}
@Override
protected Void visitBinaryLiteral(BinaryLiteral node, Integer indentLevel) {
print(indentLevel, "Binary[" + node.toHexString() + "]");
return null;
}
@Override
protected Void visitBooleanLiteral(BooleanLiteral node, Integer indentLevel) {
print(indentLevel, "Boolean[" + node.getValue() + "]");
return null;
}
@Override
protected Void visitLongLiteral(LongLiteral node, Integer indentLevel) {
print(indentLevel, "Long[" + node.getValue() + "]");
return null;
}
@Override
protected Void visitLikePredicate(LikePredicate node, Integer indentLevel) {
print(indentLevel, "LIKE");
super.visitLikePredicate(node, indentLevel + 1);
return null;
}
@Override
protected Void visitIdentifier(Identifier node, Integer indentLevel) {
QualifiedName resolved = resolvedNameReferences.get(node);
String resolvedName = "";
if (resolved != null) {
resolvedName = "=>" + resolved.toString();
}
print(indentLevel, "Identifier[" + node.getValue() + resolvedName + "]");
return null;
}
@Override
protected Void visitDereferenceExpression(DereferenceExpression node, Integer indentLevel) {
QualifiedName resolved = resolvedNameReferences.get(node);
String resolvedName = "";
if (resolved != null) {
resolvedName = "=>" + resolved.toString();
}
print(indentLevel, "DereferenceExpression[" + node + resolvedName + "]");
return null;
}
@Override
protected Void visitFunctionCall(FunctionCall node, Integer indentLevel) {
String name = Joiner.on('.').join(node.getName().getParts());
print(indentLevel, "FunctionCall[" + name + "]");
super.visitFunctionCall(node, indentLevel + 1);
return null;
}
@Override
protected Void visitTable(Table node, Integer indentLevel) {
String name = Joiner.on('.').join(node.getName().getParts());
print(indentLevel, "Table[" + name + "]");
return null;
}
@Override
protected Void visitValues(Values node, Integer indentLevel) {
print(indentLevel, "Values");
super.visitValues(node, indentLevel + 1);
return null;
}
@Override
protected Void visitRow(Row node, Integer indentLevel) {
print(indentLevel, "Row");
super.visitRow(node, indentLevel + 1);
return null;
}
@Override
protected Void visitAliasedRelation(AliasedRelation node, Integer indentLevel) {
print(indentLevel, "Alias[" + node.getAlias() + "]");
super.visitAliasedRelation(node, indentLevel + 1);
return null;
}
@Override
protected Void visitSampledRelation(SampledRelation node, Integer indentLevel) {
print(indentLevel, "TABLESAMPLE[" + node.getType() + " (" + node.getSamplePercentage() + ")]");
super.visitSampledRelation(node, indentLevel + 1);
return null;
}
@Override
protected Void visitTableSubquery(TableSubquery node, Integer indentLevel) {
print(indentLevel, "SubQuery");
super.visitTableSubquery(node, indentLevel + 1);
return null;
}
@Override
protected Void visitInPredicate(InPredicate node, Integer indentLevel) {
print(indentLevel, "IN");
super.visitInPredicate(node, indentLevel + 1);
return null;
}
@Override
protected Void visitSubqueryExpression(SubqueryExpression node, Integer indentLevel) {
print(indentLevel, "SubQuery");
super.visitSubqueryExpression(node, indentLevel + 1);
return null;
}
};
printer.process(root, 0);
}
use of com.facebook.presto.sql.tree.Node in project presto by prestodb.
the class MaterializedViewQueryOptimizer method visitQuerySpecification.
@Override
protected Node visitQuerySpecification(QuerySpecification node, Void context) {
if (!node.getFrom().isPresent()) {
throw new IllegalStateException("Query with no From clause is not rewritable by materialized view");
}
Relation relation = node.getFrom().get();
if (relation instanceof AliasedRelation) {
removablePrefix = Optional.of(((AliasedRelation) relation).getAlias());
relation = ((AliasedRelation) relation).getRelation();
}
if (!(relation instanceof Table)) {
throw new SemanticException(NOT_SUPPORTED, node, "Relation other than Table is not supported in query optimizer");
}
Table baseTable = (Table) relation;
if (!removablePrefix.isPresent()) {
removablePrefix = Optional.of(new Identifier(baseTable.getName().toString()));
}
if (node.getGroupBy().isPresent()) {
ImmutableSet.Builder<Expression> expressionsInGroupByBuilder = ImmutableSet.builder();
for (GroupingElement element : node.getGroupBy().get().getGroupingElements()) {
element = removeGroupingElementPrefix(element, removablePrefix);
Optional<Set<Expression>> groupByOfMaterializedView = materializedViewInfo.getGroupBy();
if (groupByOfMaterializedView.isPresent()) {
for (Expression expression : element.getExpressions()) {
if (!groupByOfMaterializedView.get().contains(expression) || !materializedViewInfo.getBaseToViewColumnMap().containsKey(expression)) {
throw new IllegalStateException(format("Grouping element %s is not present in materialized view groupBy field", element));
}
}
}
expressionsInGroupByBuilder.addAll(element.getExpressions());
}
expressionsInGroupBy = Optional.of(expressionsInGroupByBuilder.build());
}
// TODO: Add HAVING validation to the validator https://github.com/prestodb/presto/issues/16406
if (node.getHaving().isPresent()) {
throw new SemanticException(NOT_SUPPORTED, node, "Having clause is not supported in query optimizer");
}
if (materializedViewInfo.getWhereClause().isPresent()) {
if (!node.getWhere().isPresent()) {
throw new IllegalStateException("Query with no where clause is not rewritable by materialized view with where clause");
}
QualifiedObjectName baseTableName = createQualifiedObjectName(session, baseTable, baseTable.getName());
Optional<TableHandle> tableHandle = metadata.getTableHandle(session, baseTableName);
if (!tableHandle.isPresent()) {
throw new SemanticException(MISSING_TABLE, node, "Table does not exist");
}
ImmutableList.Builder<Field> fields = ImmutableList.builder();
for (ColumnHandle columnHandle : metadata.getColumnHandles(session, tableHandle.get()).values()) {
ColumnMetadata columnMetadata = metadata.getColumnMetadata(session, tableHandle.get(), columnHandle);
fields.add(Field.newUnqualified(materializedViewInfo.getWhereClause().get().getLocation(), columnMetadata.getName(), columnMetadata.getType()));
}
Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields.build())).build();
// Given base query's filter condition and materialized view's filter condition, the goal is to check if MV's filters contain Base's filters (Base implies MV).
// Let base query's filter condition be A, and MV's filter condition be B.
// One way to evaluate A implies B is to evaluate logical expression A^~B and check if the output domain is none.
// If the resulting domain is none, then A^~B is false. Thus A implies B.
// For more information and examples: https://fb.quip.com/WwmxA40jLMxR
// TODO: Implement method that utilizes external SAT solver libraries. https://github.com/prestodb/presto/issues/16536
RowExpression materializedViewWhereCondition = convertToRowExpression(materializedViewInfo.getWhereClause().get(), scope);
RowExpression baseQueryWhereCondition = convertToRowExpression(node.getWhere().get(), scope);
RowExpression rewriteLogicExpression = and(baseQueryWhereCondition, call(baseQueryWhereCondition.getSourceLocation(), "not", new FunctionResolution(metadata.getFunctionAndTypeManager()).notFunction(), materializedViewWhereCondition.getType(), materializedViewWhereCondition));
RowExpression disjunctiveNormalForm = logicalRowExpressions.convertToDisjunctiveNormalForm(rewriteLogicExpression);
ExtractionResult<VariableReferenceExpression> result = domainTranslator.fromPredicate(session.toConnectorSession(), disjunctiveNormalForm, BASIC_COLUMN_EXTRACTOR);
if (!result.getTupleDomain().equals(TupleDomain.none())) {
throw new IllegalStateException("View filter condition does not contain base query's filter condition");
}
}
return new QuerySpecification((Select) process(node.getSelect(), context), node.getFrom().map(from -> (Relation) process(from, context)), node.getWhere().map(where -> (Expression) process(where, context)), node.getGroupBy().map(groupBy -> (GroupBy) process(groupBy, context)), node.getHaving().map(having -> (Expression) process(having, context)), node.getOrderBy().map(orderBy -> (OrderBy) process(orderBy, context)), node.getOffset(), node.getLimit());
}
Aggregations