Examples with QuerySpecification io.prestosql.sql.tree.QuerySpecification used on opensource projects

Example 11 with QuerySpecification

use of io.prestosql.sql.tree.QuerySpecification in project hetu-core by openlookeng.

the class QueryPlanner method handleGroupingOperations.

private PlanBuilder handleGroupingOperations(PlanBuilder subPlan, QuerySpecification node, Optional<Symbol> groupIdSymbol, List<Set<FieldId>> groupingSets) {
    if (analysis.getGroupingOperations(node).isEmpty()) {
        return subPlan;
    }
    TranslationMap newTranslations = subPlan.copyTranslations();
    Assignments.Builder projections = Assignments.builder();
    projections.putAll(AssignmentUtils.identityAsSymbolReferences(subPlan.getRoot().getOutputSymbols()));
    List<Set<Integer>> descriptor = groupingSets.stream().map(set -> set.stream().map(FieldId::getFieldIndex).collect(toImmutableSet())).collect(toImmutableList());
    for (GroupingOperation groupingOperation : analysis.getGroupingOperations(node)) {
        Expression rewritten = GroupingOperationRewriter.rewriteGroupingOperation(groupingOperation, descriptor, analysis.getColumnReferenceFields(), groupIdSymbol);
        Type coercion = analysis.getCoercion(groupingOperation);
        Symbol symbol = planSymbolAllocator.newSymbol(rewritten, analysis.getTypeWithCoercions(groupingOperation));
        if (coercion != null) {
            rewritten = new Cast(rewritten, coercion.getTypeSignature().toString(), false, typeCoercion.isTypeOnlyCoercion(analysis.getType(groupingOperation), coercion));
        }
        projections.put(symbol, castToRowExpression(rewritten));
        newTranslations.put(groupingOperation, symbol);
    }
    return new PlanBuilder(newTranslations, new ProjectNode(idAllocator.getNextId(), subPlan.getRoot(), projections.build()));
}

Example 12 with QuerySpecification

use of io.prestosql.sql.tree.QuerySpecification in project hetu-core by openlookeng.

the class QueryPlanner method aggregate.

private PlanBuilder aggregate(PlanBuilder inputSubPlan, QuerySpecification node) {
    PlanBuilder subPlan = inputSubPlan;
    if (!analysis.isAggregation(node)) {
        return subPlan;
    }
    // 1. Pre-project all scalar inputs (arguments and non-trivial group by expressions)
    Set<Expression> groupByExpressions = ImmutableSet.copyOf(analysis.getGroupByExpressions(node));
    ImmutableList.Builder<Expression> arguments = ImmutableList.builder();
    analysis.getAggregates(node).stream().map(FunctionCall::getArguments).flatMap(List::stream).filter(// lambda expression is generated at execution time
    exp -> !(exp instanceof LambdaExpression)).forEach(arguments::add);
    analysis.getAggregates(node).stream().map(FunctionCall::getOrderBy).filter(Optional::isPresent).map(Optional::get).map(OrderBy::getSortItems).flatMap(List::stream).map(SortItem::getSortKey).forEach(arguments::add);
    // filter expressions need to be projected first
    analysis.getAggregates(node).stream().map(FunctionCall::getFilter).filter(Optional::isPresent).map(Optional::get).forEach(arguments::add);
    Iterable<Expression> inputs = Iterables.concat(groupByExpressions, arguments.build());
    subPlan = handleSubqueries(subPlan, node, inputs);
    if (!Iterables.isEmpty(inputs)) {
        // avoid an empty projection if the only aggregation is COUNT (which has no arguments)
        subPlan = project(subPlan, inputs);
    }
    // 2. Aggregate
    // 2.a. Rewrite aggregate arguments
    TranslationMap argumentTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToSymbolMap);
    ImmutableList.Builder<Symbol> aggregationArgumentsBuilder = ImmutableList.builder();
    for (Expression argument : arguments.build()) {
        Symbol symbol = subPlan.translate(argument);
        argumentTranslations.put(argument, symbol);
        aggregationArgumentsBuilder.add(symbol);
    }
    List<Symbol> aggregationArguments = aggregationArgumentsBuilder.build();
    // 2.b. Rewrite grouping columns
    TranslationMap groupingTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToSymbolMap);
    Map<Symbol, Symbol> groupingSetMappings = new LinkedHashMap<>();
    for (Expression expression : groupByExpressions) {
        Symbol input = subPlan.translate(expression);
        Symbol output = planSymbolAllocator.newSymbol(expression, analysis.getTypeWithCoercions(expression), "gid");
        groupingTranslations.put(expression, output);
        groupingSetMappings.put(output, input);
    }
    // This tracks the grouping sets before complex expressions are considered (see comments below)
    // It's also used to compute the descriptors needed to implement grouping()
    List<Set<FieldId>> columnOnlyGroupingSets = ImmutableList.of(ImmutableSet.of());
    List<List<Symbol>> groupingSets = ImmutableList.of(ImmutableList.of());
    if (node.getGroupBy().isPresent()) {
        // For the purpose of "distinct", we need to canonicalize column references that may have varying
        // syntactic forms (e.g., "t.a" vs "a"). Thus we need to enumerate grouping sets based on the underlying
        // fieldId associated with each column reference expression.
        // The catch is that simple group-by expressions can be arbitrary expressions (this is a departure from the SQL specification).
        // But, they don't affect the number of grouping sets or the behavior of "distinct" . We can compute all the candidate
        // grouping sets in terms of fieldId, dedup as appropriate and then cross-join them with the complex expressions.
        Analysis.GroupingSetAnalysis groupingSetAnalysis = analysis.getGroupingSets(node);
        columnOnlyGroupingSets = enumerateGroupingSets(groupingSetAnalysis);
        if (node.getGroupBy().get().isDistinct()) {
            columnOnlyGroupingSets = columnOnlyGroupingSets.stream().distinct().collect(toImmutableList());
        }
        // add in the complex expressions an turn materialize the grouping sets in terms of plan columns
        ImmutableList.Builder<List<Symbol>> groupingSetBuilder = ImmutableList.builder();
        for (Set<FieldId> groupingSet : columnOnlyGroupingSets) {
            ImmutableList.Builder<Symbol> columns = ImmutableList.builder();
            groupingSetAnalysis.getComplexExpressions().stream().map(groupingTranslations::get).forEach(columns::add);
            groupingSet.stream().map(field -> groupingTranslations.get(new FieldReference(field.getFieldIndex()))).forEach(columns::add);
            groupingSetBuilder.add(columns.build());
        }
        groupingSets = groupingSetBuilder.build();
    }
    // 2.c. Generate GroupIdNode (multiple grouping sets) or ProjectNode (single grouping set)
    Optional<Symbol> groupIdSymbol = Optional.empty();
    if (groupingSets.size() > 1) {
        groupIdSymbol = Optional.of(planSymbolAllocator.newSymbol("groupId", BIGINT));
        GroupIdNode groupId = new GroupIdNode(idAllocator.getNextId(), subPlan.getRoot(), groupingSets, groupingSetMappings, aggregationArguments, groupIdSymbol.get());
        subPlan = new PlanBuilder(groupingTranslations, groupId);
    } else {
        Assignments.Builder assignments = Assignments.builder();
        aggregationArguments.forEach(symbol -> assignments.put(symbol, castToRowExpression(toSymbolReference(symbol))));
        groupingSetMappings.forEach((key, value) -> assignments.put(key, castToRowExpression(toSymbolReference(value))));
        ProjectNode project = new ProjectNode(idAllocator.getNextId(), subPlan.getRoot(), assignments.build());
        subPlan = new PlanBuilder(groupingTranslations, project);
    }
    TranslationMap aggregationTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToSymbolMap);
    aggregationTranslations.copyMappingsFrom(groupingTranslations);
    // 2.d. Rewrite aggregates
    ImmutableMap.Builder<Symbol, Aggregation> aggregationsBuilder = ImmutableMap.builder();
    boolean needPostProjectionCoercion = false;
    for (FunctionCall aggregate : analysis.getAggregates(node)) {
        Expression rewritten = argumentTranslations.rewrite(aggregate);
        Symbol newSymbol = planSymbolAllocator.newSymbol(rewritten, analysis.getType(aggregate));
        // Therefore we can end up with this implicit cast, and have to move it into a post-projection
        if (rewritten instanceof Cast) {
            rewritten = ((Cast) rewritten).getExpression();
            needPostProjectionCoercion = true;
        }
        aggregationTranslations.put(aggregate, newSymbol);
        FunctionCall functionCall = (FunctionCall) rewritten;
        aggregationsBuilder.put(newSymbol, new Aggregation(call(aggregate.getName().getSuffix(), analysis.getFunctionHandle(aggregate), analysis.getType(aggregate), functionCall.getArguments().stream().map(OriginalExpressionUtils::castToRowExpression).collect(toImmutableList())), functionCall.getArguments().stream().map(OriginalExpressionUtils::castToRowExpression).collect(toImmutableList()), functionCall.isDistinct(), functionCall.getFilter().map(SymbolUtils::from), functionCall.getOrderBy().map(OrderingSchemeUtils::fromOrderBy), Optional.empty()));
    }
    Map<Symbol, Aggregation> aggregations = aggregationsBuilder.build();
    ImmutableSet.Builder<Integer> globalGroupingSets = ImmutableSet.builder();
    for (int i = 0; i < groupingSets.size(); i++) {
        if (groupingSets.get(i).isEmpty()) {
            globalGroupingSets.add(i);
        }
    }
    ImmutableList.Builder<Symbol> groupingKeys = ImmutableList.builder();
    groupingSets.stream().flatMap(List::stream).distinct().forEach(groupingKeys::add);
    groupIdSymbol.ifPresent(groupingKeys::add);
    AggregationNode aggregationNode = new AggregationNode(idAllocator.getNextId(), subPlan.getRoot(), aggregations, groupingSets(groupingKeys.build(), groupingSets.size(), globalGroupingSets.build()), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), groupIdSymbol, AggregationNode.AggregationType.HASH, Optional.empty());
    subPlan = new PlanBuilder(aggregationTranslations, aggregationNode);
    // TODO: this is a hack, we should change type coercions to coerce the inputs to functions/operators instead of coercing the output
    if (needPostProjectionCoercion) {
        ImmutableList.Builder<Expression> alreadyCoerced = ImmutableList.builder();
        alreadyCoerced.addAll(groupByExpressions);
        groupIdSymbol.map(SymbolUtils::toSymbolReference).ifPresent(alreadyCoerced::add);
        subPlan = explicitCoercionFields(subPlan, alreadyCoerced.build(), analysis.getAggregates(node));
    }
    // 4. Project and re-write all grouping functions
    return handleGroupingOperations(subPlan, node, groupIdSymbol, columnOnlyGroupingSets);
}

Example 13 with QuerySpecification

use of io.prestosql.sql.tree.QuerySpecification in project hetu-core by openlookeng.

the class ImpalaAstBuilder method visitQuerySpecification.

@Override
public Node visitQuerySpecification(ImpalaSqlParser.QuerySpecificationContext context) {
    Optional<Relation> from = Optional.empty();
    List<SelectItem> selectItems = visit(context.selectItem(), SelectItem.class);
    List<Relation> relations = visit(context.relation(), Relation.class);
    if (!relations.isEmpty()) {
        // synthesize implicit join nodes
        Iterator<Relation> iterator = relations.iterator();
        Relation relation = iterator.next();
        while (iterator.hasNext()) {
            relation = new Join(getLocation(context), Join.Type.IMPLICIT, relation, iterator.next(), Optional.empty());
        }
        from = Optional.of(relation);
    }
    return new QuerySpecification(getLocation(context), new Select(getLocation(context.SELECT()), isDistinct(context.setQuantifier()), selectItems), from, visitIfPresent(context.where, Expression.class), visitIfPresent(context.groupBy(), GroupBy.class), visitIfPresent(context.having, Expression.class), Optional.empty(), Optional.empty(), Optional.empty());
}

Example 14 with QuerySpecification

use of io.prestosql.sql.tree.QuerySpecification in project hetu-core by openlookeng.

the class ImpalaAstBuilder method visitQueryNoWith.

@Override
public Node visitQueryNoWith(ImpalaSqlParser.QueryNoWithContext context) {
    QueryBody term = (QueryBody) visit(context.queryTerm());
    Optional<OrderBy> orderBy = Optional.empty();
    if (context.ORDER() != null) {
        orderBy = Optional.of(new OrderBy(getLocation(context.ORDER()), visit(context.sortItem(), SortItem.class)));
    }
    Optional<Node> limit = Optional.empty();
    Optional<Offset> offset = Optional.empty();
    if (context.LIMIT() != null) {
        Optional<String> offsetValue = getTextIfPresent(context.offset);
        Optional<String> rowsValue = getTextIfPresent(context.rows);
        if (offsetValue.isPresent()) {
            offset = Optional.of(new Offset(offsetValue.get()));
        }
        if (rowsValue.isPresent()) {
            limit = Optional.of(new Limit(rowsValue.get()));
        }
    }
    if (term instanceof QuerySpecification) {
        // When we have a simple query specification
        // followed by order by, offset, limit or fetch,
        // fold the order by, limit, offset or fetch clauses
        // into the query specification (analyzer/planner
        // expects this structure to resolve references with respect
        // to columns defined in the query specification)
        QuerySpecification query = (QuerySpecification) term;
        return new Query(getLocation(context), Optional.empty(), new QuerySpecification(getLocation(context), query.getSelect(), query.getFrom(), query.getWhere(), query.getGroupBy(), query.getHaving(), orderBy, offset, limit), Optional.empty(), Optional.empty(), Optional.empty());
    }
    return new Query(getLocation(context), Optional.empty(), term, orderBy, offset, limit);
}

Example 15 with QuerySpecification

use of io.prestosql.sql.tree.QuerySpecification in project hetu-core by openlookeng.

the class HiveAstBuilder method visitQueryNoWith.

@Override
public Node visitQueryNoWith(HiveSqlParser.QueryNoWithContext context) {
    QueryBody term = (QueryBody) visit(context.queryTerm());
    Optional<OrderBy> orderBy = Optional.empty();
    if (context.ORDER() != null) {
        orderBy = Optional.of(new OrderBy(getLocation(context.ORDER()), visit(context.sortItem(), SortItem.class)));
    }
    if (context.clusteredBy() != null) {
        addDiff(DiffType.UNSUPPORTED, context.CLUSTER().getText(), "[CLUSTERED BY] is not supported");
        throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "Unsupported attribute: CLUSTERED BY", context.clusteredBy());
    }
    if (context.distributeBy() != null) {
        addDiff(DiffType.UNSUPPORTED, context.DISTRIBUTE().getText(), "[DISTRIBUTE BY] is not supported");
        throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "Unsupported attribute: DISTRIBUTE BY", context.distributeBy());
    }
    if (context.sortedBy() != null) {
        addDiff(DiffType.UNSUPPORTED, context.SORT().getText(), "[SORT BY] is not supported");
        throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "Unsupported attribute: SORT BY", context.sortedBy());
    }
    Optional<Node> limit = Optional.empty();
    Optional<Offset> offset = Optional.empty();
    if (context.LIMIT() != null) {
        Optional<String> offsetValue = getTextIfPresent(context.offset);
        Optional<String> rowsValue = getTextIfPresent(context.rows);
        if (offsetValue.isPresent()) {
            offset = Optional.of(new Offset(offsetValue.get()));
        }
        if (rowsValue.isPresent()) {
            limit = Optional.of(new Limit(rowsValue.get()));
        }
    }
    if (term instanceof QuerySpecification) {
        QuerySpecification query = (QuerySpecification) term;
        return new Query(getLocation(context), Optional.empty(), new QuerySpecification(getLocation(context), query.getSelect(), query.getFrom(), query.getWhere(), query.getGroupBy(), query.getHaving(), orderBy, offset, limit), Optional.empty(), Optional.empty(), Optional.empty());
    }
    return new Query(getLocation(context), Optional.empty(), term, orderBy, offset, limit);
}