Examples with OrderingScheme io.prestosql.spi.plan.OrderingScheme used on opensource projects

Example 6 with OrderingScheme

use of io.prestosql.spi.plan.OrderingScheme in project hetu-core by openlookeng.

the class QueryPlanner method planUpdateRowAsInsert.

public UpdateDeleteRelationPlan planUpdateRowAsInsert(Update node) {
    Table table = node.getTable();
    RelationType descriptor = analysis.getOutputDescriptor(table);
    TableHandle handle = analysis.getTableHandle(table);
    ColumnHandle rowIdHandle = analysis.getRowIdHandle(table);
    ColumnMetadata rowIdColumnMetadata = metadata.getColumnMetadata(session, handle, rowIdHandle);
    // add table columns
    ImmutableList.Builder<Symbol> outputSymbols = ImmutableList.builder();
    ImmutableMap.Builder<Symbol, ColumnHandle> columnsBuilder = ImmutableMap.builder();
    ImmutableList.Builder<Field> fields = ImmutableList.builder();
    for (Field field : descriptor.getAllFields()) {
        Symbol symbol = planSymbolAllocator.newSymbol(field);
        outputSymbols.add(symbol);
        columnsBuilder.put(symbol, analysis.getColumn(field));
        fields.add(field);
    }
    // create table scan
    ImmutableMap<Symbol, ColumnHandle> columns = columnsBuilder.build();
    PlanNode tableScan = TableScanNode.newInstance(idAllocator.getNextId(), handle, outputSymbols.build(), columns, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, true);
    Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields.build())).build();
    RelationPlan relationPlan = new RelationPlan(tableScan, scope, outputSymbols.build());
    TranslationMap translations = new TranslationMap(relationPlan, analysis, lambdaDeclarationToSymbolMap);
    translations.setFieldMappings(relationPlan.getFieldMappings());
    PlanBuilder builder = new PlanBuilder(translations, relationPlan.getRoot());
    Optional<RowExpression> predicate = Optional.empty();
    if (node.getWhere().isPresent()) {
        builder = filter(builder, node.getWhere().get(), node);
        if (builder.getRoot() instanceof FilterNode) {
            predicate = Optional.of(((FilterNode) builder.getRoot()).getPredicate());
        }
    }
    List<AssignmentItem> assignmentItems = node.getAssignmentItems();
    Analysis.Update update = analysis.getUpdate().get();
    Assignments.Builder assignments = Assignments.builder();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, update.getTarget());
    Symbol orderBySymbol = null;
    for (Map.Entry<Symbol, ColumnHandle> entry : columns.entrySet()) {
        ColumnMetadata column;
        ColumnHandle columnHandle = entry.getValue();
        Symbol input = entry.getKey();
        if (columnHandle.getColumnName().equals(rowIdHandle.getColumnName())) {
            column = rowIdColumnMetadata;
        } else {
            column = tableMetadata.getColumn(columnHandle.getColumnName());
        }
        if (column != rowIdColumnMetadata && column.isHidden()) {
            continue;
        }
        Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
        Type tableType = column.getType();
        Type queryType = planSymbolAllocator.getTypes().get(input);
        List<AssignmentItem> assignment = assignmentItems.stream().filter(item -> item.getName().equals(QualifiedName.of(column.getName()))).collect(Collectors.toList());
        if (!assignment.isEmpty()) {
            Expression expression = assignment.get(0).getValue();
            Expression cast;
            if (expression instanceof Identifier) {
                // assigning by column reference
                Optional<Symbol> first = columns.entrySet().stream().filter(e -> e.getValue().getColumnName().equals(((Identifier) expression).getValue())).map(Entry::getKey).findFirst();
                Symbol source = (first.orElseThrow(() -> new IllegalArgumentException("Unable to find column " + ((Identifier) expression).getValue())));
                cast = new Cast(toSymbolReference(source), tableType.getTypeSignature().toString());
            } else {
                cast = new Cast(expression, tableType.getTypeSignature().toString());
            }
            assignments.put(output, castToRowExpression(cast));
        } else if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
            assignments.put(output, castToRowExpression(toSymbolReference(input)));
        } else {
            Expression cast = new Cast(toSymbolReference(input), tableType.getTypeSignature().toString());
            assignments.put(output, castToRowExpression(cast));
        }
        if (column == rowIdColumnMetadata) {
            orderBySymbol = output;
        }
    }
    ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), builder.getRoot(), assignments.build());
    PlanBuilder planBuilder = new PlanBuilder(translations, projectNode);
    SortOrder sortOrder = SortOrder.ASC_NULLS_LAST;
    Symbol sortSymbol = orderBySymbol;
    Map<Symbol, SortOrder> sortOrderMap = ImmutableMap.<Symbol, SortOrder>builder().put(sortSymbol, sortOrder).build();
    OrderingScheme orderingScheme = new OrderingScheme(ImmutableList.of(sortSymbol), sortOrderMap);
    builder = sort(planBuilder, Optional.of(orderingScheme));
    ImmutableList.Builder<Field> projectFields = ImmutableList.builder();
    projectFields.addAll(fields.build().stream().filter(x -> !x.isHidden()).collect(toImmutableList()));
    scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(projectFields.build())).build();
    RelationPlan plan = new RelationPlan(builder.getRoot(), scope, projectNode.getOutputSymbols());
    List<String> visibleTableColumnNames = tableMetadata.getColumns().stream().filter(c -> !c.isHidden()).map(ColumnMetadata::getName).collect(Collectors.toList());
    visibleTableColumnNames.add(rowIdColumnMetadata.getName());
    return new UpdateDeleteRelationPlan(plan, visibleTableColumnNames, columns, predicate);
}

Example 7 with OrderingScheme

use of io.prestosql.spi.plan.OrderingScheme in project hetu-core by openlookeng.

the class QueryPlanner method orderingScheme.

private Optional<OrderingScheme> orderingScheme(PlanBuilder subPlan, Optional<OrderBy> orderBy, List<Expression> orderByExpressions) {
    if (!orderBy.isPresent() || (isSkipRedundantSort(session)) && analysis.isOrderByRedundant(orderBy.get())) {
        return Optional.empty();
    }
    Iterator<SortItem> sortItems = orderBy.get().getSortItems().iterator();
    ImmutableList.Builder<Symbol> orderBySymbols = ImmutableList.builder();
    Map<Symbol, SortOrder> orderings = new HashMap<>();
    for (Expression fieldOrExpression : orderByExpressions) {
        Symbol symbol = subPlan.translate(fieldOrExpression);
        SortItem sortItem = sortItems.next();
        if (!orderings.containsKey(symbol)) {
            orderBySymbols.add(symbol);
            orderings.put(symbol, sortItemToSortOrder(sortItem));
        }
    }
    return Optional.of(new OrderingScheme(orderBySymbols.build(), orderings));
}

Example 8 with OrderingScheme

use of io.prestosql.spi.plan.OrderingScheme in project hetu-core by openlookeng.

the class TestBaseBaseJdbcQueryGenerator method testWindowFunctionWithProjectAndRange.

@Test
public void testWindowFunctionWithProjectAndRange() {
    PlanNode scanNode = buildPlan(planBuilder -> tableScan(planBuilder, jdbcTable, regionId, city, fare, amount, startValueColumn, endValueColumn));
    testJQL(planBuilder -> planBuilder.project(Assignments.builder().put(symbol("regionid"), variable("regionid")).put(symbol("city"), variable("city")).put(symbol("amount_out"), variable("amount_out", BIGINT)).build(), planBuilder.window(new WindowNode.Specification(ImmutableList.of(), Optional.empty()), ImmutableMap.of(symbol("amount_out"), new WindowNode.Function(call("min", metadata.getFunctionAndTypeManager().lookupFunction("min", fromTypes(BIGINT)), BIGINT, ImmutableList.of(new VariableReferenceExpression("amount", types.get(symbol("amount"))))), ImmutableList.of(new VariableReferenceExpression("amount", types.get(symbol("amount")))), new WindowNode.Frame(Types.WindowFrameType.RANGE, Types.FrameBoundType.PRECEDING, Optional.of(startValue), Types.FrameBoundType.FOLLOWING, Optional.of(endValue), Optional.of(startValue.getName()), Optional.of(endValue.getName())))), symbol("city"), scanNode)), "SELECT regionid, city, amount_out FROM (SELECT regionid, city, fare, amount, startvalue, endvalue, min(amount) OVER (RANGE BETWEEN startValue PRECEDING AND endValue FOLLOWING) AS amount_out FROM (SELECT regionid, city, fare, amount, startValue AS startvalue, endValue AS endvalue FROM 'table') hetu_table_1) hetu_table_2");
    testJQL(planBuilder -> planBuilder.project(Assignments.builder().put(symbol("regionid"), variable("regionid")).put(symbol("city"), variable("city")).put(symbol("amount_out"), variable("amount_out", BIGINT)).build(), planBuilder.window(new WindowNode.Specification(ImmutableList.of(), Optional.of(new OrderingScheme(ImmutableList.of(symbol("fare")), ImmutableMap.of(symbol("fare"), SortOrder.ASC_NULLS_FIRST)))), ImmutableMap.of(symbol("amount_out"), new WindowNode.Function(call("min", metadata.getFunctionAndTypeManager().lookupFunction("min", fromTypes(BIGINT)), BIGINT, ImmutableList.of(new VariableReferenceExpression("amount", types.get(symbol("amount"))))), ImmutableList.of(new VariableReferenceExpression("amount", types.get(symbol("amount")))), new WindowNode.Frame(Types.WindowFrameType.ROWS, Types.FrameBoundType.PRECEDING, Optional.of(startValue), Types.FrameBoundType.UNBOUNDED_FOLLOWING, Optional.empty(), Optional.of(startValue.getName()), Optional.empty()))), symbol("city"), scanNode)), "SELECT regionid, city, amount_out FROM (SELECT regionid, city, fare, amount, startvalue, endvalue, min(amount) OVER ( ORDER BY fare ASC NULLS FIRST ROWS BETWEEN startValue PRECEDING AND UNBOUNDED FOLLOWING) AS amount_out FROM (SELECT regionid, city, fare, amount, startValue AS startvalue, endValue AS endvalue FROM 'table') hetu_table_1) hetu_table_2");
}

Example 9 with OrderingScheme

use of io.prestosql.spi.plan.OrderingScheme in project hetu-core by openlookeng.

the class QueryPlanner method window.

private PlanBuilder window(PlanBuilder inputSubPlan, List<FunctionCall> windowFunctions) {
    PlanBuilder subPlan = inputSubPlan;
    if (windowFunctions.isEmpty()) {
        return subPlan;
    }
    for (FunctionCall windowFunction : windowFunctions) {
        Window window = windowFunction.getWindow().get();
        // Extract frame
        WindowFrameType frameType = RANGE;
        FrameBoundType frameStartType = UNBOUNDED_PRECEDING;
        FrameBoundType frameEndType = CURRENT_ROW;
        Expression frameStart = null;
        Expression frameEnd = null;
        if (window.getFrame().isPresent()) {
            WindowFrame frame = window.getFrame().get();
            frameType = frame.getType();
            frameStartType = frame.getStart().getType();
            frameStart = frame.getStart().getValue().orElse(null);
            if (frame.getEnd().isPresent()) {
                frameEndType = frame.getEnd().get().getType();
                frameEnd = frame.getEnd().get().getValue().orElse(null);
            }
        }
        // Pre-project inputs
        ImmutableList.Builder<Expression> inputs = ImmutableList.<Expression>builder().addAll(windowFunction.getArguments()).addAll(window.getPartitionBy()).addAll(Iterables.transform(getSortItemsFromOrderBy(window.getOrderBy()), SortItem::getSortKey));
        if (frameStart != null) {
            inputs.add(frameStart);
        }
        if (frameEnd != null) {
            inputs.add(frameEnd);
        }
        subPlan = subPlan.appendProjections(inputs.build(), planSymbolAllocator, idAllocator);
        // Rewrite PARTITION BY in terms of pre-projected inputs
        ImmutableList.Builder<Symbol> partitionBySymbols = ImmutableList.builder();
        for (Expression expression : window.getPartitionBy()) {
            partitionBySymbols.add(subPlan.translate(expression));
        }
        // Rewrite ORDER BY in terms of pre-projected inputs
        LinkedHashMap<Symbol, SortOrder> orderings = new LinkedHashMap<>();
        for (SortItem item : getSortItemsFromOrderBy(window.getOrderBy())) {
            Symbol symbol = subPlan.translate(item.getSortKey());
            // don't override existing keys, i.e. when "ORDER BY a ASC, a DESC" is specified
            orderings.putIfAbsent(symbol, sortItemToSortOrder(item));
        }
        // Rewrite frame bounds in terms of pre-projected inputs
        Optional<Symbol> frameStartSymbol = Optional.empty();
        Optional<Symbol> frameEndSymbol = Optional.empty();
        if (frameStart != null) {
            frameStartSymbol = Optional.of(subPlan.translate(frameStart));
        }
        if (frameEnd != null) {
            frameEndSymbol = Optional.of(subPlan.translate(frameEnd));
        }
        WindowNode.Frame frame = new WindowNode.Frame(frameType, frameStartType, frameStartSymbol, frameEndType, frameEndSymbol, Optional.ofNullable(frameStart).map(Expression::toString), Optional.ofNullable(frameEnd).map(Expression::toString));
        TranslationMap outputTranslations = subPlan.copyTranslations();
        // Rewrite function call in terms of pre-projected inputs
        Expression rewritten = subPlan.rewrite(windowFunction);
        boolean needCoercion = rewritten instanceof Cast;
        // Strip out the cast and add it back as a post-projection
        if (rewritten instanceof Cast) {
            rewritten = ((Cast) rewritten).getExpression();
        }
        // If refers to existing symbol, don't create another PlanNode
        if (rewritten instanceof SymbolReference) {
            if (needCoercion) {
                subPlan = explicitCoercionSymbols(subPlan, subPlan.getRoot().getOutputSymbols(), ImmutableList.of(windowFunction));
            }
            continue;
        }
        Type returnType = analysis.getType(windowFunction);
        Symbol newSymbol = planSymbolAllocator.newSymbol(rewritten, analysis.getType(windowFunction));
        outputTranslations.put(windowFunction, newSymbol);
        List<RowExpression> arguments = new ArrayList<>();
        for (int i = 0; i < ((FunctionCall) rewritten).getArguments().size(); i++) {
            arguments.add(castToRowExpression(((FunctionCall) rewritten).getArguments().get(i)));
        }
        WindowNode.Function function = new WindowNode.Function(call(windowFunction.getName().toString(), analysis.getFunctionHandle(windowFunction), returnType, ((FunctionCall) rewritten).getArguments().stream().map(OriginalExpressionUtils::castToRowExpression).collect(toImmutableList())), arguments, frame);
        List<Symbol> sourceSymbols = subPlan.getRoot().getOutputSymbols();
        ImmutableList.Builder<Symbol> orderBySymbols = ImmutableList.builder();
        orderBySymbols.addAll(orderings.keySet());
        Optional<OrderingScheme> orderingScheme = Optional.empty();
        if (!orderings.isEmpty()) {
            orderingScheme = Optional.of(new OrderingScheme(orderBySymbols.build(), orderings));
        }
        // create window node
        subPlan = new PlanBuilder(outputTranslations, new WindowNode(idAllocator.getNextId(), subPlan.getRoot(), new WindowNode.Specification(partitionBySymbols.build(), orderingScheme), ImmutableMap.of(newSymbol, function), Optional.empty(), ImmutableSet.of(), 0));
        if (needCoercion) {
            subPlan = explicitCoercionSymbols(subPlan, sourceSymbols, ImmutableList.of(windowFunction));
        }
    }
    return subPlan;
}

Example 10 with OrderingScheme

use of io.prestosql.spi.plan.OrderingScheme in project hetu-core by openlookeng.

the class TestWindowNode method testSerializationRoundtrip.

@Test
public void testSerializationRoundtrip() throws Exception {
    Symbol windowSymbol = planSymbolAllocator.newSymbol("sum", BIGINT);
    FunctionHandle functionHandle = createTestMetadataManager().getFunctionAndTypeManager().lookupFunction("sum", fromTypes(BIGINT));
    WindowNode.Frame frame = new WindowNode.Frame(Types.WindowFrameType.RANGE, Types.FrameBoundType.UNBOUNDED_PRECEDING, Optional.empty(), Types.FrameBoundType.UNBOUNDED_FOLLOWING, Optional.empty(), Optional.empty(), Optional.empty());
    PlanNodeId id = newId();
    WindowNode.Specification specification = new WindowNode.Specification(ImmutableList.of(columnA), Optional.of(new OrderingScheme(ImmutableList.of(columnB), ImmutableMap.of(columnB, SortOrder.ASC_NULLS_FIRST))));
    CallExpression call = call("sum", functionHandle, BIGINT, ImmutableList.of(new VariableReferenceExpression(columnC.getName(), BIGINT)));
    Map<Symbol, WindowNode.Function> functions = ImmutableMap.of(windowSymbol, new WindowNode.Function(call, ImmutableList.of(new VariableReferenceExpression(columnC.getName(), BIGINT)), frame));
    Optional<Symbol> hashSymbol = Optional.of(columnB);
    Set<Symbol> prePartitionedInputs = ImmutableSet.of(columnA);
    WindowNode windowNode = new WindowNode(id, sourceNode, specification, functions, hashSymbol, prePartitionedInputs, 0);
    String json = codec.toJson(windowNode);
    WindowNode actualNode = codec.fromJson(json);
    assertEquals(actualNode.getId(), windowNode.getId());
    assertEquals(actualNode.getSpecification(), windowNode.getSpecification());
    assertEquals(actualNode.getWindowFunctions(), windowNode.getWindowFunctions());
    assertEquals(actualNode.getFrames(), windowNode.getFrames());
    assertEquals(actualNode.getHashSymbol(), windowNode.getHashSymbol());
    assertEquals(actualNode.getPrePartitionedInputs(), windowNode.getPrePartitionedInputs());
    assertEquals(actualNode.getPreSortedOrderPrefix(), windowNode.getPreSortedOrderPrefix());
}