Examples with TypeProvider io.prestosql.sql.planner.TypeProvider used on opensource projects

Example 11 with TypeProvider

use of io.prestosql.sql.planner.TypeProvider in project hetu-core by openlookeng.

the class HetuLogicalPlanner method plan.

@Override
public Plan plan(Analysis analysis, boolean skipStatsWithPlan, Stage stage) {
    PlanNode root = planStatement(analysis, analysis.getStatement());
    PlanNode.SkipOptRuleLevel optimizationLevel = APPLY_ALL_RULES;
    planSanityChecker.validateIntermediatePlan(root, session, metadata, typeAnalyzer, planSymbolAllocator.getTypes(), warningCollector);
    if (stage.ordinal() >= Stage.OPTIMIZED.ordinal()) {
        for (PlanOptimizer optimizer : planOptimizers) {
            if (OptimizerUtils.isEnabledLegacy(optimizer, session, root)) {
                // We run it separately in the SqlQueryExecution
                if (optimizer instanceof BeginTableWrite) {
                    continue;
                }
                if (OptimizerUtils.canApplyOptimizer(optimizer, optimizationLevel)) {
                    root = optimizer.optimize(root, session, planSymbolAllocator.getTypes(), planSymbolAllocator, idAllocator, warningCollector);
                    requireNonNull(root, format("%s returned a null plan", optimizer.getClass().getName()));
                    optimizationLevel = optimizationLevel == APPLY_ALL_RULES ? root.getSkipOptRuleLevel() : optimizationLevel;
                }
            }
        }
    }
    if (stage.ordinal() >= Stage.OPTIMIZED_AND_VALIDATED.ordinal()) {
        // make sure we produce a valid plan after optimizations run. This is mainly to catch programming errors
        planSanityChecker.validateFinalPlan(root, session, metadata, typeAnalyzer, planSymbolAllocator.getTypes(), warningCollector);
    }
    TypeProvider types = planSymbolAllocator.getTypes();
    // we calculate stats here only if need to show as part of EXPLAIN, otherwise not needed.
    if (skipStatsWithPlan && isSkipAttachingStatsWithPlan(session)) {
        return new Plan(root, types, StatsAndCosts.empty());
    } else {
        StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types);
        CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session, types);
        return new Plan(root, types, StatsAndCosts.create(root, statsProvider, costProvider));
    }
}

Example 12 with TypeProvider

use of io.prestosql.sql.planner.TypeProvider in project hetu-core by openlookeng.

the class PropertyDerivations method deriveProperties.

public static ActualProperties deriveProperties(PlanNode node, List<ActualProperties> inputProperties, Metadata metadata, Session session, TypeProvider types, TypeAnalyzer typeAnalyzer) {
    ActualProperties output = node.accept(new Visitor(metadata, session, types, typeAnalyzer), inputProperties);
    output.getNodePartitioning().ifPresent(partitioning -> verify(node.getOutputSymbols().containsAll(partitioning.getColumns()), "Node-level partitioning properties contain columns not present in node's output"));
    verify(node.getOutputSymbols().containsAll(output.getConstants().keySet()), "Node-level constant properties contain columns not present in node's output");
    Set<Symbol> localPropertyColumns = output.getLocalProperties().stream().flatMap(property -> property.getColumns().stream()).collect(Collectors.toSet());
    verify(node.getOutputSymbols().containsAll(localPropertyColumns), "Node-level local properties contain columns not present in node's output");
    return output;
}

Example 13 with TypeProvider

use of io.prestosql.sql.planner.TypeProvider in project hetu-core by openlookeng.

the class TransformFilteringSemiJoinToInnerJoin method apply.

@Override
public Result apply(FilterNode filterNode, Captures captures, Context context) {
    SemiJoinNode semiJoin = captures.get(SEMI_JOIN);
    // Do no transform semi-join in context of DELETE
    if (PlanNodeSearcher.searchFrom(semiJoin.getSource(), context.getLookup()).where(node -> node instanceof TableScanNode && ((TableScanNode) node).isForDelete()).matches()) {
        return Result.empty();
    }
    Symbol semiJoinSymbol = semiJoin.getSemiJoinOutput();
    TypeProvider types = context.getSymbolAllocator().getTypes();
    Predicate<RowExpression> isSemiJoinSymbol = expression -> expression.equals(toVariableReference(semiJoinSymbol, types));
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(metadata), new FunctionResolution(metadata.getFunctionAndTypeManager()), metadata.getFunctionAndTypeManager());
    List<RowExpression> conjuncts = LogicalRowExpressions.extractConjuncts(filterNode.getPredicate());
    if (conjuncts.stream().noneMatch(isSemiJoinSymbol)) {
        return Result.empty();
    }
    RowExpression filteredPredicate = LogicalRowExpressions.and(conjuncts.stream().filter(expression -> !expression.equals(toVariableReference(semiJoinSymbol, types))).collect(toImmutableList()));
    RowExpression simplifiedPredicate = inlineVariables(variable -> {
        if (variable.equals(toVariableReference(semiJoinSymbol, types))) {
            return TRUE_CONSTANT;
        }
        return variable;
    }, filteredPredicate);
    Optional<RowExpression> joinFilter = simplifiedPredicate.equals(TRUE_CONSTANT) ? Optional.empty() : Optional.of(simplifiedPredicate);
    PlanNode filteringSourceDistinct = new AggregationNode(context.getIdAllocator().getNextId(), semiJoin.getFilteringSource(), ImmutableMap.of(), singleGroupingSet(ImmutableList.of(semiJoin.getFilteringSourceJoinSymbol())), ImmutableList.of(), SINGLE, Optional.empty(), Optional.empty(), AggregationNode.AggregationType.HASH, Optional.empty());
    JoinNode innerJoin = new JoinNode(semiJoin.getId(), INNER, semiJoin.getSource(), filteringSourceDistinct, ImmutableList.of(new EquiJoinClause(semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), semiJoin.getSource().getOutputSymbols(), joinFilter.isPresent() ? Optional.of(joinFilter.get()) : Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), // TODO: dynamic filter from SemiJoinNode
    ImmutableMap.of());
    ProjectNode project = new ProjectNode(context.getIdAllocator().getNextId(), innerJoin, Assignments.builder().putAll(AssignmentUtils.identityAsSymbolReferences(innerJoin.getOutputSymbols())).put(semiJoinSymbol, TRUE_CONSTANT).build());
    return Result.ofPlanNode(project);
}

Example 14 with TypeProvider

use of io.prestosql.sql.planner.TypeProvider in project hetu-core by openlookeng.

the class ActualProperties method translateRowExpression.

public ActualProperties translateRowExpression(Map<Symbol, RowExpression> assignments, TypeProvider types) {
    Map<Symbol, SymbolReference> inputToOutputSymbol = new HashMap<>();
    for (Map.Entry<Symbol, RowExpression> assignment : assignments.entrySet()) {
        RowExpression expression = assignment.getValue();
        SymbolReference symbolReference = SymbolUtils.toSymbolReference(assignment.getKey());
        if (isExpression(expression)) {
            if (castToExpression(expression) instanceof SymbolReference) {
                inputToOutputSymbol.put(SymbolUtils.from(castToExpression(expression)), symbolReference);
            }
        } else {
            if (expression instanceof VariableReferenceExpression) {
                inputToOutputSymbol.put(new Symbol(((VariableReferenceExpression) expression).getName()), symbolReference);
            }
        }
    }
    Map<Symbol, Partitioning.ArgumentBinding> inputToOutputMappings = inputToOutputSymbol.entrySet().stream().collect(Collectors.toMap(v -> v.getKey(), v -> expressionBinding(v.getValue())));
    Map<Symbol, NullableValue> translatedConstants = new HashMap<>();
    for (Map.Entry<Symbol, NullableValue> entry : constants.entrySet()) {
        if (inputToOutputSymbol.containsKey(entry.getKey())) {
            Symbol symbol = SymbolUtils.from(inputToOutputSymbol.get(entry.getKey()));
            translatedConstants.put(symbol, entry.getValue());
        } else {
            inputToOutputMappings.put(entry.getKey(), constantBinding(entry.getValue()));
        }
    }
    return builder().global(global.translateRowExpression(inputToOutputMappings, assignments, types)).local(LocalProperties.translate(localProperties, symbol -> inputToOutputSymbol.containsKey(symbol) ? Optional.of(SymbolUtils.from(inputToOutputSymbol.get(symbol))) : Optional.empty())).constants(translatedConstants).build();
}

Example 15 with TypeProvider

use of io.prestosql.sql.planner.TypeProvider in project hetu-core by openlookeng.

the class DetermineJoinDistributionType method getJoinNodeWithCost.

private PlanNodeWithCost getJoinNodeWithCost(Context context, JoinNode possibleJoinNode) {
    TypeProvider types = context.getSymbolAllocator().getTypes();
    StatsProvider stats = context.getStatsProvider();
    boolean replicated = possibleJoinNode.getDistributionType().get().equals(REPLICATED);
    /*
         *   HACK!
         *
         *   Currently cost model always has to compute the total cost of an operation.
         *   For JOIN the total cost consist of 4 parts:
         *     - Cost of exchanges that have to be introduced to execute a JOIN
         *     - Cost of building a hash table
         *     - Cost of probing a hash table
         *     - Cost of building an output for matched rows
         *
         *   When output size for a JOIN cannot be estimated the cost model returns
         *   UNKNOWN cost for the join.
         *
         *   However assuming the cost of JOIN output is always the same, we can still make
         *   cost based decisions based on the input cost for different types of JOINs.
         *
         *   Although the side flipping can be made purely based on stats (smaller side
         *   always goes to the right), determining JOIN type is not that simple. As when
         *   choosing REPLICATED over REPARTITIONED join the cost of exchanging and building
         *   the hash table scales with the number of nodes where the build side is replicated.
         *
         *   TODO Decision about the distribution should be based on LocalCostEstimate only when PlanCostEstimate cannot be calculated. Otherwise cost comparator cannot take query.max-memory into account.
         */
    int estimatedSourceDistributedTaskCount = taskCountEstimator.estimateSourceDistributedTaskCount();
    LocalCostEstimate cost = calculateJoinCostWithoutOutput(possibleJoinNode.getLeft(), possibleJoinNode.getRight(), stats, types, replicated, estimatedSourceDistributedTaskCount);
    return new PlanNodeWithCost(cost.toPlanCost(), possibleJoinNode);
}