Search in sources :

Example 1 with TypeProvider

use of io.prestosql.sql.planner.TypeProvider in project hetu-core by openlookeng.

the class RuleAssert method matches.

public void matches(PlanMatchPattern pattern) {
    RuleApplication ruleApplication = applyRule();
    TypeProvider typeProvider = ruleApplication.types;
    if (!ruleApplication.wasRuleApplied()) {
        fail(format("%s did not fire for:\n%s", rule.getClass().getName(), formatPlan(plan, typeProvider)));
    }
    PlanNode actual = ruleApplication.getTransformedPlan();
    if (actual == plan) {
        // plans are not comparable, so we can only ensure they are not the same instance
        fail(format("%s: rule fired but return the original plan:\n%s", rule.getClass().getName(), formatPlan(plan, typeProvider)));
    }
    if (!ImmutableSet.copyOf(plan.getOutputSymbols()).equals(ImmutableSet.copyOf(actual.getOutputSymbols()))) {
        fail(format("%s: output schema of transformed and original plans are not equivalent\n" + "\texpected: %s\n" + "\tactual:   %s", rule.getClass().getName(), plan.getOutputSymbols(), actual.getOutputSymbols()));
    }
    inTransaction(session -> {
        assertPlan(session, metadata, ruleApplication.statsProvider, new Plan(actual, typeProvider, StatsAndCosts.empty()), ruleApplication.lookup, pattern);
        return null;
    });
}
Also used : PlanNode(io.prestosql.spi.plan.PlanNode) TypeProvider(io.prestosql.sql.planner.TypeProvider) Plan(io.prestosql.sql.planner.Plan) PlanPrinter.textLogicalPlan(io.prestosql.sql.planner.planprinter.PlanPrinter.textLogicalPlan) PlanAssert.assertPlan(io.prestosql.sql.planner.assertions.PlanAssert.assertPlan)

Example 2 with TypeProvider

use of io.prestosql.sql.planner.TypeProvider in project hetu-core by openlookeng.

the class TestExpressionEquivalence method assertNotEquivalent.

private static void assertNotEquivalent(@Language("SQL") String left, @Language("SQL") String right) {
    ParsingOptions parsingOptions = new ParsingOptions(AS_DOUBLE);
    Expression leftExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(left, parsingOptions));
    Expression rightExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(right, parsingOptions));
    Set<Symbol> symbols = extractUnique(ImmutableList.of(leftExpression, rightExpression));
    TypeProvider types = TypeProvider.copyOf(symbols.stream().collect(toMap(identity(), TestExpressionEquivalence::generateType)));
    assertFalse(EQUIVALENCE.areExpressionsEquivalent(TEST_SESSION, leftExpression, rightExpression, types), format("Expected (%s) and (%s) to not be equivalent", left, right));
    assertFalse(EQUIVALENCE.areExpressionsEquivalent(TEST_SESSION, rightExpression, leftExpression, types), format("Expected (%s) and (%s) to not be equivalent", right, left));
}
Also used : ParsingOptions(io.prestosql.sql.parser.ParsingOptions) Expression(io.prestosql.sql.tree.Expression) Symbol(io.prestosql.spi.plan.Symbol) TypeProvider(io.prestosql.sql.planner.TypeProvider)

Example 3 with TypeProvider

use of io.prestosql.sql.planner.TypeProvider in project hetu-core by openlookeng.

the class TestValidateStreamingAggregations method validatePlan.

private void validatePlan(Function<PlanBuilder, PlanNode> planProvider) {
    PlanBuilder builder = new PlanBuilder(idAllocator, metadata);
    PlanNode planNode = planProvider.apply(builder);
    TypeProvider types = builder.getTypes();
    getQueryRunner().inTransaction(session -> {
        // metadata.getCatalogHandle() registers the catalog for the transaction
        session.getCatalog().ifPresent(catalog -> metadata.getCatalogHandle(session, catalog));
        new ValidateStreamingAggregations().validate(planNode, session, metadata, typeAnalyzer, types, WarningCollector.NOOP);
        return null;
    });
}
Also used : PlanNode(io.prestosql.spi.plan.PlanNode) TypeProvider(io.prestosql.sql.planner.TypeProvider) PlanBuilder(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder)

Example 4 with TypeProvider

use of io.prestosql.sql.planner.TypeProvider in project hetu-core by openlookeng.

the class TableScanStatsRule method doCalculate.

@Override
protected Optional<PlanNodeStatsEstimate> doCalculate(TableScanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) {
    // TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator
    TupleDomain<ColumnHandle> predicate = metadata.getTableProperties(session, node.getTable()).getPredicate();
    Constraint constraint = new Constraint(predicate);
    TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), constraint, true);
    verify(tableStatistics != null, "tableStatistics is null for %s", node);
    Map<Symbol, SymbolStatsEstimate> outputSymbolStats = new HashMap<>();
    Map<ColumnHandle, Symbol> remainingSymbols = new HashMap<>();
    Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
    boolean isPredicatesPushDown = false;
    if ((predicate.isAll() || predicate.getDomains().get().equals(node.getEnforcedConstraint().getDomains().get())) && !(node.getEnforcedConstraint().isAll() || node.getEnforcedConstraint().isNone())) {
        predicate = node.getEnforcedConstraint();
        isPredicatesPushDown = true;
        predicate.getDomains().get().entrySet().stream().forEach(e -> {
            remainingSymbols.put(e.getKey(), new Symbol(e.getKey().getColumnName()));
        });
    }
    for (Map.Entry<Symbol, ColumnHandle> entry : node.getAssignments().entrySet()) {
        Symbol symbol = entry.getKey();
        Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue()));
        SymbolStatsEstimate symbolStatistics = columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics, types.get(symbol))).orElse(SymbolStatsEstimate.unknown());
        outputSymbolStats.put(symbol, symbolStatistics);
        remainingSymbols.remove(entry.getValue());
    }
    PlanNodeStatsEstimate tableEstimates = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).addSymbolStatistics(outputSymbolStats).build();
    if (isPredicatesPushDown) {
        if (remainingSymbols.size() > 0) {
            ImmutableBiMap.Builder<ColumnHandle, Symbol> assignmentBuilder = ImmutableBiMap.builder();
            assignments = assignmentBuilder.putAll(assignments).putAll(remainingSymbols).build();
            for (Map.Entry<ColumnHandle, Symbol> entry : remainingSymbols.entrySet()) {
                Symbol symbol = entry.getValue();
                Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getKey()));
                SymbolStatsEstimate symbolStatistics = columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics, types.get(symbol))).orElse(SymbolStatsEstimate.unknown());
                outputSymbolStats.put(symbol, symbolStatistics);
            }
            /* Refresh TableEstimates for remaining columns */
            tableEstimates = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).addSymbolStatistics(outputSymbolStats).build();
        }
        Expression pushDownExpression = domainTranslator.toPredicate(predicate.transform(assignments::get));
        PlanNodeStatsEstimate estimate = filterStatsCalculator.filterStats(tableEstimates, pushDownExpression, session, types);
        if ((isDefaultFilterFactorEnabled(session) || sourceStats.isEnforceDefaultFilterFactor()) && estimate.isOutputRowCountUnknown()) {
            PlanNodeStatsEstimate finalTableEstimates = tableEstimates;
            estimate = tableEstimates.mapOutputRowCount(sourceRowCount -> finalTableEstimates.getOutputRowCount() * UNKNOWN_FILTER_COEFFICIENT);
        }
        return Optional.of(estimate);
    }
    return Optional.of(tableEstimates);
}
Also used : ColumnStatistics(io.prestosql.spi.statistics.ColumnStatistics) TableStatistics(io.prestosql.spi.statistics.TableStatistics) Lookup(io.prestosql.sql.planner.iterative.Lookup) TypeProvider(io.prestosql.sql.planner.TypeProvider) HashMap(java.util.HashMap) Pattern(io.prestosql.matching.Pattern) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) NaN(java.lang.Double.NaN) ExpressionDomainTranslator(io.prestosql.sql.planner.ExpressionDomainTranslator) Verify.verify(com.google.common.base.Verify.verify) FixedWidthType(io.prestosql.spi.type.FixedWidthType) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) Session(io.prestosql.Session) Type(io.prestosql.spi.type.Type) ColumnStatistics(io.prestosql.spi.statistics.ColumnStatistics) Constraint(io.prestosql.spi.connector.Constraint) Symbol(io.prestosql.spi.plan.Symbol) UNKNOWN_FILTER_COEFFICIENT(io.prestosql.cost.FilterStatsCalculator.UNKNOWN_FILTER_COEFFICIENT) TupleDomain(io.prestosql.spi.predicate.TupleDomain) TableScanNode(io.prestosql.spi.plan.TableScanNode) SystemSessionProperties.isDefaultFilterFactorEnabled(io.prestosql.SystemSessionProperties.isDefaultFilterFactorEnabled) Metadata(io.prestosql.metadata.Metadata) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) LiteralEncoder(io.prestosql.sql.planner.LiteralEncoder) Optional(java.util.Optional) Patterns.tableScan(io.prestosql.sql.planner.plan.Patterns.tableScan) Expression(io.prestosql.sql.tree.Expression) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Constraint(io.prestosql.spi.connector.Constraint) HashMap(java.util.HashMap) Symbol(io.prestosql.spi.plan.Symbol) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) Expression(io.prestosql.sql.tree.Expression) TableStatistics(io.prestosql.spi.statistics.TableStatistics) HashMap(java.util.HashMap) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) Map(java.util.Map)

Example 5 with TypeProvider

use of io.prestosql.sql.planner.TypeProvider in project hetu-core by openlookeng.

the class TopNStatsRule method doCalculate.

@Override
protected Optional<PlanNodeStatsEstimate> doCalculate(TopNNode node, StatsProvider statsProvider, Lookup lookup, Session session, TypeProvider types) {
    PlanNodeStatsEstimate sourceStats = statsProvider.getStats(node.getSource());
    double rowCount = sourceStats.getOutputRowCount();
    if (rowCount <= node.getCount()) {
        return Optional.of(sourceStats);
    }
    long limitCount = node.getCount();
    PlanNodeStatsEstimate resultStats = PlanNodeStatsEstimate.buildFrom(sourceStats).setOutputRowCount(limitCount).build();
    if (limitCount == 0) {
        return Optional.of(resultStats);
    }
    // augment null fraction estimation for first ORDER BY symbol
    // Assuming not empty list
    Symbol firstOrderSymbol = node.getOrderingScheme().getOrderBy().get(0);
    SortOrder sortOrder = node.getOrderingScheme().getOrdering(firstOrderSymbol);
    resultStats = resultStats.mapSymbolColumnStatistics(firstOrderSymbol, symbolStats -> {
        SymbolStatsEstimate.Builder newStats = SymbolStatsEstimate.buildFrom(symbolStats);
        double nullCount = rowCount * symbolStats.getNullsFraction();
        if (sortOrder.isNullsFirst()) {
            if (nullCount > limitCount) {
                newStats.setNullsFraction(1.0);
            } else {
                newStats.setNullsFraction(nullCount / limitCount);
            }
        } else {
            double nonNullCount = (rowCount - nullCount);
            if (nonNullCount > limitCount) {
                newStats.setNullsFraction(0.0);
            } else {
                newStats.setNullsFraction((limitCount - nonNullCount) / limitCount);
            }
        }
        return newStats.build();
    });
    // TopN actually limits (or when there was no row count estimated for source)
    return Optional.of(resultStats);
}
Also used : Symbol(io.prestosql.spi.plan.Symbol) Patterns.topN(io.prestosql.sql.planner.plan.Patterns.topN) Lookup(io.prestosql.sql.planner.iterative.Lookup) TopNNode(io.prestosql.spi.plan.TopNNode) Session(io.prestosql.Session) TypeProvider(io.prestosql.sql.planner.TypeProvider) Optional(java.util.Optional) Pattern(io.prestosql.matching.Pattern) SortOrder(io.prestosql.spi.block.SortOrder) Symbol(io.prestosql.spi.plan.Symbol) SortOrder(io.prestosql.spi.block.SortOrder)

Aggregations

TypeProvider (io.prestosql.sql.planner.TypeProvider)26 Symbol (io.prestosql.spi.plan.Symbol)19 Optional (java.util.Optional)12 Session (io.prestosql.Session)11 PlanNode (io.prestosql.spi.plan.PlanNode)9 RowExpression (io.prestosql.spi.relation.RowExpression)9 Expression (io.prestosql.sql.tree.Expression)9 Map (java.util.Map)9 Metadata (io.prestosql.metadata.Metadata)8 List (java.util.List)8 Objects.requireNonNull (java.util.Objects.requireNonNull)8 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)7 OriginalExpressionUtils.castToExpression (io.prestosql.sql.relational.OriginalExpressionUtils.castToExpression)7 HashMap (java.util.HashMap)7 ImmutableList (com.google.common.collect.ImmutableList)6 Pattern (io.prestosql.matching.Pattern)6 FilterNode (io.prestosql.spi.plan.FilterNode)6 OriginalExpressionUtils.isExpression (io.prestosql.sql.relational.OriginalExpressionUtils.isExpression)6 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)5 ImmutableMap (com.google.common.collect.ImmutableMap)5