Search in sources :

Example 11 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class TestEffectivePredicateExtractor method testInnerJoinWithFalseFilter.

@Test
public void testInnerJoinWithFalseFilter() {
    Map<Symbol, ColumnHandle> leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(A, B, C)));
    TableScanNode leftScan = tableScanNode(leftAssignments);
    Map<Symbol, ColumnHandle> rightAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(D, E, F)));
    TableScanNode rightScan = tableScanNode(rightAssignments);
    PlanNode node = new JoinNode(newId(), JoinNode.Type.INNER, leftScan, rightScan, ImmutableList.of(new JoinNode.EquiJoinClause(A, D)), ImmutableList.<Symbol>builder().addAll(leftScan.getOutputSymbols()).addAll(rightScan.getOutputSymbols()).build(), Optional.of(castToRowExpression(FALSE_LITERAL)), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableMap.of());
    Expression effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer);
    assertEquals(effectivePredicate, FALSE_LITERAL);
}
Also used : TestingColumnHandle(io.prestosql.testing.TestingMetadata.TestingColumnHandle) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) PlanNode(io.prestosql.spi.plan.PlanNode) TableScanNode(io.prestosql.spi.plan.TableScanNode) CallExpression(io.prestosql.spi.relation.CallExpression) OriginalExpressionUtils.castToRowExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToRowExpression) InListExpression(io.prestosql.sql.tree.InListExpression) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) RowExpression(io.prestosql.spi.relation.RowExpression) Expression(io.prestosql.sql.tree.Expression) Symbol(io.prestosql.spi.plan.Symbol) SemiJoinNode(io.prestosql.sql.planner.plan.SemiJoinNode) JoinNode(io.prestosql.spi.plan.JoinNode) Test(org.testng.annotations.Test)

Example 12 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class TableScanStatsRule method doCalculate.

@Override
protected Optional<PlanNodeStatsEstimate> doCalculate(TableScanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) {
    // TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator
    TupleDomain<ColumnHandle> predicate = metadata.getTableProperties(session, node.getTable()).getPredicate();
    Constraint constraint = new Constraint(predicate);
    TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), constraint, true);
    verify(tableStatistics != null, "tableStatistics is null for %s", node);
    Map<Symbol, SymbolStatsEstimate> outputSymbolStats = new HashMap<>();
    Map<ColumnHandle, Symbol> remainingSymbols = new HashMap<>();
    Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
    boolean isPredicatesPushDown = false;
    if ((predicate.isAll() || predicate.getDomains().get().equals(node.getEnforcedConstraint().getDomains().get())) && !(node.getEnforcedConstraint().isAll() || node.getEnforcedConstraint().isNone())) {
        predicate = node.getEnforcedConstraint();
        isPredicatesPushDown = true;
        predicate.getDomains().get().entrySet().stream().forEach(e -> {
            remainingSymbols.put(e.getKey(), new Symbol(e.getKey().getColumnName()));
        });
    }
    for (Map.Entry<Symbol, ColumnHandle> entry : node.getAssignments().entrySet()) {
        Symbol symbol = entry.getKey();
        Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue()));
        SymbolStatsEstimate symbolStatistics = columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics, types.get(symbol))).orElse(SymbolStatsEstimate.unknown());
        outputSymbolStats.put(symbol, symbolStatistics);
        remainingSymbols.remove(entry.getValue());
    }
    PlanNodeStatsEstimate tableEstimates = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).addSymbolStatistics(outputSymbolStats).build();
    if (isPredicatesPushDown) {
        if (remainingSymbols.size() > 0) {
            ImmutableBiMap.Builder<ColumnHandle, Symbol> assignmentBuilder = ImmutableBiMap.builder();
            assignments = assignmentBuilder.putAll(assignments).putAll(remainingSymbols).build();
            for (Map.Entry<ColumnHandle, Symbol> entry : remainingSymbols.entrySet()) {
                Symbol symbol = entry.getValue();
                Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getKey()));
                SymbolStatsEstimate symbolStatistics = columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics, types.get(symbol))).orElse(SymbolStatsEstimate.unknown());
                outputSymbolStats.put(symbol, symbolStatistics);
            }
            /* Refresh TableEstimates for remaining columns */
            tableEstimates = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).addSymbolStatistics(outputSymbolStats).build();
        }
        Expression pushDownExpression = domainTranslator.toPredicate(predicate.transform(assignments::get));
        PlanNodeStatsEstimate estimate = filterStatsCalculator.filterStats(tableEstimates, pushDownExpression, session, types);
        if ((isDefaultFilterFactorEnabled(session) || sourceStats.isEnforceDefaultFilterFactor()) && estimate.isOutputRowCountUnknown()) {
            PlanNodeStatsEstimate finalTableEstimates = tableEstimates;
            estimate = tableEstimates.mapOutputRowCount(sourceRowCount -> finalTableEstimates.getOutputRowCount() * UNKNOWN_FILTER_COEFFICIENT);
        }
        return Optional.of(estimate);
    }
    return Optional.of(tableEstimates);
}
Also used : ColumnStatistics(io.prestosql.spi.statistics.ColumnStatistics) TableStatistics(io.prestosql.spi.statistics.TableStatistics) Lookup(io.prestosql.sql.planner.iterative.Lookup) TypeProvider(io.prestosql.sql.planner.TypeProvider) HashMap(java.util.HashMap) Pattern(io.prestosql.matching.Pattern) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) NaN(java.lang.Double.NaN) ExpressionDomainTranslator(io.prestosql.sql.planner.ExpressionDomainTranslator) Verify.verify(com.google.common.base.Verify.verify) FixedWidthType(io.prestosql.spi.type.FixedWidthType) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) Session(io.prestosql.Session) Type(io.prestosql.spi.type.Type) ColumnStatistics(io.prestosql.spi.statistics.ColumnStatistics) Constraint(io.prestosql.spi.connector.Constraint) Symbol(io.prestosql.spi.plan.Symbol) UNKNOWN_FILTER_COEFFICIENT(io.prestosql.cost.FilterStatsCalculator.UNKNOWN_FILTER_COEFFICIENT) TupleDomain(io.prestosql.spi.predicate.TupleDomain) TableScanNode(io.prestosql.spi.plan.TableScanNode) SystemSessionProperties.isDefaultFilterFactorEnabled(io.prestosql.SystemSessionProperties.isDefaultFilterFactorEnabled) Metadata(io.prestosql.metadata.Metadata) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) LiteralEncoder(io.prestosql.sql.planner.LiteralEncoder) Optional(java.util.Optional) Patterns.tableScan(io.prestosql.sql.planner.plan.Patterns.tableScan) Expression(io.prestosql.sql.tree.Expression) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Constraint(io.prestosql.spi.connector.Constraint) HashMap(java.util.HashMap) Symbol(io.prestosql.spi.plan.Symbol) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) Expression(io.prestosql.sql.tree.Expression) TableStatistics(io.prestosql.spi.statistics.TableStatistics) HashMap(java.util.HashMap) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) Map(java.util.Map)

Example 13 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class PlanPrinter method formatFragment.

private static String formatFragment(Function<TableScanNode, TableInfo> tableInfoSupplier, ValuePrinter valuePrinter, PlanFragment fragment, Optional<StageInfo> stageInfo, Optional<Map<PlanNodeId, PlanNodeStats>> planNodeStats, boolean verbose, List<PlanFragment> allFragments, Metadata metadata) {
    StringBuilder builder = new StringBuilder();
    builder.append(format("Fragment %s [%s]\n", fragment.getId(), fragment.getPartitioning()));
    if (stageInfo.isPresent()) {
        StageStats stageStats = stageInfo.get().getStageStats();
        double avgPositionsPerTask = stageInfo.get().getTasks().stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN);
        double squaredDifferences = stageInfo.get().getTasks().stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum();
        double sdAmongTasks = Math.sqrt(squaredDifferences / stageInfo.get().getTasks().size());
        builder.append(indentString(1)).append(format("CPU: %s, Scheduled: %s, Input: %s (%s); per task: avg.: %s std.dev.: %s, Output: %s (%s)\n", stageStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), stageStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), formatPositions(stageStats.getProcessedInputPositions()), stageStats.getProcessedInputDataSize(), formatDouble(avgPositionsPerTask), formatDouble(sdAmongTasks), formatPositions(stageStats.getOutputPositions()), stageStats.getOutputDataSize()));
    }
    PartitioningScheme partitioningScheme = fragment.getPartitioningScheme();
    builder.append(indentString(1)).append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout())));
    boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny();
    List<String> arguments = partitioningScheme.getPartitioning().getArguments().stream().map(argument -> {
        if (argument.isConstant()) {
            NullableValue constant = argument.getConstant();
            String printableValue = valuePrinter.castToVarchar(constant.getType(), constant.getValue());
            return constant.getType().getDisplayName() + "(" + printableValue + ")";
        }
        return argument.getColumn().toString();
    }).collect(toImmutableList());
    builder.append(indentString(1));
    if (replicateNullsAndAny) {
        builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    } else {
        builder.append(format("Output partitioning: %s [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    }
    builder.append(indentString(1)).append(format("Stage Execution Strategy: %s\n", fragment.getStageExecutionDescriptor().getStageExecutionStrategy()));
    TypeProvider typeProvider = TypeProvider.copyOf(allFragments.stream().flatMap(f -> f.getSymbols().entrySet().stream()).distinct().collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)));
    builder.append(new PlanPrinter(fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), tableInfoSupplier, valuePrinter, fragment.getStatsAndCosts(), planNodeStats, metadata).toText(verbose, 1)).append("\n");
    return builder.toString();
}
Also used : TableDeleteNode(io.prestosql.sql.planner.plan.TableDeleteNode) SortNode(io.prestosql.sql.planner.plan.SortNode) SubPlan(io.prestosql.sql.planner.SubPlan) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) REUSE_STRATEGY_PRODUCER(io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_PRODUCER) TypeProvider(io.prestosql.sql.planner.TypeProvider) NullableValue(io.prestosql.spi.predicate.NullableValue) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) CTEScanNode(io.prestosql.spi.plan.CTEScanNode) AggregationNode(io.prestosql.spi.plan.AggregationNode) TableUpdateNode(io.prestosql.sql.planner.plan.TableUpdateNode) Map(java.util.Map) OutputNode(io.prestosql.sql.planner.plan.OutputNode) Partitioning(io.prestosql.sql.planner.Partitioning) TopNRankingNumberNode(io.prestosql.sql.planner.plan.TopNRankingNumberNode) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TypedSymbol(io.prestosql.sql.planner.planprinter.NodeRepresentation.TypedSymbol) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) CreateIndexNode(io.prestosql.sql.planner.plan.CreateIndexNode) SortExpressionContext(io.prestosql.sql.planner.SortExpressionContext) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) DynamicFilters(io.prestosql.sql.DynamicFilters) IndexSourceNode(io.prestosql.sql.planner.plan.IndexSourceNode) PlanNode(io.prestosql.spi.plan.PlanNode) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) Collectors.joining(java.util.stream.Collectors.joining) SymbolUtils.toSymbolReference(io.prestosql.sql.planner.SymbolUtils.toSymbolReference) SpatialJoinNode(io.prestosql.sql.planner.plan.SpatialJoinNode) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Stream(java.util.stream.Stream) Domain(io.prestosql.spi.predicate.Domain) StatisticAggregations(io.prestosql.sql.planner.plan.StatisticAggregations) ColumnStatisticMetadata(io.prestosql.spi.statistics.ColumnStatisticMetadata) StatisticsWriterNode(io.prestosql.sql.planner.plan.StatisticsWriterNode) VacuumTableNode(io.prestosql.sql.planner.plan.VacuumTableNode) DistinctLimitNode(io.prestosql.sql.planner.plan.DistinctLimitNode) Joiner(com.google.common.base.Joiner) GroupIdNode(io.prestosql.spi.plan.GroupIdNode) Iterables(com.google.common.collect.Iterables) IntersectNode(io.prestosql.spi.plan.IntersectNode) Marker(io.prestosql.spi.predicate.Marker) DynamicFilters.extractDynamicFilters(io.prestosql.sql.DynamicFilters.extractDynamicFilters) StageExecutionDescriptor(io.prestosql.operator.StageExecutionDescriptor) AssignUniqueId(io.prestosql.sql.planner.plan.AssignUniqueId) UnnestNode(io.prestosql.sql.planner.plan.UnnestNode) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) Session(io.prestosql.Session) DeleteNode(io.prestosql.sql.planner.plan.DeleteNode) Functions(com.google.common.base.Functions) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) Assignments(io.prestosql.spi.plan.Assignments) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) TextRenderer.formatPositions(io.prestosql.sql.planner.planprinter.TextRenderer.formatPositions) ValuesNode(io.prestosql.spi.plan.ValuesNode) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) SampleNode(io.prestosql.sql.planner.plan.SampleNode) WindowNode(io.prestosql.spi.plan.WindowNode) LimitNode(io.prestosql.spi.plan.LimitNode) Expression(io.prestosql.sql.tree.Expression) OffsetNode(io.prestosql.sql.planner.plan.OffsetNode) StatisticAggregationsDescriptor(io.prestosql.sql.planner.plan.StatisticAggregationsDescriptor) Scope(io.prestosql.sql.planner.plan.ExchangeNode.Scope) UpdateIndexNode(io.prestosql.sql.planner.plan.UpdateIndexNode) Duration(io.airlift.units.Duration) PlanNodeStatsSummarizer.aggregateStageStats(io.prestosql.sql.planner.planprinter.PlanNodeStatsSummarizer.aggregateStageStats) TableStatisticType(io.prestosql.spi.statistics.TableStatisticType) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) TableFinishNode(io.prestosql.sql.planner.plan.TableFinishNode) PlanCostEstimate(io.prestosql.cost.PlanCostEstimate) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) FilterNode(io.prestosql.spi.plan.FilterNode) TextRenderer.indentString(io.prestosql.sql.planner.planprinter.TextRenderer.indentString) TextRenderer.formatDouble(io.prestosql.sql.planner.planprinter.TextRenderer.formatDouble) Type(io.prestosql.spi.type.Type) ApplyNode(io.prestosql.sql.planner.plan.ApplyNode) ImmutableSet(com.google.common.collect.ImmutableSet) JoinNodeUtils(io.prestosql.sql.planner.optimizations.JoinNodeUtils) Collection(java.util.Collection) Streams(com.google.common.collect.Streams) IndexJoinNode(io.prestosql.sql.planner.plan.IndexJoinNode) CubeFinishNode(io.prestosql.sql.planner.plan.CubeFinishNode) Collectors(java.util.stream.Collectors) RowNumberNode(io.prestosql.sql.planner.plan.RowNumberNode) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) EnforceSingleRowNode(io.prestosql.sql.planner.plan.EnforceSingleRowNode) TopNNode(io.prestosql.spi.plan.TopNNode) StageInfo.getAllStages(io.prestosql.execution.StageInfo.getAllStages) StageInfo(io.prestosql.execution.StageInfo) Entry(java.util.Map.Entry) UnionNode(io.prestosql.spi.plan.UnionNode) Optional(java.util.Optional) ExceptNode(io.prestosql.spi.plan.ExceptNode) Arrays.stream(java.util.Arrays.stream) StageStats(io.prestosql.execution.StageStats) InternalPlanVisitor(io.prestosql.sql.planner.plan.InternalPlanVisitor) SINGLE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) LateralJoinNode(io.prestosql.sql.planner.plan.LateralJoinNode) StatsAndCosts(io.prestosql.cost.StatsAndCosts) RemoteSourceNode(io.prestosql.sql.planner.plan.RemoteSourceNode) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) TRUE_LITERAL(io.prestosql.sql.tree.BooleanLiteral.TRUE_LITERAL) SemiJoinNode(io.prestosql.sql.planner.plan.SemiJoinNode) ImmutableList(com.google.common.collect.ImmutableList) OrderingScheme(io.prestosql.spi.plan.OrderingScheme) GraphvizPrinter(io.prestosql.util.GraphvizPrinter) Verify.verify(com.google.common.base.Verify.verify) Range(io.prestosql.spi.predicate.Range) Objects.requireNonNull(java.util.Objects.requireNonNull) LinkedList(java.util.LinkedList) MarkDistinctNode(io.prestosql.spi.plan.MarkDistinctNode) JoinNode(io.prestosql.spi.plan.JoinNode) Symbol(io.prestosql.spi.plan.Symbol) PlanFragment(io.prestosql.sql.planner.PlanFragment) TableWriterNode(io.prestosql.sql.planner.plan.TableWriterNode) StageExecutionDescriptor.ungroupedExecution(io.prestosql.operator.StageExecutionDescriptor.ungroupedExecution) REUSE_STRATEGY_CONSUMER(io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_CONSUMER) TableInfo(io.prestosql.execution.TableInfo) CaseFormat(com.google.common.base.CaseFormat) GroupReference(io.prestosql.spi.plan.GroupReference) TupleDomain(io.prestosql.spi.predicate.TupleDomain) UpdateNode(io.prestosql.sql.planner.plan.UpdateNode) UPPER_UNDERSCORE(com.google.common.base.CaseFormat.UPPER_UNDERSCORE) Collectors.toList(java.util.stream.Collectors.toList) Aggregation(io.prestosql.spi.plan.AggregationNode.Aggregation) RowExpression(io.prestosql.spi.relation.RowExpression) SortExpressionExtractor(io.prestosql.sql.planner.SortExpressionExtractor) ExplainAnalyzeNode(io.prestosql.sql.planner.plan.ExplainAnalyzeNode) Entry(java.util.Map.Entry) PlanNodeStatsSummarizer.aggregateStageStats(io.prestosql.sql.planner.planprinter.PlanNodeStatsSummarizer.aggregateStageStats) StageStats(io.prestosql.execution.StageStats) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) NullableValue(io.prestosql.spi.predicate.NullableValue) TypeProvider(io.prestosql.sql.planner.TypeProvider) TextRenderer.indentString(io.prestosql.sql.planner.planprinter.TextRenderer.indentString)

Example 14 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class PlanPrinter method textDistributedPlan.

public static String textDistributedPlan(StageInfo outputStageInfo, ValuePrinter valuePrinter, boolean verbose, Metadata metadata) {
    Map<PlanNodeId, TableInfo> tableInfos = getAllStages(Optional.of(outputStageInfo)).stream().map(StageInfo::getTables).map(Map::entrySet).flatMap(Collection::stream).collect(toImmutableMap(Entry::getKey, Entry::getValue));
    StringBuilder builder = new StringBuilder();
    List<StageInfo> allStages = getAllStages(Optional.of(outputStageInfo));
    List<PlanFragment> allFragments = allStages.stream().map(StageInfo::getPlan).collect(toImmutableList());
    Map<PlanNodeId, PlanNodeStats> aggregatedStats = aggregateStageStats(allStages);
    for (StageInfo stageInfo : allStages) {
        builder.append(formatFragment(tableScanNode -> tableInfos.get(tableScanNode.getId()), valuePrinter, stageInfo.getPlan(), Optional.of(stageInfo), Optional.of(aggregatedStats), verbose, allFragments, metadata));
    }
    return builder.toString();
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TableDeleteNode(io.prestosql.sql.planner.plan.TableDeleteNode) SortNode(io.prestosql.sql.planner.plan.SortNode) SubPlan(io.prestosql.sql.planner.SubPlan) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) REUSE_STRATEGY_PRODUCER(io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_PRODUCER) TypeProvider(io.prestosql.sql.planner.TypeProvider) NullableValue(io.prestosql.spi.predicate.NullableValue) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) CTEScanNode(io.prestosql.spi.plan.CTEScanNode) AggregationNode(io.prestosql.spi.plan.AggregationNode) TableUpdateNode(io.prestosql.sql.planner.plan.TableUpdateNode) Map(java.util.Map) OutputNode(io.prestosql.sql.planner.plan.OutputNode) Partitioning(io.prestosql.sql.planner.Partitioning) TopNRankingNumberNode(io.prestosql.sql.planner.plan.TopNRankingNumberNode) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TypedSymbol(io.prestosql.sql.planner.planprinter.NodeRepresentation.TypedSymbol) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) CreateIndexNode(io.prestosql.sql.planner.plan.CreateIndexNode) SortExpressionContext(io.prestosql.sql.planner.SortExpressionContext) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) DynamicFilters(io.prestosql.sql.DynamicFilters) IndexSourceNode(io.prestosql.sql.planner.plan.IndexSourceNode) PlanNode(io.prestosql.spi.plan.PlanNode) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) Collectors.joining(java.util.stream.Collectors.joining) SymbolUtils.toSymbolReference(io.prestosql.sql.planner.SymbolUtils.toSymbolReference) SpatialJoinNode(io.prestosql.sql.planner.plan.SpatialJoinNode) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Stream(java.util.stream.Stream) Domain(io.prestosql.spi.predicate.Domain) StatisticAggregations(io.prestosql.sql.planner.plan.StatisticAggregations) ColumnStatisticMetadata(io.prestosql.spi.statistics.ColumnStatisticMetadata) StatisticsWriterNode(io.prestosql.sql.planner.plan.StatisticsWriterNode) VacuumTableNode(io.prestosql.sql.planner.plan.VacuumTableNode) DistinctLimitNode(io.prestosql.sql.planner.plan.DistinctLimitNode) Joiner(com.google.common.base.Joiner) GroupIdNode(io.prestosql.spi.plan.GroupIdNode) Iterables(com.google.common.collect.Iterables) IntersectNode(io.prestosql.spi.plan.IntersectNode) Marker(io.prestosql.spi.predicate.Marker) DynamicFilters.extractDynamicFilters(io.prestosql.sql.DynamicFilters.extractDynamicFilters) StageExecutionDescriptor(io.prestosql.operator.StageExecutionDescriptor) AssignUniqueId(io.prestosql.sql.planner.plan.AssignUniqueId) UnnestNode(io.prestosql.sql.planner.plan.UnnestNode) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) Session(io.prestosql.Session) DeleteNode(io.prestosql.sql.planner.plan.DeleteNode) Functions(com.google.common.base.Functions) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) Assignments(io.prestosql.spi.plan.Assignments) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) TextRenderer.formatPositions(io.prestosql.sql.planner.planprinter.TextRenderer.formatPositions) ValuesNode(io.prestosql.spi.plan.ValuesNode) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) SampleNode(io.prestosql.sql.planner.plan.SampleNode) WindowNode(io.prestosql.spi.plan.WindowNode) LimitNode(io.prestosql.spi.plan.LimitNode) Expression(io.prestosql.sql.tree.Expression) OffsetNode(io.prestosql.sql.planner.plan.OffsetNode) StatisticAggregationsDescriptor(io.prestosql.sql.planner.plan.StatisticAggregationsDescriptor) Scope(io.prestosql.sql.planner.plan.ExchangeNode.Scope) UpdateIndexNode(io.prestosql.sql.planner.plan.UpdateIndexNode) Duration(io.airlift.units.Duration) PlanNodeStatsSummarizer.aggregateStageStats(io.prestosql.sql.planner.planprinter.PlanNodeStatsSummarizer.aggregateStageStats) TableStatisticType(io.prestosql.spi.statistics.TableStatisticType) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) TableFinishNode(io.prestosql.sql.planner.plan.TableFinishNode) PlanCostEstimate(io.prestosql.cost.PlanCostEstimate) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) FilterNode(io.prestosql.spi.plan.FilterNode) TextRenderer.indentString(io.prestosql.sql.planner.planprinter.TextRenderer.indentString) TextRenderer.formatDouble(io.prestosql.sql.planner.planprinter.TextRenderer.formatDouble) Type(io.prestosql.spi.type.Type) ApplyNode(io.prestosql.sql.planner.plan.ApplyNode) ImmutableSet(com.google.common.collect.ImmutableSet) JoinNodeUtils(io.prestosql.sql.planner.optimizations.JoinNodeUtils) Collection(java.util.Collection) Streams(com.google.common.collect.Streams) IndexJoinNode(io.prestosql.sql.planner.plan.IndexJoinNode) CubeFinishNode(io.prestosql.sql.planner.plan.CubeFinishNode) Collectors(java.util.stream.Collectors) RowNumberNode(io.prestosql.sql.planner.plan.RowNumberNode) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) EnforceSingleRowNode(io.prestosql.sql.planner.plan.EnforceSingleRowNode) TopNNode(io.prestosql.spi.plan.TopNNode) StageInfo.getAllStages(io.prestosql.execution.StageInfo.getAllStages) StageInfo(io.prestosql.execution.StageInfo) Entry(java.util.Map.Entry) UnionNode(io.prestosql.spi.plan.UnionNode) Optional(java.util.Optional) ExceptNode(io.prestosql.spi.plan.ExceptNode) Arrays.stream(java.util.Arrays.stream) StageStats(io.prestosql.execution.StageStats) InternalPlanVisitor(io.prestosql.sql.planner.plan.InternalPlanVisitor) SINGLE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) LateralJoinNode(io.prestosql.sql.planner.plan.LateralJoinNode) StatsAndCosts(io.prestosql.cost.StatsAndCosts) RemoteSourceNode(io.prestosql.sql.planner.plan.RemoteSourceNode) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) TRUE_LITERAL(io.prestosql.sql.tree.BooleanLiteral.TRUE_LITERAL) SemiJoinNode(io.prestosql.sql.planner.plan.SemiJoinNode) ImmutableList(com.google.common.collect.ImmutableList) OrderingScheme(io.prestosql.spi.plan.OrderingScheme) GraphvizPrinter(io.prestosql.util.GraphvizPrinter) Verify.verify(com.google.common.base.Verify.verify) Range(io.prestosql.spi.predicate.Range) Objects.requireNonNull(java.util.Objects.requireNonNull) LinkedList(java.util.LinkedList) MarkDistinctNode(io.prestosql.spi.plan.MarkDistinctNode) JoinNode(io.prestosql.spi.plan.JoinNode) Symbol(io.prestosql.spi.plan.Symbol) PlanFragment(io.prestosql.sql.planner.PlanFragment) TableWriterNode(io.prestosql.sql.planner.plan.TableWriterNode) StageExecutionDescriptor.ungroupedExecution(io.prestosql.operator.StageExecutionDescriptor.ungroupedExecution) REUSE_STRATEGY_CONSUMER(io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_CONSUMER) TableInfo(io.prestosql.execution.TableInfo) CaseFormat(com.google.common.base.CaseFormat) GroupReference(io.prestosql.spi.plan.GroupReference) TupleDomain(io.prestosql.spi.predicate.TupleDomain) UpdateNode(io.prestosql.sql.planner.plan.UpdateNode) UPPER_UNDERSCORE(com.google.common.base.CaseFormat.UPPER_UNDERSCORE) Collectors.toList(java.util.stream.Collectors.toList) Aggregation(io.prestosql.spi.plan.AggregationNode.Aggregation) RowExpression(io.prestosql.spi.relation.RowExpression) SortExpressionExtractor(io.prestosql.sql.planner.SortExpressionExtractor) ExplainAnalyzeNode(io.prestosql.sql.planner.plan.ExplainAnalyzeNode) StageInfo(io.prestosql.execution.StageInfo) TableInfo(io.prestosql.execution.TableInfo) Map(java.util.Map) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) PlanFragment(io.prestosql.sql.planner.PlanFragment)

Example 15 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class TestHiveDistributedJoinQueriesWithDynamicFiltering method searchScanFilterAndProjectOperatorStats.

private OperatorStats searchScanFilterAndProjectOperatorStats(QueryId queryId, String tableName) {
    DistributedQueryRunner runner = (DistributedQueryRunner) getQueryRunner();
    Plan plan = runner.getQueryPlan(queryId);
    PlanNodeId nodeId = PlanNodeSearcher.searchFrom(plan.getRoot()).where(node -> {
        if (!(node instanceof ProjectNode)) {
            return false;
        }
        ProjectNode projectNode = (ProjectNode) node;
        FilterNode filterNode = (FilterNode) projectNode.getSource();
        TableScanNode tableScanNode = (TableScanNode) filterNode.getSource();
        return tableName.equals(tableScanNode.getTable().getConnectorHandle().toString());
    }).findOnlyElement().getId();
    return runner.getCoordinator().getQueryManager().getFullQueryInfo(queryId).getQueryStats().getOperatorSummaries().stream().filter(summary -> nodeId.equals(summary.getPlanNodeId())).collect(MoreCollectors.onlyElement());
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) DynamicFilter(io.prestosql.spi.dynamicfilter.DynamicFilter) HiveTestUtils.getDefaultHiveSelectiveFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveSelectiveFactories) Assertions.assertGreaterThan(io.airlift.testing.Assertions.assertGreaterThan) Plan(io.prestosql.sql.planner.Plan) Test(org.testng.annotations.Test) MaterializedResult(io.prestosql.testing.MaterializedResult) TypeSignature.parseTypeSignature(io.prestosql.spi.type.TypeSignature.parseTypeSignature) FilterNode(io.prestosql.spi.plan.FilterNode) ENABLE_DYNAMIC_FILTERING(io.prestosql.SystemSessionProperties.ENABLE_DYNAMIC_FILTERING) DistributedQueryRunner(io.prestosql.tests.DistributedQueryRunner) Map(java.util.Map) Assert.assertFalse(org.testng.Assert.assertFalse) HIVE_INT(io.prestosql.plugin.hive.HiveType.HIVE_INT) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) PrestoException(io.prestosql.spi.PrestoException) ImmutableMap(com.google.common.collect.ImmutableMap) HiveTestUtils.createTestHdfsEnvironment(io.prestosql.plugin.hive.HiveTestUtils.createTestHdfsEnvironment) MoreCollectors(com.google.common.collect.MoreCollectors) TableScanNode(io.prestosql.spi.plan.TableScanNode) ProjectNode(io.prestosql.spi.plan.ProjectNode) HiveQueryRunner.createQueryRunnerWithStateStore(io.prestosql.plugin.hive.HiveQueryRunner.createQueryRunnerWithStateStore) TpchTable.getTables(io.airlift.tpch.TpchTable.getTables) AbstractTestQueryFramework(io.prestosql.tests.AbstractTestQueryFramework) List(java.util.List) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) Optional(java.util.Optional) OperatorStats(io.prestosql.operator.OperatorStats) JOIN_DISTRIBUTION_TYPE(io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) StandardTypes(io.prestosql.spi.type.StandardTypes) ByteArrayOutputStream(java.io.ByteArrayOutputStream) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) TYPE_MANAGER(io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER) Assert.assertEquals(org.testng.Assert.assertEquals) OptionalInt(java.util.OptionalInt) Supplier(java.util.function.Supplier) BloomFilter(io.prestosql.spi.util.BloomFilter) ImmutableList(com.google.common.collect.ImmutableList) Assertions.assertLessThanOrEqual(io.airlift.testing.Assertions.assertLessThanOrEqual) PARTITION_KEY(io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) Session(io.prestosql.Session) DynamicFilterSupplier(io.prestosql.spi.dynamicfilter.DynamicFilterSupplier) QueryId(io.prestosql.spi.QueryId) HiveTestUtils.getNoOpIndexCache(io.prestosql.plugin.hive.HiveTestUtils.getNoOpIndexCache) Properties(java.util.Properties) DYNAMIC_FILTERING_WAIT_TIME(io.prestosql.SystemSessionProperties.DYNAMIC_FILTERING_WAIT_TIME) ResultWithQueryId(io.prestosql.tests.ResultWithQueryId) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) HiveTestUtils.getDefaultHiveRecordCursorProvider(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) Assert.fail(org.testng.Assert.fail) IOException(java.io.IOException) HiveTestUtils.getDefaultHiveDataStreamFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) PlanNodeSearcher(io.prestosql.sql.planner.optimizations.PlanNodeSearcher) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) Assert.assertTrue(org.testng.Assert.assertTrue) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) DynamicFilterFactory(io.prestosql.spi.dynamicfilter.DynamicFilterFactory) DistributedQueryRunner(io.prestosql.tests.DistributedQueryRunner) TableScanNode(io.prestosql.spi.plan.TableScanNode) FilterNode(io.prestosql.spi.plan.FilterNode) ProjectNode(io.prestosql.spi.plan.ProjectNode) Plan(io.prestosql.sql.planner.Plan)

Aggregations

TableScanNode (io.prestosql.spi.plan.TableScanNode)77 Symbol (io.prestosql.spi.plan.Symbol)42 PlanNode (io.prestosql.spi.plan.PlanNode)41 Test (org.testng.annotations.Test)33 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)30 FilterNode (io.prestosql.spi.plan.FilterNode)28 RowExpression (io.prestosql.spi.relation.RowExpression)24 Type (io.prestosql.spi.type.Type)22 JoinNode (io.prestosql.spi.plan.JoinNode)21 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)21 Map (java.util.Map)21 TableHandle (io.prestosql.spi.metadata.TableHandle)20 ProjectNode (io.prestosql.spi.plan.ProjectNode)20 Optional (java.util.Optional)20 ImmutableList (com.google.common.collect.ImmutableList)18 Expression (io.prestosql.sql.tree.Expression)18 HashMap (java.util.HashMap)16 List (java.util.List)16 ImmutableMap (com.google.common.collect.ImmutableMap)15 Session (io.prestosql.Session)15