use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.
the class TestEffectivePredicateExtractor method testInnerJoinWithFalseFilter.
@Test
public void testInnerJoinWithFalseFilter() {
Map<Symbol, ColumnHandle> leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(A, B, C)));
TableScanNode leftScan = tableScanNode(leftAssignments);
Map<Symbol, ColumnHandle> rightAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(D, E, F)));
TableScanNode rightScan = tableScanNode(rightAssignments);
PlanNode node = new JoinNode(newId(), JoinNode.Type.INNER, leftScan, rightScan, ImmutableList.of(new JoinNode.EquiJoinClause(A, D)), ImmutableList.<Symbol>builder().addAll(leftScan.getOutputSymbols()).addAll(rightScan.getOutputSymbols()).build(), Optional.of(castToRowExpression(FALSE_LITERAL)), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableMap.of());
Expression effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer);
assertEquals(effectivePredicate, FALSE_LITERAL);
}
use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.
the class TableScanStatsRule method doCalculate.
@Override
protected Optional<PlanNodeStatsEstimate> doCalculate(TableScanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) {
// TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator
TupleDomain<ColumnHandle> predicate = metadata.getTableProperties(session, node.getTable()).getPredicate();
Constraint constraint = new Constraint(predicate);
TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), constraint, true);
verify(tableStatistics != null, "tableStatistics is null for %s", node);
Map<Symbol, SymbolStatsEstimate> outputSymbolStats = new HashMap<>();
Map<ColumnHandle, Symbol> remainingSymbols = new HashMap<>();
Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
boolean isPredicatesPushDown = false;
if ((predicate.isAll() || predicate.getDomains().get().equals(node.getEnforcedConstraint().getDomains().get())) && !(node.getEnforcedConstraint().isAll() || node.getEnforcedConstraint().isNone())) {
predicate = node.getEnforcedConstraint();
isPredicatesPushDown = true;
predicate.getDomains().get().entrySet().stream().forEach(e -> {
remainingSymbols.put(e.getKey(), new Symbol(e.getKey().getColumnName()));
});
}
for (Map.Entry<Symbol, ColumnHandle> entry : node.getAssignments().entrySet()) {
Symbol symbol = entry.getKey();
Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue()));
SymbolStatsEstimate symbolStatistics = columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics, types.get(symbol))).orElse(SymbolStatsEstimate.unknown());
outputSymbolStats.put(symbol, symbolStatistics);
remainingSymbols.remove(entry.getValue());
}
PlanNodeStatsEstimate tableEstimates = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).addSymbolStatistics(outputSymbolStats).build();
if (isPredicatesPushDown) {
if (remainingSymbols.size() > 0) {
ImmutableBiMap.Builder<ColumnHandle, Symbol> assignmentBuilder = ImmutableBiMap.builder();
assignments = assignmentBuilder.putAll(assignments).putAll(remainingSymbols).build();
for (Map.Entry<ColumnHandle, Symbol> entry : remainingSymbols.entrySet()) {
Symbol symbol = entry.getValue();
Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getKey()));
SymbolStatsEstimate symbolStatistics = columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics, types.get(symbol))).orElse(SymbolStatsEstimate.unknown());
outputSymbolStats.put(symbol, symbolStatistics);
}
/* Refresh TableEstimates for remaining columns */
tableEstimates = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).addSymbolStatistics(outputSymbolStats).build();
}
Expression pushDownExpression = domainTranslator.toPredicate(predicate.transform(assignments::get));
PlanNodeStatsEstimate estimate = filterStatsCalculator.filterStats(tableEstimates, pushDownExpression, session, types);
if ((isDefaultFilterFactorEnabled(session) || sourceStats.isEnforceDefaultFilterFactor()) && estimate.isOutputRowCountUnknown()) {
PlanNodeStatsEstimate finalTableEstimates = tableEstimates;
estimate = tableEstimates.mapOutputRowCount(sourceRowCount -> finalTableEstimates.getOutputRowCount() * UNKNOWN_FILTER_COEFFICIENT);
}
return Optional.of(estimate);
}
return Optional.of(tableEstimates);
}
use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.
the class PlanPrinter method formatFragment.
private static String formatFragment(Function<TableScanNode, TableInfo> tableInfoSupplier, ValuePrinter valuePrinter, PlanFragment fragment, Optional<StageInfo> stageInfo, Optional<Map<PlanNodeId, PlanNodeStats>> planNodeStats, boolean verbose, List<PlanFragment> allFragments, Metadata metadata) {
StringBuilder builder = new StringBuilder();
builder.append(format("Fragment %s [%s]\n", fragment.getId(), fragment.getPartitioning()));
if (stageInfo.isPresent()) {
StageStats stageStats = stageInfo.get().getStageStats();
double avgPositionsPerTask = stageInfo.get().getTasks().stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN);
double squaredDifferences = stageInfo.get().getTasks().stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum();
double sdAmongTasks = Math.sqrt(squaredDifferences / stageInfo.get().getTasks().size());
builder.append(indentString(1)).append(format("CPU: %s, Scheduled: %s, Input: %s (%s); per task: avg.: %s std.dev.: %s, Output: %s (%s)\n", stageStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), stageStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), formatPositions(stageStats.getProcessedInputPositions()), stageStats.getProcessedInputDataSize(), formatDouble(avgPositionsPerTask), formatDouble(sdAmongTasks), formatPositions(stageStats.getOutputPositions()), stageStats.getOutputDataSize()));
}
PartitioningScheme partitioningScheme = fragment.getPartitioningScheme();
builder.append(indentString(1)).append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout())));
boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny();
List<String> arguments = partitioningScheme.getPartitioning().getArguments().stream().map(argument -> {
if (argument.isConstant()) {
NullableValue constant = argument.getConstant();
String printableValue = valuePrinter.castToVarchar(constant.getType(), constant.getValue());
return constant.getType().getDisplayName() + "(" + printableValue + ")";
}
return argument.getColumn().toString();
}).collect(toImmutableList());
builder.append(indentString(1));
if (replicateNullsAndAny) {
builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
} else {
builder.append(format("Output partitioning: %s [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
}
builder.append(indentString(1)).append(format("Stage Execution Strategy: %s\n", fragment.getStageExecutionDescriptor().getStageExecutionStrategy()));
TypeProvider typeProvider = TypeProvider.copyOf(allFragments.stream().flatMap(f -> f.getSymbols().entrySet().stream()).distinct().collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)));
builder.append(new PlanPrinter(fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), tableInfoSupplier, valuePrinter, fragment.getStatsAndCosts(), planNodeStats, metadata).toText(verbose, 1)).append("\n");
return builder.toString();
}
use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.
the class PlanPrinter method textDistributedPlan.
public static String textDistributedPlan(StageInfo outputStageInfo, ValuePrinter valuePrinter, boolean verbose, Metadata metadata) {
Map<PlanNodeId, TableInfo> tableInfos = getAllStages(Optional.of(outputStageInfo)).stream().map(StageInfo::getTables).map(Map::entrySet).flatMap(Collection::stream).collect(toImmutableMap(Entry::getKey, Entry::getValue));
StringBuilder builder = new StringBuilder();
List<StageInfo> allStages = getAllStages(Optional.of(outputStageInfo));
List<PlanFragment> allFragments = allStages.stream().map(StageInfo::getPlan).collect(toImmutableList());
Map<PlanNodeId, PlanNodeStats> aggregatedStats = aggregateStageStats(allStages);
for (StageInfo stageInfo : allStages) {
builder.append(formatFragment(tableScanNode -> tableInfos.get(tableScanNode.getId()), valuePrinter, stageInfo.getPlan(), Optional.of(stageInfo), Optional.of(aggregatedStats), verbose, allFragments, metadata));
}
return builder.toString();
}
use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.
the class TestHiveDistributedJoinQueriesWithDynamicFiltering method searchScanFilterAndProjectOperatorStats.
private OperatorStats searchScanFilterAndProjectOperatorStats(QueryId queryId, String tableName) {
DistributedQueryRunner runner = (DistributedQueryRunner) getQueryRunner();
Plan plan = runner.getQueryPlan(queryId);
PlanNodeId nodeId = PlanNodeSearcher.searchFrom(plan.getRoot()).where(node -> {
if (!(node instanceof ProjectNode)) {
return false;
}
ProjectNode projectNode = (ProjectNode) node;
FilterNode filterNode = (FilterNode) projectNode.getSource();
TableScanNode tableScanNode = (TableScanNode) filterNode.getSource();
return tableName.equals(tableScanNode.getTable().getConnectorHandle().toString());
}).findOnlyElement().getId();
return runner.getCoordinator().getQueryManager().getFullQueryInfo(queryId).getQueryStats().getOperatorSummaries().stream().filter(summary -> nodeId.equals(summary.getPlanNodeId())).collect(MoreCollectors.onlyElement());
}
Aggregations