use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.
the class PlanPrinter method formatFragment.
private static String formatFragment(Function<TableScanNode, TableInfo> tableInfoSupplier, ValuePrinter valuePrinter, PlanFragment fragment, Optional<StageInfo> stageInfo, Optional<Map<PlanNodeId, PlanNodeStats>> planNodeStats, boolean verbose, List<PlanFragment> allFragments, Metadata metadata) {
StringBuilder builder = new StringBuilder();
builder.append(format("Fragment %s [%s]\n", fragment.getId(), fragment.getPartitioning()));
if (stageInfo.isPresent()) {
StageStats stageStats = stageInfo.get().getStageStats();
double avgPositionsPerTask = stageInfo.get().getTasks().stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN);
double squaredDifferences = stageInfo.get().getTasks().stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum();
double sdAmongTasks = Math.sqrt(squaredDifferences / stageInfo.get().getTasks().size());
builder.append(indentString(1)).append(format("CPU: %s, Scheduled: %s, Input: %s (%s); per task: avg.: %s std.dev.: %s, Output: %s (%s)\n", stageStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), stageStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), formatPositions(stageStats.getProcessedInputPositions()), stageStats.getProcessedInputDataSize(), formatDouble(avgPositionsPerTask), formatDouble(sdAmongTasks), formatPositions(stageStats.getOutputPositions()), stageStats.getOutputDataSize()));
}
PartitioningScheme partitioningScheme = fragment.getPartitioningScheme();
builder.append(indentString(1)).append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout())));
boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny();
List<String> arguments = partitioningScheme.getPartitioning().getArguments().stream().map(argument -> {
if (argument.isConstant()) {
NullableValue constant = argument.getConstant();
String printableValue = valuePrinter.castToVarchar(constant.getType(), constant.getValue());
return constant.getType().getDisplayName() + "(" + printableValue + ")";
}
return argument.getColumn().toString();
}).collect(toImmutableList());
builder.append(indentString(1));
if (replicateNullsAndAny) {
builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
} else {
builder.append(format("Output partitioning: %s [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
}
builder.append(indentString(1)).append(format("Stage Execution Strategy: %s\n", fragment.getStageExecutionDescriptor().getStageExecutionStrategy()));
TypeProvider typeProvider = TypeProvider.copyOf(allFragments.stream().flatMap(f -> f.getSymbols().entrySet().stream()).distinct().collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)));
builder.append(new PlanPrinter(fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), tableInfoSupplier, valuePrinter, fragment.getStatsAndCosts(), planNodeStats, metadata).toText(verbose, 1)).append("\n");
return builder.toString();
}
use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.
the class PlanPrinter method textDistributedPlan.
public static String textDistributedPlan(StageInfo outputStageInfo, ValuePrinter valuePrinter, boolean verbose, Metadata metadata) {
Map<PlanNodeId, TableInfo> tableInfos = getAllStages(Optional.of(outputStageInfo)).stream().map(StageInfo::getTables).map(Map::entrySet).flatMap(Collection::stream).collect(toImmutableMap(Entry::getKey, Entry::getValue));
StringBuilder builder = new StringBuilder();
List<StageInfo> allStages = getAllStages(Optional.of(outputStageInfo));
List<PlanFragment> allFragments = allStages.stream().map(StageInfo::getPlan).collect(toImmutableList());
Map<PlanNodeId, PlanNodeStats> aggregatedStats = aggregateStageStats(allStages);
for (StageInfo stageInfo : allStages) {
builder.append(formatFragment(tableScanNode -> tableInfos.get(tableScanNode.getId()), valuePrinter, stageInfo.getPlan(), Optional.of(stageInfo), Optional.of(aggregatedStats), verbose, allFragments, metadata));
}
return builder.toString();
}
use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.
the class Query method globalUniqueNodes.
public static Set<String> globalUniqueNodes(StageInfo stageInfo, boolean getSpilledNodes) {
boolean isSpilledNode = false;
if (stageInfo == null) {
return ImmutableSet.of();
}
ImmutableSet.Builder<String> nodes = ImmutableSet.builder();
for (TaskInfo task : stageInfo.getTasks()) {
if (getSpilledNodes) {
for (PipelineStats pipeline : task.getStats().getPipelines()) {
isSpilledNode = pipeline.getOperatorSummaries().stream().filter(operatorStats -> operatorStats.getSpilledDataSize().toBytes() > 0).collect(Collectors.toList()).size() > 0;
if (isSpilledNode) {
break;
}
}
}
// todo add nodeId to TaskInfo
if (isSpilledNode || !getSpilledNodes) {
URI uri = task.getTaskStatus().getSelf();
nodes.add(uri.getHost() + ":" + uri.getPort());
}
}
for (StageInfo subStage : stageInfo.getSubStages()) {
nodes.addAll(globalUniqueNodes(subStage, getSpilledNodes));
}
return nodes.build();
}
use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.
the class Query method toStatementStats.
private StatementStats toStatementStats(QueryInfo queryInfo) {
QueryStats queryStats = queryInfo.getQueryStats();
// Dont print any more stats for Async Query,
StageInfo outputStage = queryInfo.isRunningAsync() ? null : queryInfo.getOutputStage().orElse(null);
return StatementStats.builder().setState(queryInfo.getState().toString()).setQueued(queryInfo.getState() == QueryState.QUEUED).setScheduled(queryInfo.isScheduled()).setNodes(globalUniqueNodes(outputStage, false).size()).setTotalSplits(queryStats.getTotalDrivers()).setQueuedSplits(queryStats.getQueuedDrivers()).setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()).setCompletedSplits(queryStats.getCompletedDrivers()).setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()).setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()).setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()).setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()).setProcessedRows(queryStats.getRawInputPositions()).setProcessedBytes(queryStats.getRawInputDataSize().toBytes()).setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()).setSpilledBytes(queryStats.getSpilledDataSize().toBytes()).setSpilledReadTimeMillis(queryStats.getSpilledReadTime().toMillis()).setSpilledWriteTimeMillis(queryStats.getSpilledWriteTime().toMillis()).setSpilledNodes(globalUniqueNodes(outputStage, true).size()).setRootStage(toStageStats(outputStage)).setSnapshotStats(toSnapshotStats(queryInfo.getQueryId())).build();
}
use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.
the class ExplainAnalyzeOperator method getSubStages.
private static void getSubStages(StageId stageId, StageInfo rootStage, ImmutableList.Builder<StageInfo> collector, boolean add) {
boolean status = add;
if (rootStage.getStageId().equals(stageId)) {
status = true;
}
List<StageInfo> subStages = rootStage.getSubStages();
for (StageInfo subStage : subStages) {
getSubStages(stageId, subStage, collector, status);
}
if (status && !rootStage.getStageId().equals(stageId)) {
collector.add(rootStage);
}
}
Aggregations