use of com.facebook.presto.client.StatementStats in project presto by prestodb.
the class StatusPrinter method printQueryInfo.
private void printQueryInfo(QueryResults results) {
StatementStats stats = results.getStats();
Duration wallTime = nanosSince(start);
// cap progress at 99%, otherwise it looks weird when the query is still running and it says 100%
int progressPercentage = (int) min(99, percentage(stats.getCompletedSplits(), stats.getTotalSplits()));
if (console.isRealTerminal()) {
// blank line
reprintLine("");
int terminalWidth = console.getWidth();
if (terminalWidth < 75) {
reprintLine("WARNING: Terminal");
reprintLine("must be at least");
reprintLine("80 characters wide");
reprintLine("");
reprintLine(stats.getState());
reprintLine(String.format("%s %d%%", formatTime(wallTime), progressPercentage));
return;
}
int nodes = stats.getNodes();
// Query 10, RUNNING, 1 node, 778 splits
String querySummary = String.format("Query %s, %s, %,d %s, %,d splits", results.getId(), stats.getState(), nodes, pluralize("node", nodes), stats.getTotalSplits());
reprintLine(querySummary);
String url = results.getInfoUri().toString();
if (debug && (url.length() < terminalWidth)) {
reprintLine(url);
}
if ((nodes == 0) || (stats.getTotalSplits() == 0)) {
return;
}
if (debug) {
// Splits: 620 queued, 34 running, 124 done
String splitsSummary = String.format("Splits: %,d queued, %,d running, %,d done", stats.getQueuedSplits(), stats.getRunningSplits(), stats.getCompletedSplits());
reprintLine(splitsSummary);
// CPU Time: 56.5s total, 36.4K rows/s, 4.44MB/s, 60% active
Duration cpuTime = millis(stats.getCpuTimeMillis());
String cpuTimeSummary = String.format("CPU Time: %.1fs total, %5s rows/s, %8s, %d%% active", cpuTime.getValue(SECONDS), formatCountRate(stats.getProcessedRows(), cpuTime, false), formatDataRate(bytes(stats.getProcessedBytes()), cpuTime, true), (int) percentage(stats.getCpuTimeMillis(), stats.getWallTimeMillis()));
reprintLine(cpuTimeSummary);
double parallelism = cpuTime.getValue(MILLISECONDS) / wallTime.getValue(MILLISECONDS);
// Per Node: 3.5 parallelism, 83.3K rows/s, 0.7 MB/s
String perNodeSummary = String.format("Per Node: %.1f parallelism, %5s rows/s, %8s", parallelism / nodes, formatCountRate((double) stats.getProcessedRows() / nodes, wallTime, false), formatDataRate(bytes(stats.getProcessedBytes() / nodes), wallTime, true));
reprintLine(perNodeSummary);
reprintLine(String.format("Parallelism: %.1f", parallelism));
}
// otherwise handled above
verify(terminalWidth >= 75);
// progress bar is 17-42 characters wide
int progressWidth = (min(terminalWidth, 100) - 75) + 17;
if (stats.isScheduled()) {
String progressBar = formatProgressBar(progressWidth, stats.getCompletedSplits(), max(0, stats.getRunningSplits()), stats.getTotalSplits());
// 0:17 [ 103MB, 802K rows] [5.74MB/s, 44.9K rows/s] [=====>> ] 10%
String progressLine = String.format("%s [%5s rows, %6s] [%5s rows/s, %8s] [%s] %d%%", formatTime(wallTime), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), true), formatCountRate(stats.getProcessedRows(), wallTime, false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true), progressBar, progressPercentage);
reprintLine(progressLine);
} else {
String progressBar = formatProgressBar(progressWidth, Ints.saturatedCast(nanosSince(start).roundTo(SECONDS)));
// 0:17 [ 103MB, 802K rows] [5.74MB/s, 44.9K rows/s] [ <=> ]
String progressLine = String.format("%s [%5s rows, %6s] [%5s rows/s, %8s] [%s]", formatTime(wallTime), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), true), formatCountRate(stats.getProcessedRows(), wallTime, false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true), progressBar);
reprintLine(progressLine);
}
// todo Mem: 1949M shared, 7594M private
// blank line
reprintLine("");
// STAGE S ROWS RPS BYTES BPS QUEUED RUN DONE
String stagesHeader = String.format("%10s%1s %5s %6s %5s %7s %6s %5s %5s", "STAGE", "S", "ROWS", "ROWS/s", "BYTES", "BYTES/s", "QUEUED", "RUN", "DONE");
reprintLine(stagesHeader);
printStageTree(stats.getRootStage(), "", new AtomicInteger());
} else {
// Query 31 [S] i[2.7M 67.3MB 62.7MBps] o[35 6.1KB 1KBps] splits[252/16/380]
String querySummary = String.format("Query %s [%s] i[%s %s %s] o[%s %s %s] splits[%,d/%,d/%,d]", results.getId(), stats.getState(), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, false), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, false), stats.getQueuedSplits(), stats.getRunningSplits(), stats.getCompletedSplits());
reprintLine(querySummary);
}
}
use of com.facebook.presto.client.StatementStats in project presto by prestodb.
the class PrestoSparkQueryExecutionFactory method createPrestoSparkQueryInfo.
private static PrestoSparkQueryStatusInfo createPrestoSparkQueryInfo(QueryInfo queryInfo, Optional<PlanAndMore> planAndMore, WarningCollector warningCollector, OptionalLong updateCount) {
StatementStats stats = toStatementStats(queryInfo);
// nullify stage stats to keep the object slim
stats = new StatementStats(stats.getState(), stats.isWaitingForPrerequisites(), stats.isQueued(), stats.isScheduled(), stats.getNodes(), stats.getTotalSplits(), stats.getQueuedSplits(), stats.getRunningSplits(), stats.getCompletedSplits(), stats.getCpuTimeMillis(), stats.getWallTimeMillis(), stats.getWaitingForPrerequisitesTimeMillis(), stats.getQueuedTimeMillis(), stats.getElapsedTimeMillis(), stats.getProcessedRows(), stats.getProcessedBytes(), stats.getPeakMemoryBytes(), stats.getPeakTotalMemoryBytes(), stats.getPeakTaskTotalMemoryBytes(), stats.getSpilledBytes(), null, null);
return new PrestoSparkQueryStatusInfo(queryInfo.getQueryId().getId(), planAndMore.map(PrestoSparkQueryExecutionFactory::getOutputColumns), stats, Optional.ofNullable(queryInfo.getFailureInfo()).map(PrestoSparkQueryExecutionFactory::toQueryError), warningCollector.getWarnings(), planAndMore.flatMap(PlanAndMore::getUpdateType), updateCount);
}
use of com.facebook.presto.client.StatementStats in project presto by prestodb.
the class StatusPrinter method printFinalInfo.
public void printFinalInfo() {
Duration wallTime = nanosSince(start);
QueryStatusInfo results = client.finalStatusInfo();
StatementStats stats = results.getStats();
int nodes = stats.getNodes();
if ((nodes == 0) || (stats.getTotalSplits() == 0)) {
return;
}
// blank line
out.println();
// Query 12, FINISHED, 1 node
String querySummary = format("Query %s, %s, %,d %s", results.getId(), stats.getState(), nodes, pluralize("node", nodes));
out.println(querySummary);
if (debug) {
out.println(results.getInfoUri().toString());
}
// Splits: 1000 total, 842 done (84.20%)
String splitsSummary = format("Splits: %,d total, %,d done (%.2f%%)", stats.getTotalSplits(), stats.getCompletedSplits(), stats.getProgressPercentage().orElse(0.0));
out.println(splitsSummary);
if (debug) {
// CPU Time: 565.2s total, 26K rows/s, 3.85MB/s
Duration cpuTime = millis(stats.getCpuTimeMillis());
String cpuTimeSummary = format("CPU Time: %.1fs total, %5s rows/s, %8s, %d%% active", cpuTime.getValue(SECONDS), formatCountRate(stats.getProcessedRows(), cpuTime, false), formatDataRate(bytes(stats.getProcessedBytes()), cpuTime, true), (int) percentage(stats.getCpuTimeMillis(), stats.getWallTimeMillis()));
out.println(cpuTimeSummary);
double parallelism = cpuTime.getValue(MILLISECONDS) / wallTime.getValue(MILLISECONDS);
// Per Node: 3.5 parallelism, 83.3K rows/s, 0.7 MB/s
String perNodeSummary = format("Per Node: %.1f parallelism, %5s rows/s, %8s", parallelism / nodes, formatCountRate((double) stats.getProcessedRows() / nodes, wallTime, false), formatDataRate(bytes(stats.getProcessedBytes() / nodes), wallTime, true));
reprintLine(perNodeSummary);
// Parallelism: 5.3
out.println(format("Parallelism: %.1f", parallelism));
// Peak User Memory: 1.97GB
reprintLine("Peak User Memory: " + formatDataSize(bytes(stats.getPeakMemoryBytes()), true));
// Peak Total Memory: 1.98GB
reprintLine("Peak Total Memory: " + formatDataSize(bytes(stats.getPeakTotalMemoryBytes()), true));
// Peak Task Total Memory: 1.99GB
reprintLine("Peak Task Total Memory: " + formatDataSize(bytes(stats.getPeakTaskTotalMemoryBytes()), true));
// Spilled Data: 20GB
if (stats.getSpilledBytes() > 0) {
reprintLine("Spilled: " + formatDataSize(bytes(stats.getSpilledBytes()), true));
}
// bytesFromCache: sum=2K count=2 min=1K max=1K
if (stats.getRuntimeStats() != null) {
stats.getRuntimeStats().getMetrics().values().stream().sorted(Comparator.comparing(RuntimeMetric::getName)).forEach(metric -> reprintLine(format("%s: sum=%s count=%s min=%s max=%s", metric.getName(), autoFormatMetricValue(metric.getName(), metric.getSum()), formatCount(metric.getCount()), autoFormatMetricValue(metric.getName(), metric.getMin()), autoFormatMetricValue(metric.getName(), metric.getMax()))));
}
}
// 0:32 [2.12GB, 15M rows] [67MB/s, 463K rows/s]
String statsLine = format("%s [%s rows, %s] [%s rows/s, %s]", formatTime(wallTime), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), true), formatCountRate(stats.getProcessedRows(), wallTime, false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true));
out.println(statsLine);
// blank line
out.println();
}
use of com.facebook.presto.client.StatementStats in project presto by prestodb.
the class BenchmarkQueryRunner method execute.
@SuppressWarnings("AssignmentToForLoopParameter")
public BenchmarkQueryResult execute(Suite suite, ClientSession session, BenchmarkQuery query) {
failures = 0;
for (int i = 0; i < warm; ) {
try {
execute(session, query.getName(), query.getSql());
i++;
failures = 0;
} catch (BenchmarkDriverExecutionException e) {
return failResult(suite, query, e.getCause().getMessage());
} catch (Exception e) {
handleFailure(e);
}
}
double[] wallTimeNanos = new double[runs];
double[] processCpuTimeNanos = new double[runs];
double[] queryCpuTimeNanos = new double[runs];
for (int i = 0; i < runs; ) {
try {
long startCpuTime = getTotalCpuTime();
long startWallTime = System.nanoTime();
StatementStats statementStats = execute(session, query.getName(), query.getSql());
long endWallTime = System.nanoTime();
long endCpuTime = getTotalCpuTime();
wallTimeNanos[i] = endWallTime - startWallTime;
processCpuTimeNanos[i] = endCpuTime - startCpuTime;
queryCpuTimeNanos[i] = MILLISECONDS.toNanos(statementStats.getCpuTimeMillis());
i++;
failures = 0;
} catch (BenchmarkDriverExecutionException e) {
return failResult(suite, query, e.getCause().getMessage());
} catch (Exception e) {
handleFailure(e);
}
}
return passResult(suite, query, new Stat(wallTimeNanos), new Stat(processCpuTimeNanos), new Stat(queryCpuTimeNanos));
}
use of com.facebook.presto.client.StatementStats in project presto by prestodb.
the class StatusPrinter method printQueryInfo.
private void printQueryInfo(QueryStatusInfo results, WarningsPrinter warningsPrinter) {
StatementStats stats = results.getStats();
Duration wallTime = nanosSince(start);
// cap progress at 99%, otherwise it looks weird when the query is still running and it says 100%
int progressPercentage = (int) min(99, stats.getProgressPercentage().orElse(0.0));
if (console.isRealTerminal()) {
// blank line
reprintLine("");
int terminalWidth = console.getWidth();
if (terminalWidth < 75) {
reprintLine("WARNING: Terminal");
reprintLine("must be at least");
reprintLine("80 characters wide");
reprintLine("");
reprintLine(stats.getState());
reprintLine(format("%s %d%%", formatTime(wallTime), progressPercentage));
return;
}
int nodes = stats.getNodes();
// Query 10, RUNNING, 1 node, 778 splits
String querySummary = format("Query %s, %s, %,d %s, %,d splits", results.getId(), stats.getState(), nodes, pluralize("node", nodes), stats.getTotalSplits());
reprintLine(querySummary);
String url = results.getInfoUri().toString();
if (debug && (url.length() < terminalWidth)) {
reprintLine(url);
}
if ((nodes == 0) || (stats.getTotalSplits() == 0)) {
return;
}
if (debug) {
// Splits: 620 queued, 34 running, 124 done
String splitsSummary = format("Splits: %,d queued, %,d running, %,d done", stats.getQueuedSplits(), stats.getRunningSplits(), stats.getCompletedSplits());
reprintLine(splitsSummary);
// CPU Time: 56.5s total, 36.4K rows/s, 4.44MB/s, 60% active
Duration cpuTime = millis(stats.getCpuTimeMillis());
String cpuTimeSummary = format("CPU Time: %.1fs total, %5s rows/s, %8s, %d%% active", cpuTime.getValue(SECONDS), formatCountRate(stats.getProcessedRows(), cpuTime, false), formatDataRate(bytes(stats.getProcessedBytes()), cpuTime, true), (int) percentage(stats.getCpuTimeMillis(), stats.getWallTimeMillis()));
reprintLine(cpuTimeSummary);
double parallelism = cpuTime.getValue(MILLISECONDS) / wallTime.getValue(MILLISECONDS);
// Per Node: 3.5 parallelism, 83.3K rows/s, 0.7 MB/s
String perNodeSummary = format("Per Node: %.1f parallelism, %5s rows/s, %8s", parallelism / nodes, formatCountRate((double) stats.getProcessedRows() / nodes, wallTime, false), formatDataRate(bytes(stats.getProcessedBytes() / nodes), wallTime, true));
reprintLine(perNodeSummary);
// Parallelism: 5.3
reprintLine(format("Parallelism: %.1f", parallelism));
// Peak Memory: 1.97GB
reprintLine("Peak Memory: " + formatDataSize(bytes(stats.getPeakMemoryBytes()), true));
// Peak Total Memory: 1.98GB
reprintLine("Peak Total Memory: " + formatDataSize(bytes(stats.getPeakTotalMemoryBytes()), true));
// Peak Task Total Memory: 1.99GB
reprintLine("Peak Task Total Memory: " + formatDataSize(bytes(stats.getPeakTaskTotalMemoryBytes()), true));
// Spilled Data: 20GB
if (stats.getSpilledBytes() > 0) {
reprintLine("Spilled: " + formatDataSize(bytes(stats.getSpilledBytes()), true));
}
if (stats.getRuntimeStats() != null) {
stats.getRuntimeStats().getMetrics().values().stream().sorted(Comparator.comparing(RuntimeMetric::getName)).forEach(metric -> reprintLine(format("%s: sum=%s count=%s min=%s max=%s", metric.getName(), autoFormatMetricValue(metric.getName(), metric.getSum()), formatCount(metric.getCount()), autoFormatMetricValue(metric.getName(), metric.getMin()), autoFormatMetricValue(metric.getName(), metric.getMax()))));
}
}
// otherwise handled above
verify(terminalWidth >= 75);
// progress bar is 17-42 characters wide
int progressWidth = (min(terminalWidth, 100) - 75) + 17;
String formattedWallTime = formatTime(wallTime);
if (formattedWallTime.length() > 5) {
// fix overflowed progress bar for queries running over 100 minutes
progressWidth -= formattedWallTime.length() - 5;
}
if (stats.isScheduled()) {
String progressBar = formatProgressBar(progressWidth, stats.getCompletedSplits(), max(0, stats.getRunningSplits()), stats.getTotalSplits());
// 0:17 [ 103MB, 802K rows] [5.74MB/s, 44.9K rows/s] [=====>> ] 10%
String progressLine = format("%s [%5s rows, %6s] [%5s rows/s, %8s] [%s] %d%%", formattedWallTime, formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), true), formatCountRate(stats.getProcessedRows(), wallTime, false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true), progressBar, progressPercentage);
reprintLine(progressLine);
} else {
String progressBar = formatProgressBar(progressWidth, Ints.saturatedCast(nanosSince(start).roundTo(SECONDS)));
// 0:17 [ 103MB, 802K rows] [5.74MB/s, 44.9K rows/s] [ <=> ]
String progressLine = format("%s [%5s rows, %6s] [%5s rows/s, %8s] [%s]", formattedWallTime, formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), true), formatCountRate(stats.getProcessedRows(), wallTime, false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true), progressBar);
reprintLine(progressLine);
}
// blank line
reprintLine("");
// STAGE S ROWS RPS BYTES BPS QUEUED RUN DONE
String stagesHeader = format("%10s%1s %5s %6s %5s %7s %6s %5s %5s", "STAGE", "S", "ROWS", "ROWS/s", "BYTES", "BYTES/s", "QUEUED", "RUN", "DONE");
reprintLine(stagesHeader);
printStageTree(stats.getRootStage(), "", new AtomicInteger());
} else {
// Query 31 [S] i[2.7M 67.3MB 62.7MBps] o[35 6.1KB 1KBps] splits[252/16/380]
String querySummary = format("Query %s [%s] i[%s %s %s] o[%s %s %s] splits[%,d/%,d/%,d]", results.getId(), stats.getState(), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, false), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, false), stats.getQueuedSplits(), stats.getRunningSplits(), stats.getCompletedSplits());
reprintLine(querySummary);
}
warningsPrinter.print(results.getWarnings(), false, false);
}
Aggregations