use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.
the class OperatorStats method add.
public OperatorStats add(Iterable<OperatorStats> operators) {
long totalDrivers = this.totalDrivers;
long addInputCalls = this.addInputCalls;
long addInputWall = this.addInputWall.roundTo(NANOSECONDS);
long addInputCpu = this.addInputCpu.roundTo(NANOSECONDS);
long addInputAllocation = this.addInputAllocation.toBytes();
long rawInputDataSize = this.rawInputDataSize.toBytes();
long rawInputPositions = this.rawInputPositions;
long inputDataSize = this.inputDataSize.toBytes();
long inputPositions = this.inputPositions;
double sumSquaredInputPositions = this.sumSquaredInputPositions;
long getOutputCalls = this.getOutputCalls;
long getOutputWall = this.getOutputWall.roundTo(NANOSECONDS);
long getOutputCpu = this.getOutputCpu.roundTo(NANOSECONDS);
long getOutputAllocation = this.getOutputAllocation.toBytes();
long outputDataSize = this.outputDataSize.toBytes();
long outputPositions = this.outputPositions;
long physicalWrittenDataSize = this.physicalWrittenDataSize.toBytes();
long additionalCpu = this.additionalCpu.roundTo(NANOSECONDS);
long blockedWall = this.blockedWall.roundTo(NANOSECONDS);
long finishCalls = this.finishCalls;
long finishWall = this.finishWall.roundTo(NANOSECONDS);
long finishCpu = this.finishCpu.roundTo(NANOSECONDS);
long finishAllocation = this.finishAllocation.toBytes();
long memoryReservation = this.userMemoryReservation.toBytes();
long revocableMemoryReservation = this.revocableMemoryReservation.toBytes();
long systemMemoryReservation = this.systemMemoryReservation.toBytes();
long peakUserMemory = this.peakUserMemoryReservation.toBytes();
long peakSystemMemory = this.peakSystemMemoryReservation.toBytes();
long peakTotalMemory = this.peakTotalMemoryReservation.toBytes();
long spilledDataSize = this.spilledDataSize.toBytes();
Optional<BlockedReason> blockedReason = this.blockedReason;
RuntimeStats runtimeStats = RuntimeStats.copyOf(this.runtimeStats);
Mergeable<OperatorInfo> base = getMergeableInfoOrNull(info);
for (OperatorStats operator : operators) {
checkArgument(operator.getOperatorId() == operatorId, "Expected operatorId to be %s but was %s", operatorId, operator.getOperatorId());
totalDrivers += operator.totalDrivers;
addInputCalls += operator.getAddInputCalls();
addInputWall += operator.getAddInputWall().roundTo(NANOSECONDS);
addInputCpu += operator.getAddInputCpu().roundTo(NANOSECONDS);
addInputAllocation += operator.getAddInputAllocation().toBytes();
rawInputDataSize += operator.getRawInputDataSize().toBytes();
rawInputPositions += operator.getRawInputPositions();
inputDataSize += operator.getInputDataSize().toBytes();
inputPositions += operator.getInputPositions();
sumSquaredInputPositions += operator.getSumSquaredInputPositions();
getOutputCalls += operator.getGetOutputCalls();
getOutputWall += operator.getGetOutputWall().roundTo(NANOSECONDS);
getOutputCpu += operator.getGetOutputCpu().roundTo(NANOSECONDS);
getOutputAllocation += operator.getGetOutputAllocation().toBytes();
outputDataSize += operator.getOutputDataSize().toBytes();
outputPositions += operator.getOutputPositions();
physicalWrittenDataSize += operator.getPhysicalWrittenDataSize().toBytes();
finishCalls += operator.getFinishCalls();
finishWall += operator.getFinishWall().roundTo(NANOSECONDS);
finishCpu += operator.getFinishCpu().roundTo(NANOSECONDS);
finishAllocation += operator.getFinishAllocation().toBytes();
additionalCpu += operator.getAdditionalCpu().roundTo(NANOSECONDS);
blockedWall += operator.getBlockedWall().roundTo(NANOSECONDS);
memoryReservation += operator.getUserMemoryReservation().toBytes();
revocableMemoryReservation += operator.getRevocableMemoryReservation().toBytes();
systemMemoryReservation += operator.getSystemMemoryReservation().toBytes();
peakUserMemory = max(peakUserMemory, operator.getPeakUserMemoryReservation().toBytes());
peakSystemMemory = max(peakSystemMemory, operator.getPeakSystemMemoryReservation().toBytes());
peakTotalMemory = max(peakTotalMemory, operator.getPeakTotalMemoryReservation().toBytes());
spilledDataSize += operator.getSpilledDataSize().toBytes();
if (operator.getBlockedReason().isPresent()) {
blockedReason = operator.getBlockedReason();
}
OperatorInfo info = operator.getInfo();
if (base != null && info != null && base.getClass() == info.getClass()) {
base = mergeInfo(base, info);
}
runtimeStats.mergeWith(operator.getRuntimeStats());
}
return new OperatorStats(stageId, stageExecutionId, pipelineId, operatorId, planNodeId, operatorType, totalDrivers, addInputCalls, succinctNanos(addInputWall), succinctNanos(addInputCpu), succinctBytes(addInputAllocation), succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(inputDataSize), inputPositions, sumSquaredInputPositions, getOutputCalls, succinctNanos(getOutputWall), succinctNanos(getOutputCpu), succinctBytes(getOutputAllocation), succinctBytes(outputDataSize), outputPositions, succinctBytes(physicalWrittenDataSize), succinctNanos(additionalCpu), succinctNanos(blockedWall), finishCalls, succinctNanos(finishWall), succinctNanos(finishCpu), succinctBytes(finishAllocation), succinctBytes(memoryReservation), succinctBytes(revocableMemoryReservation), succinctBytes(systemMemoryReservation), succinctBytes(peakUserMemory), succinctBytes(peakSystemMemory), succinctBytes(peakTotalMemory), succinctBytes(spilledDataSize), blockedReason, (OperatorInfo) base, runtimeStats);
}
use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.
the class StageExecutionInfo method create.
public static StageExecutionInfo create(StageExecutionId stageExecutionId, StageExecutionState state, Optional<ExecutionFailureInfo> failureInfo, List<TaskInfo> taskInfos, DateTime schedulingComplete, DistributionSnapshot getSplitDistribution, DataSize peakUserMemoryReservation, DataSize peakNodeTotalMemoryReservation, int finishedLifespans, int totalLifespans) {
int totalTasks = taskInfos.size();
int runningTasks = 0;
int completedTasks = 0;
int totalDrivers = 0;
int queuedDrivers = 0;
int runningDrivers = 0;
int blockedDrivers = 0;
int completedDrivers = 0;
double cumulativeUserMemory = 0;
double cumulativeTotalMemory = 0;
long userMemoryReservation = 0;
long totalMemoryReservation = 0;
long totalScheduledTime = 0;
long totalCpuTime = 0;
long retriedCpuTime = 0;
long totalBlockedTime = 0;
long totalAllocation = 0;
long rawInputDataSize = 0;
long rawInputPositions = 0;
long processedInputDataSize = 0;
long processedInputPositions = 0;
long bufferedDataSize = 0;
long outputDataSize = 0;
long outputPositions = 0;
long physicalWrittenDataSize = 0;
int fullGcCount = 0;
int fullGcTaskCount = 0;
int minFullGcSec = 0;
int maxFullGcSec = 0;
int totalFullGcSec = 0;
boolean fullyBlocked = true;
Set<BlockedReason> blockedReasons = new HashSet<>();
Map<String, OperatorStats> operatorToStats = new HashMap<>();
RuntimeStats mergedRuntimeStats = new RuntimeStats();
mergedRuntimeStats.addMetricValueIgnoreZero(GET_SPLITS_TIME_NANOS, (long) getSplitDistribution.getTotal());
for (TaskInfo taskInfo : taskInfos) {
TaskState taskState = taskInfo.getTaskStatus().getState();
if (taskState.isDone()) {
completedTasks++;
} else {
runningTasks++;
}
TaskStats taskStats = taskInfo.getStats();
totalDrivers += taskStats.getTotalDrivers();
queuedDrivers += taskStats.getQueuedDrivers();
runningDrivers += taskStats.getRunningDrivers();
blockedDrivers += taskStats.getBlockedDrivers();
completedDrivers += taskStats.getCompletedDrivers();
cumulativeUserMemory += taskStats.getCumulativeUserMemory();
cumulativeTotalMemory += taskStats.getCumulativeTotalMemory();
long taskUserMemory = taskStats.getUserMemoryReservationInBytes();
long taskSystemMemory = taskStats.getSystemMemoryReservationInBytes();
userMemoryReservation += taskUserMemory;
totalMemoryReservation += taskUserMemory + taskSystemMemory;
totalScheduledTime += taskStats.getTotalScheduledTimeInNanos();
totalCpuTime += taskStats.getTotalCpuTimeInNanos();
if (state == FINISHED && taskInfo.getTaskStatus().getState() == TaskState.FAILED) {
retriedCpuTime += taskStats.getTotalCpuTimeInNanos();
}
totalBlockedTime += taskStats.getTotalBlockedTimeInNanos();
if (!taskState.isDone()) {
fullyBlocked &= taskStats.isFullyBlocked();
blockedReasons.addAll(taskStats.getBlockedReasons());
}
totalAllocation += taskStats.getTotalAllocationInBytes();
rawInputDataSize += taskStats.getRawInputDataSizeInBytes();
rawInputPositions += taskStats.getRawInputPositions();
processedInputDataSize += taskStats.getProcessedInputDataSizeInBytes();
processedInputPositions += taskStats.getProcessedInputPositions();
bufferedDataSize += taskInfo.getOutputBuffers().getTotalBufferedBytes();
outputDataSize += taskStats.getOutputDataSizeInBytes();
outputPositions += taskStats.getOutputPositions();
physicalWrittenDataSize += taskStats.getPhysicalWrittenDataSizeInBytes();
fullGcCount += taskStats.getFullGcCount();
fullGcTaskCount += taskStats.getFullGcCount() > 0 ? 1 : 0;
int gcSec = toIntExact(MILLISECONDS.toSeconds(taskStats.getFullGcTimeInMillis()));
totalFullGcSec += gcSec;
minFullGcSec = min(minFullGcSec, gcSec);
maxFullGcSec = max(maxFullGcSec, gcSec);
for (PipelineStats pipeline : taskStats.getPipelines()) {
for (OperatorStats operatorStats : pipeline.getOperatorSummaries()) {
String id = pipeline.getPipelineId() + "." + operatorStats.getOperatorId();
operatorToStats.compute(id, (k, v) -> v == null ? operatorStats : v.add(operatorStats));
}
}
mergedRuntimeStats.mergeWith(taskStats.getRuntimeStats());
mergedRuntimeStats.addMetricValue(RuntimeMetricName.DRIVER_COUNT_PER_TASK, taskStats.getTotalDrivers());
mergedRuntimeStats.addMetricValue(RuntimeMetricName.TASK_ELAPSED_TIME_NANOS, taskStats.getElapsedTimeInNanos());
mergedRuntimeStats.addMetricValueIgnoreZero(RuntimeMetricName.TASK_QUEUED_TIME_NANOS, taskStats.getQueuedTimeInNanos());
mergedRuntimeStats.addMetricValue(RuntimeMetricName.TASK_SCHEDULED_TIME_NANOS, taskStats.getTotalScheduledTimeInNanos());
mergedRuntimeStats.addMetricValueIgnoreZero(RuntimeMetricName.TASK_BLOCKED_TIME_NANOS, taskStats.getTotalBlockedTimeInNanos());
}
StageExecutionStats stageExecutionStats = new StageExecutionStats(schedulingComplete, getSplitDistribution, totalTasks, runningTasks, completedTasks, totalLifespans, finishedLifespans, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, cumulativeTotalMemory, succinctBytes(userMemoryReservation), succinctBytes(totalMemoryReservation), peakUserMemoryReservation, peakNodeTotalMemoryReservation, succinctDuration(totalScheduledTime, NANOSECONDS), succinctDuration(totalCpuTime, NANOSECONDS), succinctDuration(retriedCpuTime, NANOSECONDS), succinctDuration(totalBlockedTime, NANOSECONDS), fullyBlocked && runningTasks > 0, blockedReasons, succinctBytes(totalAllocation), succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(bufferedDataSize), succinctBytes(outputDataSize), outputPositions, succinctBytes(physicalWrittenDataSize), new StageGcStatistics(stageExecutionId.getStageId().getId(), stageExecutionId.getId(), totalTasks, fullGcTaskCount, minFullGcSec, maxFullGcSec, totalFullGcSec, (int) (1.0 * totalFullGcSec / fullGcCount)), ImmutableList.copyOf(operatorToStats.values()), mergedRuntimeStats);
return new StageExecutionInfo(state, stageExecutionStats, taskInfos, failureInfo);
}
use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.
the class QueryStats method create.
public static QueryStats create(QueryStateTimer queryStateTimer, Optional<StageInfo> rootStage, int peakRunningTasks, DataSize peakUserMemoryReservation, DataSize peakTotalMemoryReservation, DataSize peakTaskUserMemory, DataSize peakTaskTotalMemory, DataSize peakNodeTotalMemory, RuntimeStats runtimeStats) {
int totalTasks = 0;
int runningTasks = 0;
int completedTasks = 0;
int totalDrivers = 0;
int queuedDrivers = 0;
int runningDrivers = 0;
int blockedDrivers = 0;
int completedDrivers = 0;
double cumulativeUserMemory = 0;
double cumulativeTotalMemory = 0;
long userMemoryReservation = 0;
long totalMemoryReservation = 0;
long totalScheduledTime = 0;
long totalCpuTime = 0;
long retriedCpuTime = 0;
long totalBlockedTime = 0;
long totalAllocation = 0;
long rawInputDataSize = 0;
long rawInputPositions = 0;
long processedInputDataSize = 0;
long processedInputPositions = 0;
long outputDataSize = 0;
long outputPositions = 0;
long writtenOutputPositions = 0;
long writtenOutputLogicalDataSize = 0;
long writtenOutputPhysicalDataSize = 0;
long writtenIntermediatePhysicalDataSize = 0;
ImmutableList.Builder<StageGcStatistics> stageGcStatistics = ImmutableList.builder();
boolean fullyBlocked = rootStage.isPresent();
Set<BlockedReason> blockedReasons = new HashSet<>();
ImmutableList.Builder<OperatorStats> operatorStatsSummary = ImmutableList.builder();
boolean completeInfo = true;
RuntimeStats mergedRuntimeStats = RuntimeStats.copyOf(runtimeStats);
for (StageInfo stageInfo : getAllStages(rootStage)) {
StageExecutionStats stageExecutionStats = stageInfo.getLatestAttemptExecutionInfo().getStats();
totalTasks += stageExecutionStats.getTotalTasks();
runningTasks += stageExecutionStats.getRunningTasks();
completedTasks += stageExecutionStats.getCompletedTasks();
totalDrivers += stageExecutionStats.getTotalDrivers();
queuedDrivers += stageExecutionStats.getQueuedDrivers();
runningDrivers += stageExecutionStats.getRunningDrivers();
blockedDrivers += stageExecutionStats.getBlockedDrivers();
completedDrivers += stageExecutionStats.getCompletedDrivers();
cumulativeUserMemory += stageExecutionStats.getCumulativeUserMemory();
cumulativeTotalMemory += stageExecutionStats.getCumulativeTotalMemory();
userMemoryReservation += stageExecutionStats.getUserMemoryReservation().toBytes();
totalMemoryReservation += stageExecutionStats.getTotalMemoryReservation().toBytes();
totalScheduledTime += stageExecutionStats.getTotalScheduledTime().roundTo(MILLISECONDS);
totalCpuTime += stageExecutionStats.getTotalCpuTime().roundTo(MILLISECONDS);
retriedCpuTime += computeRetriedCpuTime(stageInfo);
totalBlockedTime += stageExecutionStats.getTotalBlockedTime().roundTo(MILLISECONDS);
if (!stageInfo.getLatestAttemptExecutionInfo().getState().isDone()) {
fullyBlocked &= stageExecutionStats.isFullyBlocked();
blockedReasons.addAll(stageExecutionStats.getBlockedReasons());
}
totalAllocation += stageExecutionStats.getTotalAllocation().toBytes();
if (stageInfo.getPlan().isPresent()) {
PlanFragment plan = stageInfo.getPlan().get();
if (!plan.getTableScanSchedulingOrder().isEmpty()) {
rawInputDataSize += stageExecutionStats.getRawInputDataSize().toBytes();
rawInputPositions += stageExecutionStats.getRawInputPositions();
processedInputDataSize += stageExecutionStats.getProcessedInputDataSize().toBytes();
processedInputPositions += stageExecutionStats.getProcessedInputPositions();
}
if (plan.isOutputTableWriterFragment()) {
writtenOutputPositions += stageExecutionStats.getOperatorSummaries().stream().filter(stats -> stats.getOperatorType().equals(TableWriterOperator.class.getSimpleName())).mapToLong(OperatorStats::getInputPositions).sum();
writtenOutputLogicalDataSize += stageExecutionStats.getOperatorSummaries().stream().filter(stats -> stats.getOperatorType().equals(TableWriterOperator.class.getSimpleName())).mapToLong(stats -> stats.getInputDataSize().toBytes()).sum();
writtenOutputPhysicalDataSize += stageExecutionStats.getPhysicalWrittenDataSize().toBytes();
} else {
writtenIntermediatePhysicalDataSize += stageExecutionStats.getPhysicalWrittenDataSize().toBytes();
}
}
stageGcStatistics.add(stageExecutionStats.getGcInfo());
completeInfo = completeInfo && stageInfo.isFinalStageInfo();
operatorStatsSummary.addAll(stageExecutionStats.getOperatorSummaries());
// We prepend each metric name with the stage id to avoid merging metrics across stages.
int stageId = stageInfo.getStageId().getId();
stageExecutionStats.getRuntimeStats().getMetrics().forEach((name, metric) -> {
String metricName = String.format("S%d-%s", stageId, name);
mergedRuntimeStats.mergeMetric(metricName, metric);
});
}
if (rootStage.isPresent()) {
StageExecutionStats outputStageStats = rootStage.get().getLatestAttemptExecutionInfo().getStats();
outputDataSize += outputStageStats.getOutputDataSize().toBytes();
outputPositions += outputStageStats.getOutputPositions();
}
return new QueryStats(queryStateTimer.getCreateTime(), queryStateTimer.getExecutionStartTime().orElse(null), queryStateTimer.getLastHeartbeat(), queryStateTimer.getEndTime().orElse(null), queryStateTimer.getElapsedTime(), queryStateTimer.getWaitingForPrerequisitesTime(), queryStateTimer.getQueuedTime(), queryStateTimer.getResourceWaitingTime(), queryStateTimer.getSemanticAnalyzingTime(), queryStateTimer.getColumnAccessPermissionCheckingTime(), queryStateTimer.getDispatchingTime(), queryStateTimer.getExecutionTime(), queryStateTimer.getAnalysisTime(), queryStateTimer.getPlanningTime(), queryStateTimer.getFinishingTime(), totalTasks, runningTasks, peakRunningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, cumulativeTotalMemory, succinctBytes(userMemoryReservation), succinctBytes(totalMemoryReservation), peakUserMemoryReservation, peakTotalMemoryReservation, peakTaskUserMemory, peakTaskTotalMemory, peakNodeTotalMemory, isScheduled(rootStage), succinctDuration(totalScheduledTime, MILLISECONDS), succinctDuration(totalCpuTime, MILLISECONDS), succinctDuration(retriedCpuTime, MILLISECONDS), succinctDuration(totalBlockedTime, MILLISECONDS), fullyBlocked, blockedReasons, succinctBytes(totalAllocation), succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(outputDataSize), outputPositions, writtenOutputPositions, succinctBytes(writtenOutputLogicalDataSize), succinctBytes(writtenOutputPhysicalDataSize), succinctBytes(writtenIntermediatePhysicalDataSize), stageGcStatistics.build(), operatorStatsSummary.build(), mergedRuntimeStats);
}
use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.
the class OrcTester method createCustomOrcReader.
static OrcReader createCustomOrcReader(TempFile tempFile, OrcEncoding orcEncoding, boolean mapNullKeysEnabled, Map<Integer, Slice> intermediateEncryptionKeys) throws IOException {
OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true);
OrcReader orcReader = new OrcReader(orcDataSource, orcEncoding, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), NOOP_ORC_AGGREGATED_MEMORY_CONTEXT, new OrcReaderOptions(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), MAX_BLOCK_SIZE, false, mapNullKeysEnabled, false, false), false, new DwrfEncryptionProvider(new UnsupportedEncryptionLibrary(), new TestingEncryptionLibrary()), DwrfKeyProvider.of(intermediateEncryptionKeys), new RuntimeStats());
return orcReader;
}
use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.
the class OrcTester method getFileMetadata.
public static FileMetadata getFileMetadata(File inputFile, OrcEncoding encoding) throws IOException {
boolean zstdJniDecompressionEnabled = true;
DataSize dataSize = new DataSize(1, MEGABYTE);
OrcDataSource orcDataSource = new FileOrcDataSource(inputFile, dataSize, dataSize, dataSize, true);
RuntimeStats runtimeStats = new RuntimeStats();
OrcReader reader = new OrcReader(orcDataSource, encoding, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), NOOP_ORC_AGGREGATED_MEMORY_CONTEXT, new OrcReaderOptions(dataSize, dataSize, dataSize, zstdJniDecompressionEnabled), false, NO_ENCRYPTION, DwrfKeyProvider.EMPTY, runtimeStats);
Footer footer = reader.getFooter();
Optional<OrcDecompressor> decompressor = createOrcDecompressor(orcDataSource.getId(), reader.getCompressionKind(), reader.getBufferSize(), zstdJniDecompressionEnabled);
ImmutableList.Builder<StripeFooter> stripes = new ImmutableList.Builder<>();
for (StripeInformation stripe : footer.getStripes()) {
// read the footer
byte[] tailBuffer = new byte[toIntExact(stripe.getFooterLength())];
orcDataSource.readFully(stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(), tailBuffer);
try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), new SharedBuffer(NOOP_ORC_LOCAL_MEMORY_CONTEXT), Slices.wrappedBuffer(tailBuffer).getInput(), decompressor, Optional.empty(), new TestingHiveOrcAggregatedMemoryContext(), tailBuffer.length)) {
StripeFooter stripeFooter = encoding.createMetadataReader(runtimeStats).readStripeFooter(orcDataSource.getId(), footer.getTypes(), inputStream);
stripes.add(stripeFooter);
}
}
return new FileMetadata(footer, stripes.build());
}
Aggregations