Examples with ShuffleReadMetrics org.apache.hive.spark.client.metrics.ShuffleReadMetrics used on opensource projects

Example 1 with ShuffleReadMetrics

use of org.apache.hive.spark.client.metrics.ShuffleReadMetrics in project hive by apache.

the class SparkMetricsUtils method collectMetrics.

static Map<String, Long> collectMetrics(Metrics allMetrics) {
    Map<String, Long> results = new LinkedHashMap<String, Long>();
    results.put(SparkStatisticsNames.EXECUTOR_DESERIALIZE_TIME, allMetrics.executorDeserializeTime);
    results.put(SparkStatisticsNames.EXECUTOR_RUN_TIME, allMetrics.executorRunTime);
    results.put(SparkStatisticsNames.RESULT_SIZE, allMetrics.resultSize);
    results.put(SparkStatisticsNames.JVM_GC_TIME, allMetrics.jvmGCTime);
    results.put(SparkStatisticsNames.RESULT_SERIALIZATION_TIME, allMetrics.resultSerializationTime);
    results.put(SparkStatisticsNames.MEMORY_BYTES_SPLIED, allMetrics.memoryBytesSpilled);
    results.put(SparkStatisticsNames.DISK_BYTES_SPILLED, allMetrics.diskBytesSpilled);
    results.put(SparkStatisticsNames.TASK_DURATION_TIME, allMetrics.taskDurationTime);
    if (allMetrics.inputMetrics != null) {
        results.put(SparkStatisticsNames.BYTES_READ, allMetrics.inputMetrics.bytesRead);
    }
    if (allMetrics.shuffleReadMetrics != null) {
        ShuffleReadMetrics shuffleReadMetrics = allMetrics.shuffleReadMetrics;
        long rbf = shuffleReadMetrics.remoteBlocksFetched;
        long lbf = shuffleReadMetrics.localBlocksFetched;
        results.put(SparkStatisticsNames.REMOTE_BLOCKS_FETCHED, rbf);
        results.put(SparkStatisticsNames.LOCAL_BLOCKS_FETCHED, lbf);
        results.put(SparkStatisticsNames.TOTAL_BLOCKS_FETCHED, rbf + lbf);
        results.put(SparkStatisticsNames.FETCH_WAIT_TIME, shuffleReadMetrics.fetchWaitTime);
        results.put(SparkStatisticsNames.REMOTE_BYTES_READ, shuffleReadMetrics.remoteBytesRead);
    }
    if (allMetrics.shuffleWriteMetrics != null) {
        results.put(SparkStatisticsNames.SHUFFLE_BYTES_WRITTEN, allMetrics.shuffleWriteMetrics.shuffleBytesWritten);
        results.put(SparkStatisticsNames.SHUFFLE_WRITE_TIME, allMetrics.shuffleWriteMetrics.shuffleWriteTime);
    }
    return results;
}

Example 2 with ShuffleReadMetrics

use of org.apache.hive.spark.client.metrics.ShuffleReadMetrics in project hive by apache.

the class MetricsCollection method aggregate.

private Metrics aggregate(Predicate<TaskInfo> filter) {
    lock.readLock().lock();
    try {
        // Task metrics.
        long executorDeserializeTime = 0L;
        long executorRunTime = 0L;
        long resultSize = 0L;
        long jvmGCTime = 0L;
        long resultSerializationTime = 0L;
        long memoryBytesSpilled = 0L;
        long diskBytesSpilled = 0L;
        long taskDurationTime = 0L;
        // Input metrics.
        boolean hasInputMetrics = false;
        long bytesRead = 0L;
        // Shuffle read metrics.
        boolean hasShuffleReadMetrics = false;
        int remoteBlocksFetched = 0;
        int localBlocksFetched = 0;
        long fetchWaitTime = 0L;
        long remoteBytesRead = 0L;
        // Shuffle write metrics.
        long shuffleBytesWritten = 0L;
        long shuffleWriteTime = 0L;
        for (TaskInfo info : Collections2.filter(taskMetrics, filter)) {
            Metrics m = info.metrics;
            executorDeserializeTime += m.executorDeserializeTime;
            executorRunTime += m.executorRunTime;
            resultSize += m.resultSize;
            jvmGCTime += m.jvmGCTime;
            resultSerializationTime += m.resultSerializationTime;
            memoryBytesSpilled += m.memoryBytesSpilled;
            diskBytesSpilled += m.diskBytesSpilled;
            taskDurationTime += m.taskDurationTime;
            if (m.inputMetrics != null) {
                hasInputMetrics = true;
                bytesRead += m.inputMetrics.bytesRead;
            }
            if (m.shuffleReadMetrics != null) {
                hasShuffleReadMetrics = true;
                remoteBlocksFetched += m.shuffleReadMetrics.remoteBlocksFetched;
                localBlocksFetched += m.shuffleReadMetrics.localBlocksFetched;
                fetchWaitTime += m.shuffleReadMetrics.fetchWaitTime;
                remoteBytesRead += m.shuffleReadMetrics.remoteBytesRead;
            }
            if (m.shuffleWriteMetrics != null) {
                shuffleBytesWritten += m.shuffleWriteMetrics.shuffleBytesWritten;
                shuffleWriteTime += m.shuffleWriteMetrics.shuffleWriteTime;
            }
        }
        InputMetrics inputMetrics = null;
        if (hasInputMetrics) {
            inputMetrics = new InputMetrics(bytesRead);
        }
        ShuffleReadMetrics shuffleReadMetrics = null;
        if (hasShuffleReadMetrics) {
            shuffleReadMetrics = new ShuffleReadMetrics(remoteBlocksFetched, localBlocksFetched, fetchWaitTime, remoteBytesRead);
        }
        ShuffleWriteMetrics shuffleWriteMetrics = null;
        if (hasShuffleReadMetrics) {
            shuffleWriteMetrics = new ShuffleWriteMetrics(shuffleBytesWritten, shuffleWriteTime);
        }
        return new Metrics(executorDeserializeTime, executorRunTime, resultSize, jvmGCTime, resultSerializationTime, memoryBytesSpilled, diskBytesSpilled, taskDurationTime, inputMetrics, shuffleReadMetrics, shuffleWriteMetrics);
    } finally {
        lock.readLock().unlock();
    }
}

Example 3 with ShuffleReadMetrics

use of org.apache.hive.spark.client.metrics.ShuffleReadMetrics in project hive by apache.

the class TestMetricsCollection method testOptionalMetrics.

@Test
public void testOptionalMetrics() {
    long value = taskValue(1, 1, 1L);
    Metrics metrics = new Metrics(value, value, value, value, value, value, value, value, null, null, null);
    MetricsCollection collection = new MetricsCollection();
    for (int i : Arrays.asList(1, 2)) {
        collection.addMetrics(i, 1, 1, metrics);
    }
    Metrics global = collection.getAllMetrics();
    assertNull(global.inputMetrics);
    assertNull(global.shuffleReadMetrics);
    assertNull(global.shuffleWriteMetrics);
    collection.addMetrics(3, 1, 1, makeMetrics(3, 1, 1));
    Metrics global2 = collection.getAllMetrics();
    assertNotNull(global2.inputMetrics);
    assertEquals(taskValue(3, 1, 1), global2.inputMetrics.bytesRead);
    assertNotNull(global2.shuffleReadMetrics);
    assertNotNull(global2.shuffleWriteMetrics);
}