Examples with MetricsSource org.apache.hadoop.metrics2.MetricsSource used on opensource projects

Example 1 with MetricsSource

use of org.apache.hadoop.metrics2.MetricsSource in project hadoop by apache.

the class TestShuffleHandler method checkShuffleMetrics.

static void checkShuffleMetrics(MetricsSystem ms, long bytes, int failed, int succeeded, int connections) {
    MetricsSource source = ms.getSource("ShuffleMetrics");
    MetricsRecordBuilder rb = getMetrics(source);
    assertCounter("ShuffleOutputBytes", bytes, rb);
    assertCounter("ShuffleOutputsFailed", failed, rb);
    assertCounter("ShuffleOutputsOK", succeeded, rb);
    assertGauge("ShuffleConnections", connections, rb);
}

Example 2 with MetricsSource

use of org.apache.hadoop.metrics2.MetricsSource in project hadoop by apache.

the class TestQueueMetrics method testQueueAppMetricsForMultipleFailures.

@Test
public void testQueueAppMetricsForMultipleFailures() {
    String queueName = "single";
    String user = "alice";
    QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, new Configuration());
    MetricsSource queueSource = queueSource(ms, queueName);
    AppSchedulingInfo app = mockApp(user);
    metrics.submitApp(user);
    MetricsSource userSource = userSource(ms, queueName, user);
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    // As the application has failed, framework retries the same application
    // based on configuration
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    // Suppose say application has failed this time as well.
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    // As the application has failed, framework retries the same application
    // based on configuration
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    // Suppose say application has failed, and there's no more retries.
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    metrics.finishApp(user, RMAppState.FAILED);
    checkApps(queueSource, 1, 0, 0, 0, 1, 0, true);
    assertNull(userSource);
}

Example 3 with MetricsSource

use of org.apache.hadoop.metrics2.MetricsSource in project hadoop by apache.

the class TestQueueMetrics method checkResources.

public static void checkResources(MetricsSource source, long allocatedMB, int allocatedCores, int allocCtnrs, long aggreAllocCtnrs, long aggreReleasedCtnrs, long availableMB, int availableCores, long pendingMB, int pendingCores, int pendingCtnrs, long reservedMB, int reservedCores, int reservedCtnrs) {
    MetricsRecordBuilder rb = getMetrics(source);
    assertGauge("AllocatedMB", allocatedMB, rb);
    assertGauge("AllocatedVCores", allocatedCores, rb);
    assertGauge("AllocatedContainers", allocCtnrs, rb);
    assertCounter("AggregateContainersAllocated", aggreAllocCtnrs, rb);
    assertCounter("AggregateContainersReleased", aggreReleasedCtnrs, rb);
    assertGauge("AvailableMB", availableMB, rb);
    assertGauge("AvailableVCores", availableCores, rb);
    assertGauge("PendingMB", pendingMB, rb);
    assertGauge("PendingVCores", pendingCores, rb);
    assertGauge("PendingContainers", pendingCtnrs, rb);
    assertGauge("ReservedMB", reservedMB, rb);
    assertGauge("ReservedVCores", reservedCores, rb);
    assertGauge("ReservedContainers", reservedCtnrs, rb);
}

Example 4 with MetricsSource

use of org.apache.hadoop.metrics2.MetricsSource in project hadoop by apache.

the class TestQueueMetrics method testCollectAllMetrics.

// This is to test all metrics can consistently show up if specified true to
// collect all metrics, even though they are not modified from last time they
// are collected. If not collecting all metrics, only modified metrics will show up.
@Test
public void testCollectAllMetrics() {
    String queueName = "single";
    QueueMetrics.forQueue(ms, queueName, null, false, conf);
    MetricsSource queueSource = queueSource(ms, queueName);
    checkApps(queueSource, 0, 0, 0, 0, 0, 0, true);
    try {
        // do not collect all metrics
        checkApps(queueSource, 0, 0, 0, 0, 0, 0, false);
        Assert.fail();
    } catch (AssertionError e) {
        Assert.assertTrue(e.getMessage().contains("Expected exactly one metric for name "));
    }
    // collect all metrics
    checkApps(queueSource, 0, 0, 0, 0, 0, 0, true);
}

Example 5 with MetricsSource

use of org.apache.hadoop.metrics2.MetricsSource in project hadoop by apache.

the class TestQueueMetrics method testSingleQueueWithUserMetrics.

@Test
public void testSingleQueueWithUserMetrics() {
    String queueName = "single2";
    String user = "dodo";
    QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, true, conf);
    MetricsSource queueSource = queueSource(ms, queueName);
    AppSchedulingInfo app = mockApp(user);
    metrics.submitApp(user);
    MetricsSource userSource = userSource(ms, queueName, user);
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    checkApps(userSource, 1, 0, 0, 0, 0, 0, true);
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    checkApps(userSource, 1, 1, 0, 0, 0, 0, true);
    metrics.setAvailableResourcesToQueue(Resources.createResource(100 * GB, 100));
    metrics.setAvailableResourcesToUser(user, Resources.createResource(10 * GB, 10));
    metrics.incrPendingResources(user, 5, Resources.createResource(3 * GB, 3));
    // Available resources is set externally, as it depends on dynamic
    // configurable cluster/queue resources
    checkResources(queueSource, 0, 0, 0, 0, 0, 100 * GB, 100, 15 * GB, 15, 5, 0, 0, 0);
    checkResources(userSource, 0, 0, 0, 0, 0, 10 * GB, 10, 15 * GB, 15, 5, 0, 0, 0);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
    metrics.allocateResources(user, 3, Resources.createResource(2 * GB, 2), true);
    checkResources(queueSource, 6 * GB, 6, 3, 3, 0, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    checkResources(userSource, 6 * GB, 6, 3, 3, 0, 10 * GB, 10, 9 * GB, 9, 2, 0, 0, 0);
    metrics.releaseResources(user, 1, Resources.createResource(2 * GB, 2));
    checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    checkResources(userSource, 4 * GB, 4, 2, 3, 1, 10 * GB, 10, 9 * GB, 9, 2, 0, 0, 0);
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    checkApps(userSource, 1, 0, 0, 0, 0, 0, true);
    metrics.finishApp(user, RMAppState.FINISHED);
    checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
    checkApps(userSource, 1, 0, 0, 1, 0, 0, true);
}