Examples with Metrics org.apache.hadoop.metrics2.annotation.Metrics used on opensource projects

Example 56 with Metrics

use of org.apache.hadoop.metrics2.annotation.Metrics in project hadoop by apache.

the class TestQueueMetrics method testMetricsCache.

@Test
public void testMetricsCache() {
    MetricsSystem ms = new MetricsSystemImpl("cache");
    ms.start();
    try {
        String p1 = "root1";
        String leafQueueName = "root1.leaf";
        QueueMetrics p1Metrics = QueueMetrics.forQueue(ms, p1, null, true, conf);
        Queue parentQueue1 = make(stub(Queue.class).returning(p1Metrics).from.getMetrics());
        QueueMetrics metrics = QueueMetrics.forQueue(ms, leafQueueName, parentQueue1, true, conf);
        Assert.assertNotNull("QueueMetrics for A shoudn't be null", metrics);
        // Re-register to check for cache hit, shouldn't blow up metrics-system...
        // also, verify parent-metrics
        QueueMetrics alterMetrics = QueueMetrics.forQueue(ms, leafQueueName, parentQueue1, true, conf);
        Assert.assertNotNull("QueueMetrics for alterMetrics shoudn't be null", alterMetrics);
    } finally {
        ms.shutdown();
    }
}

Example 57 with Metrics

use of org.apache.hadoop.metrics2.annotation.Metrics in project hadoop by apache.

the class TestQueueMetrics method testQueueAppMetricsForMultipleFailures.

@Test
public void testQueueAppMetricsForMultipleFailures() {
    String queueName = "single";
    String user = "alice";
    QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, new Configuration());
    MetricsSource queueSource = queueSource(ms, queueName);
    AppSchedulingInfo app = mockApp(user);
    metrics.submitApp(user);
    MetricsSource userSource = userSource(ms, queueName, user);
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    // As the application has failed, framework retries the same application
    // based on configuration
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    // Suppose say application has failed this time as well.
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    // As the application has failed, framework retries the same application
    // based on configuration
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    // Suppose say application has failed, and there's no more retries.
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    metrics.finishApp(user, RMAppState.FAILED);
    checkApps(queueSource, 1, 0, 0, 0, 1, 0, true);
    assertNull(userSource);
}

Example 58 with Metrics

use of org.apache.hadoop.metrics2.annotation.Metrics in project hadoop by apache.

the class TestQueueMetrics method testCollectAllMetrics.

// This is to test all metrics can consistently show up if specified true to
// collect all metrics, even though they are not modified from last time they
// are collected. If not collecting all metrics, only modified metrics will show up.
@Test
public void testCollectAllMetrics() {
    String queueName = "single";
    QueueMetrics.forQueue(ms, queueName, null, false, conf);
    MetricsSource queueSource = queueSource(ms, queueName);
    checkApps(queueSource, 0, 0, 0, 0, 0, 0, true);
    try {
        // do not collect all metrics
        checkApps(queueSource, 0, 0, 0, 0, 0, 0, false);
        Assert.fail();
    } catch (AssertionError e) {
        Assert.assertTrue(e.getMessage().contains("Expected exactly one metric for name "));
    }
    // collect all metrics
    checkApps(queueSource, 0, 0, 0, 0, 0, 0, true);
}

Example 59 with Metrics

use of org.apache.hadoop.metrics2.annotation.Metrics in project hadoop by apache.

the class TestQueueMetrics method testSingleQueueWithUserMetrics.

@Test
public void testSingleQueueWithUserMetrics() {
    String queueName = "single2";
    String user = "dodo";
    QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, true, conf);
    MetricsSource queueSource = queueSource(ms, queueName);
    AppSchedulingInfo app = mockApp(user);
    metrics.submitApp(user);
    MetricsSource userSource = userSource(ms, queueName, user);
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    checkApps(userSource, 1, 0, 0, 0, 0, 0, true);
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    checkApps(userSource, 1, 1, 0, 0, 0, 0, true);
    metrics.setAvailableResourcesToQueue(Resources.createResource(100 * GB, 100));
    metrics.setAvailableResourcesToUser(user, Resources.createResource(10 * GB, 10));
    metrics.incrPendingResources(user, 5, Resources.createResource(3 * GB, 3));
    // Available resources is set externally, as it depends on dynamic
    // configurable cluster/queue resources
    checkResources(queueSource, 0, 0, 0, 0, 0, 100 * GB, 100, 15 * GB, 15, 5, 0, 0, 0);
    checkResources(userSource, 0, 0, 0, 0, 0, 10 * GB, 10, 15 * GB, 15, 5, 0, 0, 0);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
    metrics.allocateResources(user, 3, Resources.createResource(2 * GB, 2), true);
    checkResources(queueSource, 6 * GB, 6, 3, 3, 0, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    checkResources(userSource, 6 * GB, 6, 3, 3, 0, 10 * GB, 10, 9 * GB, 9, 2, 0, 0, 0);
    metrics.releaseResources(user, 1, Resources.createResource(2 * GB, 2));
    checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    checkResources(userSource, 4 * GB, 4, 2, 3, 1, 10 * GB, 10, 9 * GB, 9, 2, 0, 0, 0);
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    checkApps(userSource, 1, 0, 0, 0, 0, 0, true);
    metrics.finishApp(user, RMAppState.FINISHED);
    checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
    checkApps(userSource, 1, 0, 0, 1, 0, 0, true);
}

Example 60 with Metrics

use of org.apache.hadoop.metrics2.annotation.Metrics in project hadoop by apache.

the class TestQueueMetrics method testDefaultSingleQueueMetrics.

@Test
public void testDefaultSingleQueueMetrics() {
    String queueName = "single";
    String user = "alice";
    QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, conf);
    MetricsSource queueSource = queueSource(ms, queueName);
    AppSchedulingInfo app = mockApp(user);
    metrics.submitApp(user);
    MetricsSource userSource = userSource(ms, queueName, user);
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    metrics.setAvailableResourcesToQueue(Resources.createResource(100 * GB, 100));
    metrics.incrPendingResources(user, 5, Resources.createResource(3 * GB, 3));
    // Available resources is set externally, as it depends on dynamic
    // configurable cluster/queue resources
    checkResources(queueSource, 0, 0, 0, 0, 0, 100 * GB, 100, 15 * GB, 15, 5, 0, 0, 0);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    metrics.allocateResources(user, 3, Resources.createResource(2 * GB, 2), true);
    checkResources(queueSource, 6 * GB, 6, 3, 3, 0, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    metrics.releaseResources(user, 1, Resources.createResource(2 * GB, 2));
    checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    metrics.incrPendingResources(user, 0, Resources.createResource(2 * GB, 2));
    checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    metrics.decrPendingResources(user, 0, Resources.createResource(2 * GB, 2));
    checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    metrics.finishApp(user, RMAppState.FINISHED);
    checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
    assertNull(userSource);
}