Examples with MetricsSource org.apache.hadoop.metrics2.MetricsSource used on opensource projects

Example 6 with MetricsSource

use of org.apache.hadoop.metrics2.MetricsSource in project hadoop by apache.

the class TestQueueMetrics method testDefaultSingleQueueMetrics.

@Test
public void testDefaultSingleQueueMetrics() {
    String queueName = "single";
    String user = "alice";
    QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, conf);
    MetricsSource queueSource = queueSource(ms, queueName);
    AppSchedulingInfo app = mockApp(user);
    metrics.submitApp(user);
    MetricsSource userSource = userSource(ms, queueName, user);
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    metrics.setAvailableResourcesToQueue(Resources.createResource(100 * GB, 100));
    metrics.incrPendingResources(user, 5, Resources.createResource(3 * GB, 3));
    // Available resources is set externally, as it depends on dynamic
    // configurable cluster/queue resources
    checkResources(queueSource, 0, 0, 0, 0, 0, 100 * GB, 100, 15 * GB, 15, 5, 0, 0, 0);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    metrics.allocateResources(user, 3, Resources.createResource(2 * GB, 2), true);
    checkResources(queueSource, 6 * GB, 6, 3, 3, 0, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    metrics.releaseResources(user, 1, Resources.createResource(2 * GB, 2));
    checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    metrics.incrPendingResources(user, 0, Resources.createResource(2 * GB, 2));
    checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    metrics.decrPendingResources(user, 0, Resources.createResource(2 * GB, 2));
    checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    metrics.finishApp(user, RMAppState.FINISHED);
    checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
    assertNull(userSource);
}

Example 7 with MetricsSource

use of org.apache.hadoop.metrics2.MetricsSource in project hadoop by apache.

the class TestQueueMetrics method testTwoLevelWithUserMetrics.

@Test
public void testTwoLevelWithUserMetrics() {
    String parentQueueName = "root";
    String leafQueueName = "root.leaf";
    String user = "alice";
    QueueMetrics parentMetrics = QueueMetrics.forQueue(ms, parentQueueName, null, true, conf);
    Queue parentQueue = make(stub(Queue.class).returning(parentMetrics).from.getMetrics());
    QueueMetrics metrics = QueueMetrics.forQueue(ms, leafQueueName, parentQueue, true, conf);
    MetricsSource parentQueueSource = queueSource(ms, parentQueueName);
    MetricsSource queueSource = queueSource(ms, leafQueueName);
    AppSchedulingInfo app = mockApp(user);
    metrics.submitApp(user);
    MetricsSource userSource = userSource(ms, leafQueueName, user);
    MetricsSource parentUserSource = userSource(ms, parentQueueName, user);
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    checkApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
    checkApps(userSource, 1, 0, 0, 0, 0, 0, true);
    checkApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
    metrics.submitAppAttempt(user);
    checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
    checkApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true);
    checkApps(userSource, 1, 1, 0, 0, 0, 0, true);
    checkApps(parentUserSource, 1, 1, 0, 0, 0, 0, true);
    parentMetrics.setAvailableResourcesToQueue(Resources.createResource(100 * GB, 100));
    metrics.setAvailableResourcesToQueue(Resources.createResource(100 * GB, 100));
    parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10 * GB, 10));
    metrics.setAvailableResourcesToUser(user, Resources.createResource(10 * GB, 10));
    metrics.incrPendingResources(user, 5, Resources.createResource(3 * GB, 3));
    checkResources(queueSource, 0, 0, 0, 0, 0, 100 * GB, 100, 15 * GB, 15, 5, 0, 0, 0);
    checkResources(parentQueueSource, 0, 0, 0, 0, 0, 100 * GB, 100, 15 * GB, 15, 5, 0, 0, 0);
    checkResources(userSource, 0, 0, 0, 0, 0, 10 * GB, 10, 15 * GB, 15, 5, 0, 0, 0);
    checkResources(parentUserSource, 0, 0, 0, 0, 0, 10 * GB, 10, 15 * GB, 15, 5, 0, 0, 0);
    metrics.runAppAttempt(app.getApplicationId(), user);
    checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
    checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
    metrics.allocateResources(user, 3, Resources.createResource(2 * GB, 2), true);
    metrics.reserveResource(user, Resources.createResource(3 * GB, 3));
    // Available resources is set externally, as it depends on dynamic
    // configurable cluster/queue resources
    checkResources(queueSource, 6 * GB, 6, 3, 3, 0, 100 * GB, 100, 9 * GB, 9, 2, 3 * GB, 3, 1);
    checkResources(parentQueueSource, 6 * GB, 6, 3, 3, 0, 100 * GB, 100, 9 * GB, 9, 2, 3 * GB, 3, 1);
    checkResources(userSource, 6 * GB, 6, 3, 3, 0, 10 * GB, 10, 9 * GB, 9, 2, 3 * GB, 3, 1);
    checkResources(parentUserSource, 6 * GB, 6, 3, 3, 0, 10 * GB, 10, 9 * GB, 9, 2, 3 * GB, 3, 1);
    metrics.releaseResources(user, 1, Resources.createResource(2 * GB, 2));
    metrics.unreserveResource(user, Resources.createResource(3 * GB, 3));
    checkResources(queueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    checkResources(parentQueueSource, 4 * GB, 4, 2, 3, 1, 100 * GB, 100, 9 * GB, 9, 2, 0, 0, 0);
    checkResources(userSource, 4 * GB, 4, 2, 3, 1, 10 * GB, 10, 9 * GB, 9, 2, 0, 0, 0);
    checkResources(parentUserSource, 4 * GB, 4, 2, 3, 1, 10 * GB, 10, 9 * GB, 9, 2, 0, 0, 0);
    metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser());
    checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
    checkApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
    checkApps(userSource, 1, 0, 0, 0, 0, 0, true);
    checkApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
    metrics.finishApp(user, RMAppState.FINISHED);
    checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
    checkApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true);
    checkApps(userSource, 1, 0, 0, 1, 0, 0, true);
    checkApps(parentUserSource, 1, 0, 0, 1, 0, 0, true);
}

Example 8 with MetricsSource

use of org.apache.hadoop.metrics2.MetricsSource in project hadoop by apache.

the class MetricsSystemImpl method register.

@Override
public synchronized <T> T register(String name, String desc, T source) {
    MetricsSourceBuilder sb = MetricsAnnotations.newSourceBuilder(source);
    final MetricsSource s = sb.build();
    MetricsInfo si = sb.info();
    String name2 = name == null ? si.name() : name;
    final String finalDesc = desc == null ? si.description() : desc;
    final // be friendly to non-metrics tests
    String finalName = DefaultMetricsSystem.sourceName(name2, !monitoring);
    allSources.put(finalName, s);
    LOG.debug(finalName + ", " + finalDesc);
    if (monitoring) {
        registerSource(finalName, finalDesc, s);
    }
    // We want to re-register the source to pick up new config when the
    // metrics system restarts.
    register(finalName, new AbstractCallback() {

        @Override
        public void postStart() {
            registerSource(finalName, finalDesc, s);
        }
    });
    return source;
}

Example 9 with MetricsSource

use of org.apache.hadoop.metrics2.MetricsSource in project hadoop by apache.

the class TestNameNodeMetrics method tearDown.

@After
public void tearDown() throws Exception {
    MetricsSource source = DefaultMetricsSystem.instance().getSource("UgiMetrics");
    if (source != null) {
        // Run only once since the UGI metrics is cleaned up during teardown
        MetricsRecordBuilder rb = getMetrics(source);
        assertQuantileGauges("GetGroups1s", rb);
    }
    if (cluster != null) {
        cluster.shutdown();
        cluster = null;
    }
}

Example 10 with MetricsSource

use of org.apache.hadoop.metrics2.MetricsSource in project hbase by apache.

the class TestReplicationEndpoint method testMetricsSourceBaseSourcePassthrough.

@Test
public void testMetricsSourceBaseSourcePassthrough() {
    /*
    The replication MetricsSource wraps a MetricsReplicationSourceSourceImpl
    and a MetricsReplicationGlobalSourceSource, so that metrics get written to both namespaces.
    Both of those classes wrap a MetricsReplicationSourceImpl that implements BaseSource, which
    allows for custom JMX metrics.
    This test checks to make sure the BaseSource decorator logic on MetricsSource actually calls down through
    the two layers of wrapping to the actual BaseSource.
    */
    String id = "id";
    DynamicMetricsRegistry mockRegistry = mock(DynamicMetricsRegistry.class);
    MetricsReplicationSourceImpl singleRms = mock(MetricsReplicationSourceImpl.class);
    when(singleRms.getMetricsRegistry()).thenReturn(mockRegistry);
    MetricsReplicationSourceImpl globalRms = mock(MetricsReplicationSourceImpl.class);
    when(globalRms.getMetricsRegistry()).thenReturn(mockRegistry);
    MetricsReplicationSourceSource singleSourceSource = new MetricsReplicationSourceSourceImpl(singleRms, id);
    MetricsReplicationSourceSource globalSourceSource = new MetricsReplicationGlobalSourceSource(globalRms);
    MetricsSource source = new MetricsSource(id, singleSourceSource, globalSourceSource);
    String gaugeName = "gauge";
    String singleGaugeName = "source.id." + gaugeName;
    long delta = 1;
    String counterName = "counter";
    String singleCounterName = "source.id." + counterName;
    long count = 2;
    source.decGauge(gaugeName, delta);
    source.getMetricsContext();
    source.getMetricsDescription();
    source.getMetricsJmxContext();
    source.getMetricsName();
    source.incCounters(counterName, count);
    source.incGauge(gaugeName, delta);
    source.init();
    source.removeMetric(gaugeName);
    source.setGauge(gaugeName, delta);
    source.updateHistogram(counterName, count);
    verify(singleRms).decGauge(singleGaugeName, delta);
    verify(globalRms).decGauge(gaugeName, delta);
    verify(globalRms).getMetricsContext();
    verify(globalRms).getMetricsJmxContext();
    verify(globalRms).getMetricsName();
    verify(singleRms).incCounters(singleCounterName, count);
    verify(globalRms).incCounters(counterName, count);
    verify(singleRms).incGauge(singleGaugeName, delta);
    verify(globalRms).incGauge(gaugeName, delta);
    verify(globalRms).init();
    verify(singleRms).removeMetric(singleGaugeName);
    verify(globalRms).removeMetric(gaugeName);
    verify(singleRms).setGauge(singleGaugeName, delta);
    verify(globalRms).setGauge(gaugeName, delta);
    verify(singleRms).updateHistogram(singleCounterName, count);
    verify(globalRms).updateHistogram(counterName, count);
}