Examples with MetricsRecordBuilder org.apache.hadoop.metrics2.MetricsRecordBuilder used on opensource projects

Example 41 with MetricsRecordBuilder

use of org.apache.hadoop.metrics2.MetricsRecordBuilder in project hadoop by apache.

the class TestRPC method testClientBackOffByResponseTime.

/**
   *  Test RPC backoff by response time of each priority level.
   */
@Test(timeout = 30000)
public void testClientBackOffByResponseTime() throws Exception {
    final TestRpcService proxy;
    boolean succeeded = false;
    final int numClients = 1;
    GenericTestUtils.setLogLevel(DecayRpcScheduler.LOG, Level.DEBUG);
    GenericTestUtils.setLogLevel(RPC.LOG, Level.DEBUG);
    final List<Future<Void>> res = new ArrayList<Future<Void>>();
    final ExecutorService executorService = Executors.newFixedThreadPool(numClients);
    conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
    final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0";
    Server server = setupDecayRpcSchedulerandTestServer(ns + ".");
    @SuppressWarnings("unchecked") CallQueueManager<Call> spy = spy((CallQueueManager<Call>) Whitebox.getInternalState(server, "callQueue"));
    Whitebox.setInternalState(server, "callQueue", spy);
    Exception lastException = null;
    proxy = getClient(addr, conf);
    MetricsRecordBuilder rb1 = getMetrics("DecayRpcSchedulerMetrics2." + ns);
    final long beginDecayedCallVolume = MetricsAsserts.getLongCounter("DecayedCallVolume", rb1);
    final long beginRawCallVolume = MetricsAsserts.getLongCounter("CallVolume", rb1);
    final int beginUniqueCaller = MetricsAsserts.getIntCounter("UniqueCallers", rb1);
    try {
        // start a sleep RPC call that sleeps 3s.
        for (int i = 0; i < numClients; i++) {
            res.add(executorService.submit(new Callable<Void>() {

                @Override
                public Void call() throws ServiceException, InterruptedException {
                    proxy.sleep(null, newSleepRequest(3000));
                    return null;
                }
            }));
            verify(spy, timeout(500).times(i + 1)).offer(Mockito.<Call>anyObject());
        }
        // avg response time(3s) exceeds threshold (2s).
        try {
            // wait for the 1st response time update
            Thread.sleep(5500);
            proxy.sleep(null, newSleepRequest(100));
        } catch (ServiceException e) {
            RemoteException re = (RemoteException) e.getCause();
            IOException unwrapExeption = re.unwrapRemoteException();
            if (unwrapExeption instanceof RetriableException) {
                succeeded = true;
            } else {
                lastException = unwrapExeption;
            }
            // Lets Metric system update latest metrics
            GenericTestUtils.waitFor(new Supplier<Boolean>() {

                @Override
                public Boolean get() {
                    MetricsRecordBuilder rb2 = getMetrics("DecayRpcSchedulerMetrics2." + ns);
                    long decayedCallVolume1 = MetricsAsserts.getLongCounter("DecayedCallVolume", rb2);
                    long rawCallVolume1 = MetricsAsserts.getLongCounter("CallVolume", rb2);
                    int uniqueCaller1 = MetricsAsserts.getIntCounter("UniqueCallers", rb2);
                    long callVolumePriority0 = MetricsAsserts.getLongGauge("Priority.0.CompletedCallVolume", rb2);
                    long callVolumePriority1 = MetricsAsserts.getLongGauge("Priority.1.CompletedCallVolume", rb2);
                    double avgRespTimePriority0 = MetricsAsserts.getDoubleGauge("Priority.0.AvgResponseTime", rb2);
                    double avgRespTimePriority1 = MetricsAsserts.getDoubleGauge("Priority.1.AvgResponseTime", rb2);
                    LOG.info("DecayedCallVolume: " + decayedCallVolume1);
                    LOG.info("CallVolume: " + rawCallVolume1);
                    LOG.info("UniqueCaller: " + uniqueCaller1);
                    LOG.info("Priority.0.CompletedCallVolume: " + callVolumePriority0);
                    LOG.info("Priority.1.CompletedCallVolume: " + callVolumePriority1);
                    LOG.info("Priority.0.AvgResponseTime: " + avgRespTimePriority0);
                    LOG.info("Priority.1.AvgResponseTime: " + avgRespTimePriority1);
                    return decayedCallVolume1 > beginDecayedCallVolume && rawCallVolume1 > beginRawCallVolume && uniqueCaller1 > beginUniqueCaller;
                }
            }, 30, 60000);
        }
    } finally {
        executorService.shutdown();
        stop(server, proxy);
    }
    if (lastException != null) {
        LOG.error("Last received non-RetriableException:", lastException);
    }
    assertTrue("RetriableException not received", succeeded);
}

Example 42 with MetricsRecordBuilder

use of org.apache.hadoop.metrics2.MetricsRecordBuilder in project hadoop by apache.

the class TestRPC method testRpcMetrics.

@Test
public void testRpcMetrics() throws Exception {
    final Server server;
    TestRpcService proxy = null;
    final int interval = 1;
    conf.setBoolean(CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE, true);
    conf.set(CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval);
    server = setupTestServer(conf, 5);
    String testUser = "testUser";
    UserGroupInformation anotherUser = UserGroupInformation.createRemoteUser(testUser);
    TestRpcService proxy2 = anotherUser.doAs(new PrivilegedAction<TestRpcService>() {

        public TestRpcService run() {
            try {
                return RPC.getProxy(TestRpcService.class, 0, server.getListenerAddress(), conf);
            } catch (IOException e) {
                e.printStackTrace();
            }
            return null;
        }
    });
    try {
        proxy = getClient(addr, conf);
        for (int i = 0; i < 1000; i++) {
            proxy.ping(null, newEmptyRequest());
            proxy.echo(null, newEchoRequest("" + i));
            proxy2.echo(null, newEchoRequest("" + i));
        }
        MetricsRecordBuilder rpcMetrics = getMetrics(server.getRpcMetrics().name());
        assertTrue("Expected non-zero rpc queue time", getLongCounter("RpcQueueTimeNumOps", rpcMetrics) > 0);
        assertTrue("Expected non-zero rpc processing time", getLongCounter("RpcProcessingTimeNumOps", rpcMetrics) > 0);
        MetricsAsserts.assertQuantileGauges("RpcQueueTime" + interval + "s", rpcMetrics);
        MetricsAsserts.assertQuantileGauges("RpcProcessingTime" + interval + "s", rpcMetrics);
        String actualUserVsCon = MetricsAsserts.getStringMetric("NumOpenConnectionsPerUser", rpcMetrics);
        String proxyUser = UserGroupInformation.getCurrentUser().getShortUserName();
        assertTrue(actualUserVsCon.contains("\"" + proxyUser + "\":1"));
        assertTrue(actualUserVsCon.contains("\"" + testUser + "\":1"));
    } finally {
        if (proxy2 != null) {
            RPC.stopProxy(proxy2);
        }
        stop(server, proxy);
    }
}

Example 43 with MetricsRecordBuilder

use of org.apache.hadoop.metrics2.MetricsRecordBuilder in project hadoop by apache.

the class TestRPC method testCallsInternal.

private void testCallsInternal(Configuration myConf) throws Exception {
    Server server;
    TestRpcService proxy = null;
    server = setupTestServer(myConf, -1);
    try {
        proxy = getClient(addr, myConf);
        proxy.ping(null, newEmptyRequest());
        TestProtos.EchoResponseProto echoResp = proxy.echo(null, newEchoRequest("foo"));
        assertEquals(echoResp.getMessage(), "foo");
        echoResp = proxy.echo(null, newEchoRequest(""));
        assertEquals(echoResp.getMessage(), "");
        // Check rpcMetrics
        MetricsRecordBuilder rb = getMetrics(server.rpcMetrics.name());
        assertCounter("RpcProcessingTimeNumOps", 3L, rb);
        assertCounterGt("SentBytes", 0L, rb);
        assertCounterGt("ReceivedBytes", 0L, rb);
        // Number of calls to echo method should be 2
        rb = getMetrics(server.rpcDetailedMetrics.name());
        assertCounter("EchoNumOps", 2L, rb);
        // Number of calls to ping method should be 1
        assertCounter("PingNumOps", 1L, rb);
        String[] strings = new String[] { "foo", "bar" };
        TestProtos.EchoRequestProto2 echoRequest2 = TestProtos.EchoRequestProto2.newBuilder().addAllMessage(Arrays.asList(strings)).build();
        TestProtos.EchoResponseProto2 echoResponse2 = proxy.echo2(null, echoRequest2);
        assertTrue(Arrays.equals(echoResponse2.getMessageList().toArray(), strings));
        echoRequest2 = TestProtos.EchoRequestProto2.newBuilder().addAllMessage(Collections.<String>emptyList()).build();
        echoResponse2 = proxy.echo2(null, echoRequest2);
        assertTrue(Arrays.equals(echoResponse2.getMessageList().toArray(), new String[] {}));
        TestProtos.AddRequestProto addRequest = TestProtos.AddRequestProto.newBuilder().setParam1(1).setParam2(2).build();
        TestProtos.AddResponseProto addResponse = proxy.add(null, addRequest);
        assertEquals(addResponse.getResult(), 3);
        Integer[] integers = new Integer[] { 1, 2 };
        TestProtos.AddRequestProto2 addRequest2 = TestProtos.AddRequestProto2.newBuilder().addAllParams(Arrays.asList(integers)).build();
        addResponse = proxy.add2(null, addRequest2);
        assertEquals(addResponse.getResult(), 3);
        boolean caught = false;
        try {
            proxy.error(null, newEmptyRequest());
        } catch (ServiceException e) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Caught " + e);
            }
            caught = true;
        }
        assertTrue(caught);
        rb = getMetrics(server.rpcDetailedMetrics.name());
        assertCounter("RpcServerExceptionNumOps", 1L, rb);
        //proxy.testServerGet();
        // create multiple threads and make them do large data transfers
        System.out.println("Starting multi-threaded RPC test...");
        server.setSocketSendBufSize(1024);
        Thread[] threadId = new Thread[numThreads];
        for (int i = 0; i < numThreads; i++) {
            Transactions trans = new Transactions(proxy, datasize);
            threadId[i] = new Thread(trans, "TransactionThread-" + i);
            threadId[i].start();
        }
        // wait for all transactions to get over
        System.out.println("Waiting for all threads to finish RPCs...");
        for (int i = 0; i < numThreads; i++) {
            try {
                threadId[i].join();
            } catch (InterruptedException e) {
                // retry
                i--;
            }
        }
    } finally {
        stop(server, proxy);
    }
}

Example 44 with MetricsRecordBuilder

use of org.apache.hadoop.metrics2.MetricsRecordBuilder in project accumulo by apache.

the class Metrics2TabletServerMetrics method getMetrics.

@Override
public void getMetrics(MetricsCollector collector, boolean all) {
    MetricsRecordBuilder builder = collector.addRecord(RECORD).setContext(CONTEXT);
    // Update each MutableMetric with the new value
    snapshot();
    // Add then all to the builder
    registry.snapshot(builder, all);
    // TODO Some day, MetricsRegistry will also support the MetricsGaugeDouble or allow us to instantiate it directly
    builder.addGauge(Interns.info(FILES_PER_TABLET, "Number of files per tablet"), util.getAverageFilesPerTablet());
    builder.addGauge(Interns.info(HOLD_TIME, "Time commits held"), util.getHoldTime());
    builder.addGauge(Interns.info(INGEST_RATE, "Ingest rate (entries/sec)"), util.getIngest());
    builder.addGauge(Interns.info(INGEST_BYTE_RATE, "Ingest rate (bytes/sec)"), util.getIngestByteRate());
    builder.addGauge(Interns.info(QUERY_RATE, "Query rate (entries/sec)"), util.getQueryRate());
    builder.addGauge(Interns.info(QUERY_BYTE_RATE, "Query rate (bytes/sec)"), util.getQueryByteRate());
    builder.addGauge(Interns.info(SCANNED_RATE, "Scanned rate"), util.getScannedRate());
}

Example 45 with MetricsRecordBuilder

use of org.apache.hadoop.metrics2.MetricsRecordBuilder in project accumulo by apache.

the class Metrics2TabletServerScanMetrics method getMetrics.

@Override
public void getMetrics(MetricsCollector collector, boolean all) {
    MetricsRecordBuilder builder = collector.addRecord(RECORD).setContext(CONTEXT);
    registry.snapshot(builder, all);
}