Examples with Call org.apache.hadoop.ipc.Server.Call used on opensource projects

Example 1 with Call

use of org.apache.hadoop.ipc.Server.Call in project hadoop by apache.

the class TestRPC method testClientBackOffByResponseTime.

/**
   *  Test RPC backoff by response time of each priority level.
   */
@Test(timeout = 30000)
public void testClientBackOffByResponseTime() throws Exception {
    final TestRpcService proxy;
    boolean succeeded = false;
    final int numClients = 1;
    GenericTestUtils.setLogLevel(DecayRpcScheduler.LOG, Level.DEBUG);
    GenericTestUtils.setLogLevel(RPC.LOG, Level.DEBUG);
    final List<Future<Void>> res = new ArrayList<Future<Void>>();
    final ExecutorService executorService = Executors.newFixedThreadPool(numClients);
    conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
    final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0";
    Server server = setupDecayRpcSchedulerandTestServer(ns + ".");
    @SuppressWarnings("unchecked") CallQueueManager<Call> spy = spy((CallQueueManager<Call>) Whitebox.getInternalState(server, "callQueue"));
    Whitebox.setInternalState(server, "callQueue", spy);
    Exception lastException = null;
    proxy = getClient(addr, conf);
    MetricsRecordBuilder rb1 = getMetrics("DecayRpcSchedulerMetrics2." + ns);
    final long beginDecayedCallVolume = MetricsAsserts.getLongCounter("DecayedCallVolume", rb1);
    final long beginRawCallVolume = MetricsAsserts.getLongCounter("CallVolume", rb1);
    final int beginUniqueCaller = MetricsAsserts.getIntCounter("UniqueCallers", rb1);
    try {
        // start a sleep RPC call that sleeps 3s.
        for (int i = 0; i < numClients; i++) {
            res.add(executorService.submit(new Callable<Void>() {

                @Override
                public Void call() throws ServiceException, InterruptedException {
                    proxy.sleep(null, newSleepRequest(3000));
                    return null;
                }
            }));
            verify(spy, timeout(500).times(i + 1)).offer(Mockito.<Call>anyObject());
        }
        // avg response time(3s) exceeds threshold (2s).
        try {
            // wait for the 1st response time update
            Thread.sleep(5500);
            proxy.sleep(null, newSleepRequest(100));
        } catch (ServiceException e) {
            RemoteException re = (RemoteException) e.getCause();
            IOException unwrapExeption = re.unwrapRemoteException();
            if (unwrapExeption instanceof RetriableException) {
                succeeded = true;
            } else {
                lastException = unwrapExeption;
            }
            // Lets Metric system update latest metrics
            GenericTestUtils.waitFor(new Supplier<Boolean>() {

                @Override
                public Boolean get() {
                    MetricsRecordBuilder rb2 = getMetrics("DecayRpcSchedulerMetrics2." + ns);
                    long decayedCallVolume1 = MetricsAsserts.getLongCounter("DecayedCallVolume", rb2);
                    long rawCallVolume1 = MetricsAsserts.getLongCounter("CallVolume", rb2);
                    int uniqueCaller1 = MetricsAsserts.getIntCounter("UniqueCallers", rb2);
                    long callVolumePriority0 = MetricsAsserts.getLongGauge("Priority.0.CompletedCallVolume", rb2);
                    long callVolumePriority1 = MetricsAsserts.getLongGauge("Priority.1.CompletedCallVolume", rb2);
                    double avgRespTimePriority0 = MetricsAsserts.getDoubleGauge("Priority.0.AvgResponseTime", rb2);
                    double avgRespTimePriority1 = MetricsAsserts.getDoubleGauge("Priority.1.AvgResponseTime", rb2);
                    LOG.info("DecayedCallVolume: " + decayedCallVolume1);
                    LOG.info("CallVolume: " + rawCallVolume1);
                    LOG.info("UniqueCaller: " + uniqueCaller1);
                    LOG.info("Priority.0.CompletedCallVolume: " + callVolumePriority0);
                    LOG.info("Priority.1.CompletedCallVolume: " + callVolumePriority1);
                    LOG.info("Priority.0.AvgResponseTime: " + avgRespTimePriority0);
                    LOG.info("Priority.1.AvgResponseTime: " + avgRespTimePriority1);
                    return decayedCallVolume1 > beginDecayedCallVolume && rawCallVolume1 > beginRawCallVolume && uniqueCaller1 > beginUniqueCaller;
                }
            }, 30, 60000);
        }
    } finally {
        executorService.shutdown();
        stop(server, proxy);
    }
    if (lastException != null) {
        LOG.error("Last received non-RetriableException:", lastException);
    }
    assertTrue("RetriableException not received", succeeded);
}

Example 2 with Call

use of org.apache.hadoop.ipc.Server.Call in project hadoop by apache.

the class TestIPC method checkBlocking.

// goal is to jam a handler with a connection, fill the callq with
// connections, in turn jamming the readers - then flood the server and
// ensure that the listener blocks when the reader connection queues fill
@SuppressWarnings("unchecked")
private void checkBlocking(int readers, int readerQ, int callQ) throws Exception {
    // makes it easier
    int handlers = 1;
    final Configuration conf = new Configuration();
    conf.setInt(CommonConfigurationKeys.IPC_SERVER_RPC_READ_CONNECTION_QUEUE_SIZE_KEY, readerQ);
    // send in enough clients to block up the handlers, callq, and readers
    final int initialClients = readers + callQ + handlers;
    // max connections we should ever end up accepting at once
    // 1 = listener
    final int maxAccept = initialClients + readers * readerQ + 1;
    // stress it with 2X the max
    int clients = maxAccept * 2;
    final AtomicInteger failures = new AtomicInteger(0);
    final CountDownLatch callFinishedLatch = new CountDownLatch(clients);
    // start server
    final TestServerQueue server = new TestServerQueue(clients, readers, callQ, handlers, conf);
    CallQueueManager<Call> spy = spy((CallQueueManager<Call>) Whitebox.getInternalState(server, "callQueue"));
    Whitebox.setInternalState(server, "callQueue", spy);
    final InetSocketAddress addr = NetUtils.getConnectAddress(server);
    server.start();
    Client.setConnectTimeout(conf, 10000);
    // instantiate the threads, will start in batches
    Thread[] threads = new Thread[clients];
    for (int i = 0; i < clients; i++) {
        threads[i] = new Thread(new Runnable() {

            @Override
            public void run() {
                Client client = new Client(LongWritable.class, conf);
                try {
                    call(client, new LongWritable(Thread.currentThread().getId()), addr, 60000, conf);
                } catch (Throwable e) {
                    LOG.error(e);
                    failures.incrementAndGet();
                    return;
                } finally {
                    callFinishedLatch.countDown();
                    client.stop();
                }
            }
        });
    }
    // and others not blocking in the race to fill the callq
    for (int i = 0; i < initialClients; i++) {
        threads[i].start();
        if (i == 0) {
            // let first reader block in a call
            server.firstCallLatch.await();
        }
        // wait until reader put a call to callQueue, to make sure all readers
        // are blocking on the queue after initialClients threads are started.
        verify(spy, timeout(100).times(i + 1)).put(Mockito.<Call>anyObject());
    }
    try {
        // wait till everything is slotted, should happen immediately
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                return server.getNumOpenConnections() >= initialClients;
            }
        }, 100, 3000);
    } catch (TimeoutException e) {
        fail("timed out while waiting for connections to open.");
    }
    LOG.info("(initial clients) need:" + initialClients + " connections have:" + server.getNumOpenConnections());
    LOG.info("ipc layer should be blocked");
    assertEquals(callQ, server.getCallQueueLen());
    assertEquals(initialClients, server.getNumOpenConnections());
    // connection queues should fill and then the listener should block
    for (int i = initialClients; i < clients; i++) {
        threads[i].start();
    }
    Thread.sleep(10);
    try {
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                return server.getNumOpenConnections() >= maxAccept;
            }
        }, 100, 3000);
    } catch (TimeoutException e) {
        fail("timed out while waiting for connections to open until maxAccept.");
    }
    LOG.info("(max clients) need:" + maxAccept + " connections have:" + server.getNumOpenConnections());
    // check a few times to make sure we didn't go over
    for (int i = 0; i < 4; i++) {
        assertEquals(maxAccept, server.getNumOpenConnections());
        Thread.sleep(100);
    }
    // sanity check that no calls have finished
    assertEquals(clients, callFinishedLatch.getCount());
    LOG.info("releasing the calls");
    server.callBlockLatch.countDown();
    callFinishedLatch.await();
    for (Thread t : threads) {
        t.join();
    }
    assertEquals(0, failures.get());
    server.stop();
}

Example 3 with Call

use of org.apache.hadoop.ipc.Server.Call in project hadoop by apache.

the class TestServer method testLogExceptions.

@Test(timeout = 300000)
public void testLogExceptions() throws Exception {
    final Configuration conf = new Configuration();
    final Call dummyCall = new Call(0, 0, null, null);
    Log logger = mock(Log.class);
    Server server = new Server("0.0.0.0", 0, LongWritable.class, 1, conf) {

        @Override
        public Writable call(RPC.RpcKind rpcKind, String protocol, Writable param, long receiveTime) throws Exception {
            return null;
        }
    };
    server.addSuppressedLoggingExceptions(TestException1.class);
    server.addTerseExceptions(TestException2.class);
    // Nothing should be logged for a suppressed exception.
    server.logException(logger, new TestException1(), dummyCall);
    verifyZeroInteractions(logger);
    // No stack trace should be logged for a terse exception.
    server.logException(logger, new TestException2(), dummyCall);
    verify(logger, times(1)).info(anyObject());
    // Full stack trace should be logged for other exceptions.
    final Throwable te3 = new TestException3();
    server.logException(logger, te3, dummyCall);
    verify(logger, times(1)).info(anyObject(), eq(te3));
}

Example 4 with Call

use of org.apache.hadoop.ipc.Server.Call in project hadoop by apache.

the class TestIPCServerResponder method testDeferResponse.

// Test that IPC calls can be marked for a deferred response.
// call 0: immediate
// call 1: immediate
// call 2: delayed with wait for 1 sendResponse, check if blocked
// call 3: immediate, proves handler is freed
// call 4: delayed with wait for 2 sendResponses, check if blocked
// call 2: sendResponse, should return
// call 4: sendResponse, should remain blocked
// call 5: immediate, prove handler is still free
// call 4: sendResponse, expect it to return
@Test(timeout = 10000)
public void testDeferResponse() throws IOException, InterruptedException {
    final AtomicReference<Call> deferredCall = new AtomicReference<Call>();
    final AtomicInteger count = new AtomicInteger();
    final Writable wait0 = new IntWritable(0);
    final Writable wait1 = new IntWritable(1);
    final Writable wait2 = new IntWritable(2);
    // use only 1 handler to prove it's freed after every call
    Server server = new Server(ADDRESS, 0, IntWritable.class, 1, conf) {

        @Override
        public Writable call(RPC.RpcKind rpcKind, String protocol, Writable waitCount, long receiveTime) throws IOException {
            Call call = Server.getCurCall().get();
            int wait = ((IntWritable) waitCount).get();
            while (wait-- > 0) {
                call.postponeResponse();
                deferredCall.set(call);
            }
            return new IntWritable(count.getAndIncrement());
        }
    };
    server.start();
    final InetSocketAddress address = NetUtils.getConnectAddress(server);
    final Client client = new Client(IntWritable.class, conf);
    Call[] waitingCalls = new Call[2];
    // calls should return immediately, check the sequence number is
    // increasing
    assertEquals(0, ((IntWritable) call(client, wait0, address)).get());
    assertEquals(1, ((IntWritable) call(client, wait0, address)).get());
    // do a call in the background that will have a deferred response
    final ExecutorService exec = Executors.newCachedThreadPool();
    Future<Integer> future1 = exec.submit(new Callable<Integer>() {

        @Override
        public Integer call() throws IOException {
            return ((IntWritable) TestIPCServerResponder.call(client, wait1, address)).get();
        }
    });
    // make sure it blocked
    try {
        future1.get(1, TimeUnit.SECONDS);
        Assert.fail("ipc shouldn't have responded");
    } catch (TimeoutException te) {
    // ignore, expected
    } catch (Exception ex) {
        Assert.fail("unexpected exception:" + ex);
    }
    assertFalse(future1.isDone());
    waitingCalls[0] = deferredCall.get();
    assertNotNull(waitingCalls[0]);
    // proves the handler isn't tied up, and that the prior sequence number
    // was consumed
    assertEquals(3, ((IntWritable) call(client, wait0, address)).get());
    // another call with wait count of 2
    Future<Integer> future2 = exec.submit(new Callable<Integer>() {

        @Override
        public Integer call() throws IOException {
            return ((IntWritable) TestIPCServerResponder.call(client, wait2, address)).get();
        }
    });
    // make sure it blocked
    try {
        future2.get(1, TimeUnit.SECONDS);
        Assert.fail("ipc shouldn't have responded");
    } catch (TimeoutException te) {
    // ignore, expected
    } catch (Exception ex) {
        Assert.fail("unexpected exception:" + ex);
    }
    assertFalse(future2.isDone());
    waitingCalls[1] = deferredCall.get();
    assertNotNull(waitingCalls[1]);
    // the background calls should still be blocked
    assertFalse(future1.isDone());
    assertFalse(future2.isDone());
    // trigger responses
    waitingCalls[0].sendResponse();
    waitingCalls[1].sendResponse();
    try {
        int val = future1.get(1, TimeUnit.SECONDS);
        assertEquals(2, val);
    } catch (Exception ex) {
        Assert.fail("unexpected exception:" + ex);
    }
    // make sure it's still blocked
    try {
        future2.get(1, TimeUnit.SECONDS);
        Assert.fail("ipc shouldn't have responded");
    } catch (TimeoutException te) {
    // ignore, expected
    } catch (Exception ex) {
        Assert.fail("unexpected exception:" + ex);
    }
    assertFalse(future2.isDone());
    // call should return immediately
    assertEquals(5, ((IntWritable) call(client, wait0, address)).get());
    // trigger last waiting call
    waitingCalls[1].sendResponse();
    try {
        int val = future2.get(1, TimeUnit.SECONDS);
        assertEquals(4, val);
    } catch (Exception ex) {
        Assert.fail("unexpected exception:" + ex);
    }
    server.stop();
}

Example 5 with Call

use of org.apache.hadoop.ipc.Server.Call in project hadoop by apache.

the class TestIPC method testCallIdAndRetry.

/**
   * Test if
   * (1) the rpc server uses the call id/retry provided by the rpc client, and
   * (2) the rpc client receives the same call id/retry from the rpc server.
   */
@Test(timeout = 60000)
public void testCallIdAndRetry() throws IOException {
    final CallInfo info = new CallInfo();
    // Override client to store the call info and check response
    final Client client = new Client(LongWritable.class, conf) {

        @Override
        Call createCall(RpcKind rpcKind, Writable rpcRequest) {
            final Call call = super.createCall(rpcKind, rpcRequest);
            info.id = call.id;
            info.retry = call.retry;
            return call;
        }

        @Override
        void checkResponse(RpcResponseHeaderProto header) throws IOException {
            super.checkResponse(header);
            Assert.assertEquals(info.id, header.getCallId());
            Assert.assertEquals(info.retry, header.getRetryCount());
        }
    };
    // Attach a listener that tracks every call received by the server.
    final TestServer server = new TestServer(1, false);
    server.callListener = new Runnable() {

        @Override
        public void run() {
            Assert.assertEquals(info.id, Server.getCallId());
            Assert.assertEquals(info.retry, Server.getCallRetryCount());
        }
    };
    try {
        InetSocketAddress addr = NetUtils.getConnectAddress(server);
        server.start();
        final SerialCaller caller = new SerialCaller(client, addr, 10);
        caller.run();
        assertFalse(caller.failed);
    } finally {
        client.stop();
        server.stop();
    }
}