Examples with Callable java.util.concurrent.Callable used on opensource projects

Example 11 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class DataStorage method linkBlocks.

private static void linkBlocks(File from, File to, int oldLV, HardLink hl, Configuration conf) throws IOException {
    LOG.info("Start linking block files from " + from + " to " + to);
    boolean upgradeToIdBasedLayout = false;
    // 256x256 layouts (-56) is fortunately the same.
    if (oldLV > DataNodeLayoutVersion.Feature.BLOCKID_BASED_LAYOUT_32_by_32.getInfo().getLayoutVersion() && to.getName().equals(STORAGE_DIR_FINALIZED)) {
        upgradeToIdBasedLayout = true;
    }
    final ArrayList<LinkArgs> idBasedLayoutSingleLinks = Lists.newArrayList();
    linkBlocksHelper(from, to, oldLV, hl, upgradeToIdBasedLayout, to, idBasedLayoutSingleLinks);
    // Detect and remove duplicate entries.
    final ArrayList<LinkArgs> duplicates = findDuplicateEntries(idBasedLayoutSingleLinks);
    if (!duplicates.isEmpty()) {
        LOG.error("There are " + duplicates.size() + " duplicate block " + "entries within the same volume.");
        removeDuplicateEntries(idBasedLayoutSingleLinks, duplicates);
    }
    final int numLinkWorkers = conf.getInt(DFSConfigKeys.DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS_KEY, DFSConfigKeys.DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS);
    ExecutorService linkWorkers = Executors.newFixedThreadPool(numLinkWorkers);
    final int step = idBasedLayoutSingleLinks.size() / numLinkWorkers + 1;
    List<Future<Void>> futures = Lists.newArrayList();
    for (int i = 0; i < idBasedLayoutSingleLinks.size(); i += step) {
        final int iCopy = i;
        futures.add(linkWorkers.submit(new Callable<Void>() {

            @Override
            public Void call() throws IOException {
                int upperBound = Math.min(iCopy + step, idBasedLayoutSingleLinks.size());
                for (int j = iCopy; j < upperBound; j++) {
                    LinkArgs cur = idBasedLayoutSingleLinks.get(j);
                    HardLink.createHardLink(cur.src, cur.dst);
                }
                return null;
            }
        }));
    }
    linkWorkers.shutdown();
    for (Future<Void> f : futures) {
        Futures.get(f, IOException.class);
    }
}

Example 12 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class TestRPC method testClientBackOffByResponseTime.

/**
   *  Test RPC backoff by response time of each priority level.
   */
@Test(timeout = 30000)
public void testClientBackOffByResponseTime() throws Exception {
    final TestRpcService proxy;
    boolean succeeded = false;
    final int numClients = 1;
    GenericTestUtils.setLogLevel(DecayRpcScheduler.LOG, Level.DEBUG);
    GenericTestUtils.setLogLevel(RPC.LOG, Level.DEBUG);
    final List<Future<Void>> res = new ArrayList<Future<Void>>();
    final ExecutorService executorService = Executors.newFixedThreadPool(numClients);
    conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
    final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0";
    Server server = setupDecayRpcSchedulerandTestServer(ns + ".");
    @SuppressWarnings("unchecked") CallQueueManager<Call> spy = spy((CallQueueManager<Call>) Whitebox.getInternalState(server, "callQueue"));
    Whitebox.setInternalState(server, "callQueue", spy);
    Exception lastException = null;
    proxy = getClient(addr, conf);
    MetricsRecordBuilder rb1 = getMetrics("DecayRpcSchedulerMetrics2." + ns);
    final long beginDecayedCallVolume = MetricsAsserts.getLongCounter("DecayedCallVolume", rb1);
    final long beginRawCallVolume = MetricsAsserts.getLongCounter("CallVolume", rb1);
    final int beginUniqueCaller = MetricsAsserts.getIntCounter("UniqueCallers", rb1);
    try {
        // start a sleep RPC call that sleeps 3s.
        for (int i = 0; i < numClients; i++) {
            res.add(executorService.submit(new Callable<Void>() {

                @Override
                public Void call() throws ServiceException, InterruptedException {
                    proxy.sleep(null, newSleepRequest(3000));
                    return null;
                }
            }));
            verify(spy, timeout(500).times(i + 1)).offer(Mockito.<Call>anyObject());
        }
        // avg response time(3s) exceeds threshold (2s).
        try {
            // wait for the 1st response time update
            Thread.sleep(5500);
            proxy.sleep(null, newSleepRequest(100));
        } catch (ServiceException e) {
            RemoteException re = (RemoteException) e.getCause();
            IOException unwrapExeption = re.unwrapRemoteException();
            if (unwrapExeption instanceof RetriableException) {
                succeeded = true;
            } else {
                lastException = unwrapExeption;
            }
            // Lets Metric system update latest metrics
            GenericTestUtils.waitFor(new Supplier<Boolean>() {

                @Override
                public Boolean get() {
                    MetricsRecordBuilder rb2 = getMetrics("DecayRpcSchedulerMetrics2." + ns);
                    long decayedCallVolume1 = MetricsAsserts.getLongCounter("DecayedCallVolume", rb2);
                    long rawCallVolume1 = MetricsAsserts.getLongCounter("CallVolume", rb2);
                    int uniqueCaller1 = MetricsAsserts.getIntCounter("UniqueCallers", rb2);
                    long callVolumePriority0 = MetricsAsserts.getLongGauge("Priority.0.CompletedCallVolume", rb2);
                    long callVolumePriority1 = MetricsAsserts.getLongGauge("Priority.1.CompletedCallVolume", rb2);
                    double avgRespTimePriority0 = MetricsAsserts.getDoubleGauge("Priority.0.AvgResponseTime", rb2);
                    double avgRespTimePriority1 = MetricsAsserts.getDoubleGauge("Priority.1.AvgResponseTime", rb2);
                    LOG.info("DecayedCallVolume: " + decayedCallVolume1);
                    LOG.info("CallVolume: " + rawCallVolume1);
                    LOG.info("UniqueCaller: " + uniqueCaller1);
                    LOG.info("Priority.0.CompletedCallVolume: " + callVolumePriority0);
                    LOG.info("Priority.1.CompletedCallVolume: " + callVolumePriority1);
                    LOG.info("Priority.0.AvgResponseTime: " + avgRespTimePriority0);
                    LOG.info("Priority.1.AvgResponseTime: " + avgRespTimePriority1);
                    return decayedCallVolume1 > beginDecayedCallVolume && rawCallVolume1 > beginRawCallVolume && uniqueCaller1 > beginUniqueCaller;
                }
            }, 30, 60000);
        }
    } finally {
        executorService.shutdown();
        stop(server, proxy);
    }
    if (lastException != null) {
        LOG.error("Last received non-RetriableException:", lastException);
    }
    assertTrue("RetriableException not received", succeeded);
}

Example 13 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class TestRPCServerShutdown method testRPCServerShutdown.

/**
   *  Verify the RPC server can shutdown properly when callQueue is full.
   */
@Test(timeout = 30000)
public void testRPCServerShutdown() throws Exception {
    final int numClients = 3;
    final List<Future<Void>> res = new ArrayList<Future<Void>>();
    final ExecutorService executorService = Executors.newFixedThreadPool(numClients);
    conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
    RPC.Builder builder = newServerBuilder(conf).setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true);
    final Server server = setupTestServer(builder);
    final TestRpcService proxy = getClient(addr, conf);
    try {
        // Start another sleep RPC call to make reader thread block on CallQueue.
        for (int i = 0; i < numClients; i++) {
            res.add(executorService.submit(new Callable<Void>() {

                @Override
                public Void call() throws ServiceException, InterruptedException {
                    proxy.sleep(null, newSleepRequest(100000));
                    return null;
                }
            }));
        }
        while (server.getCallQueueLen() != 1 || countThreads(CallQueueManager.class.getName()) != 1 || countThreads(PBServerImpl.class.getName()) != 1) {
            Thread.sleep(100);
        }
    } finally {
        try {
            stop(server, proxy);
            assertEquals("Not enough clients", numClients, res.size());
            for (Future<Void> f : res) {
                try {
                    f.get();
                    fail("Future get should not return");
                } catch (ExecutionException e) {
                    ServiceException se = (ServiceException) e.getCause();
                    assertTrue("Unexpected exception: " + se, se.getCause() instanceof IOException);
                    LOG.info("Expected exception", e.getCause());
                }
            }
        } finally {
            executorService.shutdown();
        }
    }
}

Example 14 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class TestRetryCache method testOperations.

public void testOperations(final int input, final int numberOfThreads, final int pause, final boolean success, final boolean attemptedBefore, final Server.Call call) throws InterruptedException, ExecutionException {
    final int failureOutput = input + 1;
    ExecutorService executorService = Executors.newFixedThreadPool(numberOfThreads);
    List<Future<Integer>> list = new ArrayList<Future<Integer>>();
    for (int i = 0; i < numberOfThreads; i++) {
        Callable<Integer> worker = new Callable<Integer>() {

            @Override
            public Integer call() throws Exception {
                Server.getCurCall().set(call);
                Assert.assertEquals(Server.getCurCall().get(), call);
                int randomPause = pause == 0 ? pause : r.nextInt(pause);
                return testServer.echo(input, failureOutput, randomPause, success);
            }
        };
        Future<Integer> submit = executorService.submit(worker);
        list.add(submit);
    }
    Assert.assertEquals(numberOfThreads, list.size());
    for (Future<Integer> future : list) {
        if (success) {
            Assert.assertEquals(input, future.get().intValue());
        } else {
            Assert.assertEquals(failureOutput, future.get().intValue());
        }
    }
    if (success) {
        // If the operation was successful, all the subsequent operations
        // by other threads should be retries. Operation count should be 1.
        int retries = numberOfThreads + (attemptedBefore ? 0 : -1);
        Assert.assertEquals(1, testServer.operationCount.get());
        Assert.assertEquals(retries, testServer.retryCount.get());
    } else {
        // If the operation failed, all the subsequent operations
        // should execute once more, hence the retry count should be 0 and
        // operation count should be the number of tries
        int opCount = numberOfThreads + (attemptedBefore ? 1 : 0);
        Assert.assertEquals(opCount, testServer.operationCount.get());
        Assert.assertEquals(0, testServer.retryCount.get());
    }
}

Example 15 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class TestByteArrayManager method performanceTest.

static long performanceTest(final int arrayLength, final int maxArrays, final int nThreads, final int[] sleepTimeMSs, final ByteArrayManager impl) throws Exception {
    final ExecutorService pool = Executors.newFixedThreadPool(nThreads);
    final List<Future<Void>> futures = new ArrayList<Future<Void>>(sleepTimeMSs.length);
    final long startTime = Time.monotonicNow();
    for (int i = 0; i < sleepTimeMSs.length; i++) {
        final long sleepTime = sleepTimeMSs[i];
        futures.add(pool.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                byte[] array = impl.newByteArray(arrayLength);
                sleepMs(sleepTime);
                impl.release(array);
                return null;
            }
        }));
    }
    for (Future<Void> f : futures) {
        f.get();
    }
    final long endTime = Time.monotonicNow();
    pool.shutdown();
    return endTime - startTime;
}