Examples with Callable java.util.concurrent.Callable used on opensource projects

Example 16 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class TestBlockManager method testBlockReportQueueing.

@Test
public void testBlockReportQueueing() throws Exception {
    Configuration conf = new HdfsConfiguration();
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
    try {
        cluster.waitActive();
        final FSNamesystem fsn = cluster.getNamesystem();
        final BlockManager bm = fsn.getBlockManager();
        final ExecutorService executor = Executors.newCachedThreadPool();
        final CyclicBarrier startBarrier = new CyclicBarrier(2);
        final CountDownLatch endLatch = new CountDownLatch(3);
        final CountDownLatch doneLatch = new CountDownLatch(1);
        // create a task intended to block while processing, thus causing
        // the queue to backup.  simulates how a full BR is processed.
        FutureTask<?> blockingOp = new FutureTask<Void>(new Callable<Void>() {

            @Override
            public Void call() throws IOException {
                bm.runBlockOp(new Callable<Void>() {

                    @Override
                    public Void call() throws InterruptedException, BrokenBarrierException {
                        // use a barrier to control the blocking.
                        startBarrier.await();
                        endLatch.countDown();
                        return null;
                    }
                });
                // signal that runBlockOp returned
                doneLatch.countDown();
                return null;
            }
        });
        // create an async task.  simulates how an IBR is processed.
        Callable<?> asyncOp = new Callable<Void>() {

            @Override
            public Void call() throws IOException {
                bm.enqueueBlockOp(new Runnable() {

                    @Override
                    public void run() {
                        // use the latch to signal if the op has run.
                        endLatch.countDown();
                    }
                });
                return null;
            }
        };
        // calling get forces its execution so we can test if it's blocked.
        Future<?> blockedFuture = executor.submit(blockingOp);
        boolean isBlocked = false;
        try {
            // wait 1s for the future to block.  it should run instantaneously.
            blockedFuture.get(1, TimeUnit.SECONDS);
        } catch (TimeoutException te) {
            isBlocked = true;
        }
        assertTrue(isBlocked);
        // should effectively return immediately since calls are queued.
        // however they should be backed up in the queue behind the blocking
        // operation.
        executor.submit(asyncOp).get(1, TimeUnit.SECONDS);
        executor.submit(asyncOp).get(1, TimeUnit.SECONDS);
        // check the async calls are queued, and first is still blocked.
        assertEquals(2, bm.getBlockOpQueueLength());
        assertFalse(blockedFuture.isDone());
        // unblock the queue, wait for last op to complete, check the blocked
        // call has returned
        startBarrier.await(1, TimeUnit.SECONDS);
        assertTrue(endLatch.await(1, TimeUnit.SECONDS));
        assertEquals(0, bm.getBlockOpQueueLength());
        assertTrue(doneLatch.await(1, TimeUnit.SECONDS));
    } finally {
        cluster.shutdown();
    }
}

Example 17 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class TestCodecPool method testMultiThreadedDecompressorPool.

@Test(timeout = 1000)
public void testMultiThreadedDecompressorPool() throws InterruptedException {
    final int iterations = 4;
    ExecutorService threadpool = Executors.newFixedThreadPool(3);
    final LinkedBlockingDeque<Decompressor> queue = new LinkedBlockingDeque<Decompressor>(2 * iterations);
    Callable<Boolean> consumer = new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            Decompressor dc = queue.take();
            CodecPool.returnDecompressor(dc);
            return dc != null;
        }
    };
    Callable<Boolean> producer = new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            Decompressor c = CodecPool.getDecompressor(codec);
            queue.put(c);
            return c != null;
        }
    };
    for (int i = 0; i < iterations; i++) {
        threadpool.submit(consumer);
        threadpool.submit(producer);
    }
    // wait for completion
    threadpool.shutdown();
    threadpool.awaitTermination(1000, TimeUnit.SECONDS);
    assertEquals(LEASE_COUNT_ERR, 0, CodecPool.getLeasedDecompressorsCount(codec));
}

Example 18 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class BlockReportTestBase method testInterleavedBlockReports.

// See HDFS-10301
@Test(timeout = 300000)
public void testInterleavedBlockReports() throws IOException, ExecutionException, InterruptedException {
    int numConcurrentBlockReports = 3;
    DataNode dn = cluster.getDataNodes().get(DN_N0);
    final String poolId = cluster.getNamesystem().getBlockPoolId();
    LOG.info("Block pool id: " + poolId);
    final DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
    final StorageBlockReport[] reports = getBlockReports(dn, poolId, true, true);
    // Get the list of storage ids associated with the datanode
    // before the test
    BlockManager bm = cluster.getNameNode().getNamesystem().getBlockManager();
    final DatanodeDescriptor dnDescriptor = bm.getDatanodeManager().getDatanode(dn.getDatanodeId());
    DatanodeStorageInfo[] storageInfos = dnDescriptor.getStorageInfos();
    // Send the block report concurrently using
    // numThreads=numConcurrentBlockReports
    ExecutorService executorService = Executors.newFixedThreadPool(numConcurrentBlockReports);
    List<Future<Void>> futureList = new ArrayList<>(numConcurrentBlockReports);
    for (int i = 0; i < numConcurrentBlockReports; i++) {
        futureList.add(executorService.submit(new Callable<Void>() {

            @Override
            public Void call() throws IOException {
                sendBlockReports(dnR, poolId, reports);
                return null;
            }
        }));
    }
    for (Future<Void> future : futureList) {
        future.get();
    }
    executorService.shutdown();
    // Verify that the storages match before and after the test
    Assert.assertArrayEquals(storageInfos, dnDescriptor.getStorageInfos());
}

Example 19 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class TestDomainSocket method testAsyncCloseDuringIO.

/**
   * Test that we get an AsynchronousCloseException when the DomainSocket
   * we're using is closed during a read or write operation.
   *
   * @throws IOException
   */
private void testAsyncCloseDuringIO(final boolean closeDuringWrite) throws Exception {
    final String TEST_PATH = new File(sockDir.getDir(), "testAsyncCloseDuringIO(" + closeDuringWrite + ")").getAbsolutePath();
    final DomainSocket serv = DomainSocket.bindAndListen(TEST_PATH);
    ExecutorService exeServ = Executors.newFixedThreadPool(2);
    Callable<Void> serverCallable = new Callable<Void>() {

        public Void call() {
            DomainSocket serverConn = null;
            try {
                serverConn = serv.accept();
                byte[] buf = new byte[100];
                for (int i = 0; i < buf.length; i++) {
                    buf[i] = 0;
                }
                // reads return EOF, and writes get a socket error.
                if (closeDuringWrite) {
                    try {
                        while (true) {
                            serverConn.getOutputStream().write(buf);
                        }
                    } catch (IOException e) {
                    }
                } else {
                    do {
                        ;
                    } while (serverConn.getInputStream().read(buf, 0, buf.length) != -1);
                }
            } catch (IOException e) {
                throw new RuntimeException("unexpected IOException", e);
            } finally {
                IOUtils.cleanup(DomainSocket.LOG, serverConn);
            }
            return null;
        }
    };
    Future<Void> serverFuture = exeServ.submit(serverCallable);
    final DomainSocket clientConn = DomainSocket.connect(serv.getPath());
    Callable<Void> clientCallable = new Callable<Void>() {

        public Void call() {
            // The client writes or reads until another thread
            // asynchronously closes the socket.  At that point, we should
            // get ClosedChannelException, or possibly its subclass
            // AsynchronousCloseException.
            byte[] buf = new byte[100];
            for (int i = 0; i < buf.length; i++) {
                buf[i] = 0;
            }
            try {
                if (closeDuringWrite) {
                    while (true) {
                        clientConn.getOutputStream().write(buf);
                    }
                } else {
                    while (true) {
                        clientConn.getInputStream().read(buf, 0, buf.length);
                    }
                }
            } catch (ClosedChannelException e) {
                return null;
            } catch (IOException e) {
                throw new RuntimeException("unexpected IOException", e);
            }
        }
    };
    Future<Void> clientFuture = exeServ.submit(clientCallable);
    Thread.sleep(500);
    clientConn.close();
    serv.close();
    clientFuture.get(2, TimeUnit.MINUTES);
    serverFuture.get(2, TimeUnit.MINUTES);
}

Example 20 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class TestDomainSocket method testSocketAcceptAndClose.

/**
   * Test that if one thread is blocking in a read or write operation, another
   * thread can close the socket and stop the accept.
   *
   * @throws IOException
   */
@Test(timeout = 180000)
public void testSocketAcceptAndClose() throws Exception {
    final String TEST_PATH = new File(sockDir.getDir(), "test_sock_accept_and_close").getAbsolutePath();
    final DomainSocket serv = DomainSocket.bindAndListen(TEST_PATH);
    ExecutorService exeServ = Executors.newSingleThreadExecutor();
    Callable<Void> callable = new Callable<Void>() {

        public Void call() {
            try {
                serv.accept();
                throw new RuntimeException("expected the accept() to be " + "interrupted and fail");
            } catch (AsynchronousCloseException e) {
                return null;
            } catch (IOException e) {
                throw new RuntimeException("unexpected IOException", e);
            }
        }
    };
    Future<Void> future = exeServ.submit(callable);
    Thread.sleep(500);
    serv.close();
    future.get(2, TimeUnit.MINUTES);
}