Examples with ExecutorService java.util.concurrent.ExecutorService used on opensource projects

Example 96 with ExecutorService

use of java.util.concurrent.ExecutorService in project hadoop by apache.

the class ITestS3ADeleteManyFiles method testBulkRenameAndDelete.

/**
   * CAUTION: If this test starts failing, please make sure that the
   * {@link org.apache.hadoop.fs.s3a.Constants#MAX_THREADS} configuration is not
   * set too low. Alternatively, consider reducing the
   * <code>scale.test.operation.count</code> parameter in
   * <code>getOperationCount()</code>.
   *
   * @see #getOperationCount()
   */
@Test
public void testBulkRenameAndDelete() throws Throwable {
    final Path scaleTestDir = path("testBulkRenameAndDelete");
    final Path srcDir = new Path(scaleTestDir, "src");
    final Path finalDir = new Path(scaleTestDir, "final");
    final long count = getOperationCount();
    final S3AFileSystem fs = getFileSystem();
    ContractTestUtils.rm(fs, scaleTestDir, true, false);
    fs.mkdirs(srcDir);
    fs.mkdirs(finalDir);
    int testBufferSize = fs.getConf().getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE, ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
    // use Executor to speed up file creation
    ExecutorService exec = Executors.newFixedThreadPool(16);
    final ExecutorCompletionService<Boolean> completionService = new ExecutorCompletionService<>(exec);
    try {
        final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');
        for (int i = 0; i < count; ++i) {
            final String fileName = "foo-" + i;
            completionService.submit(new Callable<Boolean>() {

                @Override
                public Boolean call() throws IOException {
                    ContractTestUtils.createFile(fs, new Path(srcDir, fileName), false, data);
                    return fs.exists(new Path(srcDir, fileName));
                }
            });
        }
        for (int i = 0; i < count; ++i) {
            final Future<Boolean> future = completionService.take();
            try {
                if (!future.get()) {
                    LOG.warn("cannot create file");
                }
            } catch (ExecutionException e) {
                LOG.warn("Error while uploading file", e.getCause());
                throw e;
            }
        }
    } finally {
        exec.shutdown();
    }
    int nSrcFiles = fs.listStatus(srcDir).length;
    fs.rename(srcDir, finalDir);
    assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
    ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + 0));
    ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + count / 2));
    ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + (count - 1)));
    ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + 0));
    ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + count / 2));
    ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + (count - 1)));
    ContractTestUtils.assertDeleted(fs, finalDir, true, false);
}

Example 97 with ExecutorService

use of java.util.concurrent.ExecutorService in project hadoop by apache.

the class TestNodeStatusUpdater method testConcurrentAccessToSystemCredentials.

@Test
public void testConcurrentAccessToSystemCredentials() {
    final Map<ApplicationId, ByteBuffer> testCredentials = new HashMap<>();
    ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[300]);
    ApplicationId applicationId = ApplicationId.newInstance(123456, 120);
    testCredentials.put(applicationId, byteBuffer);
    final List<Throwable> exceptions = Collections.synchronizedList(new ArrayList<Throwable>());
    final int NUM_THREADS = 10;
    final CountDownLatch allDone = new CountDownLatch(NUM_THREADS);
    final ExecutorService threadPool = HadoopExecutors.newFixedThreadPool(NUM_THREADS);
    final AtomicBoolean stop = new AtomicBoolean(false);
    try {
        for (int i = 0; i < NUM_THREADS; i++) {
            threadPool.submit(new Runnable() {

                @Override
                public void run() {
                    try {
                        for (int i = 0; i < 100 && !stop.get(); i++) {
                            NodeHeartbeatResponse nodeHeartBeatResponse = newNodeHeartbeatResponse(0, NodeAction.NORMAL, null, null, null, null, 0);
                            nodeHeartBeatResponse.setSystemCredentialsForApps(testCredentials);
                            NodeHeartbeatResponseProto proto = ((NodeHeartbeatResponsePBImpl) nodeHeartBeatResponse).getProto();
                            Assert.assertNotNull(proto);
                        }
                    } catch (Throwable t) {
                        exceptions.add(t);
                        stop.set(true);
                    } finally {
                        allDone.countDown();
                    }
                }
            });
        }
        int testTimeout = 2;
        Assert.assertTrue("Timeout waiting for more than " + testTimeout + " " + "seconds", allDone.await(testTimeout, TimeUnit.SECONDS));
    } catch (InterruptedException ie) {
        exceptions.add(ie);
    } finally {
        threadPool.shutdownNow();
    }
    Assert.assertTrue("Test failed with exception(s)" + exceptions, exceptions.isEmpty());
}

Example 98 with ExecutorService

use of java.util.concurrent.ExecutorService in project camel by apache.

the class LevelDBAggregateConcurrentDifferentGroupsTest method doSendMessages.

private void doSendMessages(int files, int poolSize) throws Exception {
    MockEndpoint mock = getMockEndpoint("mock:aggregated");
    mock.expectedMessageCount(2);
    mock.setResultWaitTime(30 * 1000L);
    ExecutorService executor = Executors.newFixedThreadPool(poolSize);
    for (int i = 0; i < files; i++) {
        final int index = i;
        executor.submit(new Callable<Object>() {

            public Object call() throws Exception {
                String id = index % 2 == 0 ? "A" : "B";
                template.sendBodyAndHeader("direct:start", index, "id", id);
                // simulate a little delay
                Thread.sleep(3);
                return null;
            }
        });
    }
    assertMockEndpointsSatisfied();
    executor.shutdownNow();
}

Example 99 with ExecutorService

use of java.util.concurrent.ExecutorService in project camel by apache.

the class LevelDBAggregateConcurrentSameGroupTest method doSendMessages.

private void doSendMessages(int files, int poolSize) throws Exception {
    MockEndpoint mock = getMockEndpoint("mock:aggregated");
    mock.setResultWaitTime(30 * 1000L);
    mock.expectedMessageCount(1);
    ExecutorService executor = Executors.newFixedThreadPool(poolSize);
    for (int i = 0; i < files; i++) {
        final int index = i;
        executor.submit(new Callable<Object>() {

            public Object call() throws Exception {
                template.sendBodyAndHeader("direct:start", index, "id", 123);
                // simulate a little delay
                Thread.sleep(3);
                return null;
            }
        });
    }
    assertMockEndpointsSatisfied();
    executor.shutdownNow();
}

Example 100 with ExecutorService

use of java.util.concurrent.ExecutorService in project camel by apache.

the class MailProducerConcurrentTest method doSendMessages.

private void doSendMessages(int files, int poolSize) throws Exception {
    Mailbox.clearAll();
    NotifyBuilder builder = new NotifyBuilder(context).whenDone(files).create();
    getMockEndpoint("mock:result").expectedMessageCount(files);
    getMockEndpoint("mock:result").expectsNoDuplicates(body());
    final CountDownLatch latch = new CountDownLatch(files);
    ExecutorService executor = Executors.newFixedThreadPool(poolSize);
    for (int i = 0; i < files; i++) {
        final int index = i;
        executor.submit(new Callable<Object>() {

            public Object call() throws Exception {
                template.sendBodyAndHeader("direct:start", "Message " + index, "To", "someone@localhost");
                latch.countDown();
                return null;
            }
        });
    }
    // wait first for all the exchanges above to be thoroughly sent asynchronously
    assertTrue(latch.await(5, TimeUnit.SECONDS));
    assertMockEndpointsSatisfied();
    assertTrue(builder.matchesMockWaitTime());
    Mailbox box = Mailbox.get("someone@localhost");
    assertEquals(files, box.size());
    // as we use concurrent producers the mails can arrive out of order
    Set<Object> bodies = new HashSet<Object>();
    for (int i = 0; i < files; i++) {
        bodies.add(box.get(i).getContent());
    }
    assertEquals("There should be " + files + " unique mails", files, bodies.size());
    executor.shutdownNow();
}