Search in sources :

Example 96 with ExecutorService

use of java.util.concurrent.ExecutorService in project hadoop by apache.

the class ITestS3ADeleteManyFiles method testBulkRenameAndDelete.

/**
   * CAUTION: If this test starts failing, please make sure that the
   * {@link org.apache.hadoop.fs.s3a.Constants#MAX_THREADS} configuration is not
   * set too low. Alternatively, consider reducing the
   * <code>scale.test.operation.count</code> parameter in
   * <code>getOperationCount()</code>.
   *
   * @see #getOperationCount()
   */
@Test
public void testBulkRenameAndDelete() throws Throwable {
    final Path scaleTestDir = path("testBulkRenameAndDelete");
    final Path srcDir = new Path(scaleTestDir, "src");
    final Path finalDir = new Path(scaleTestDir, "final");
    final long count = getOperationCount();
    final S3AFileSystem fs = getFileSystem();
    ContractTestUtils.rm(fs, scaleTestDir, true, false);
    fs.mkdirs(srcDir);
    fs.mkdirs(finalDir);
    int testBufferSize = fs.getConf().getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE, ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
    // use Executor to speed up file creation
    ExecutorService exec = Executors.newFixedThreadPool(16);
    final ExecutorCompletionService<Boolean> completionService = new ExecutorCompletionService<>(exec);
    try {
        final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');
        for (int i = 0; i < count; ++i) {
            final String fileName = "foo-" + i;
            completionService.submit(new Callable<Boolean>() {

                @Override
                public Boolean call() throws IOException {
                    ContractTestUtils.createFile(fs, new Path(srcDir, fileName), false, data);
                    return fs.exists(new Path(srcDir, fileName));
                }
            });
        }
        for (int i = 0; i < count; ++i) {
            final Future<Boolean> future = completionService.take();
            try {
                if (!future.get()) {
                    LOG.warn("cannot create file");
                }
            } catch (ExecutionException e) {
                LOG.warn("Error while uploading file", e.getCause());
                throw e;
            }
        }
    } finally {
        exec.shutdown();
    }
    int nSrcFiles = fs.listStatus(srcDir).length;
    fs.rename(srcDir, finalDir);
    assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
    ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + 0));
    ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + count / 2));
    ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename", new Path(srcDir, "foo-" + (count - 1)));
    ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + 0));
    ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + count / 2));
    ContractTestUtils.assertPathExists(fs, "not renamed to dest dir", new Path(finalDir, "foo-" + (count - 1)));
    ContractTestUtils.assertDeleted(fs, finalDir, true, false);
}
Also used : Path(org.apache.hadoop.fs.Path) S3AFileSystem(org.apache.hadoop.fs.s3a.S3AFileSystem) ExecutorCompletionService(java.util.concurrent.ExecutorCompletionService) IOException(java.io.IOException) ExecutorService(java.util.concurrent.ExecutorService) ExecutionException(java.util.concurrent.ExecutionException) Test(org.junit.Test)

Example 97 with ExecutorService

use of java.util.concurrent.ExecutorService in project hadoop by apache.

the class TestNodeStatusUpdater method testConcurrentAccessToSystemCredentials.

@Test
public void testConcurrentAccessToSystemCredentials() {
    final Map<ApplicationId, ByteBuffer> testCredentials = new HashMap<>();
    ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[300]);
    ApplicationId applicationId = ApplicationId.newInstance(123456, 120);
    testCredentials.put(applicationId, byteBuffer);
    final List<Throwable> exceptions = Collections.synchronizedList(new ArrayList<Throwable>());
    final int NUM_THREADS = 10;
    final CountDownLatch allDone = new CountDownLatch(NUM_THREADS);
    final ExecutorService threadPool = HadoopExecutors.newFixedThreadPool(NUM_THREADS);
    final AtomicBoolean stop = new AtomicBoolean(false);
    try {
        for (int i = 0; i < NUM_THREADS; i++) {
            threadPool.submit(new Runnable() {

                @Override
                public void run() {
                    try {
                        for (int i = 0; i < 100 && !stop.get(); i++) {
                            NodeHeartbeatResponse nodeHeartBeatResponse = newNodeHeartbeatResponse(0, NodeAction.NORMAL, null, null, null, null, 0);
                            nodeHeartBeatResponse.setSystemCredentialsForApps(testCredentials);
                            NodeHeartbeatResponseProto proto = ((NodeHeartbeatResponsePBImpl) nodeHeartBeatResponse).getProto();
                            Assert.assertNotNull(proto);
                        }
                    } catch (Throwable t) {
                        exceptions.add(t);
                        stop.set(true);
                    } finally {
                        allDone.countDown();
                    }
                }
            });
        }
        int testTimeout = 2;
        Assert.assertTrue("Timeout waiting for more than " + testTimeout + " " + "seconds", allDone.await(testTimeout, TimeUnit.SECONDS));
    } catch (InterruptedException ie) {
        exceptions.add(ie);
    } finally {
        threadPool.shutdownNow();
    }
    Assert.assertTrue("Test failed with exception(s)" + exceptions, exceptions.isEmpty());
}
Also used : YarnServerBuilderUtils.newNodeHeartbeatResponse(org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils.newNodeHeartbeatResponse) NodeHeartbeatResponse(org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse) HashMap(java.util.HashMap) NodeHeartbeatResponseProto(org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto) CountDownLatch(java.util.concurrent.CountDownLatch) ByteBuffer(java.nio.ByteBuffer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ExecutorService(java.util.concurrent.ExecutorService) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Test(org.junit.Test)

Example 98 with ExecutorService

use of java.util.concurrent.ExecutorService in project camel by apache.

the class LevelDBAggregateConcurrentDifferentGroupsTest method doSendMessages.

private void doSendMessages(int files, int poolSize) throws Exception {
    MockEndpoint mock = getMockEndpoint("mock:aggregated");
    mock.expectedMessageCount(2);
    mock.setResultWaitTime(30 * 1000L);
    ExecutorService executor = Executors.newFixedThreadPool(poolSize);
    for (int i = 0; i < files; i++) {
        final int index = i;
        executor.submit(new Callable<Object>() {

            public Object call() throws Exception {
                String id = index % 2 == 0 ? "A" : "B";
                template.sendBodyAndHeader("direct:start", index, "id", id);
                // simulate a little delay
                Thread.sleep(3);
                return null;
            }
        });
    }
    assertMockEndpointsSatisfied();
    executor.shutdownNow();
}
Also used : MockEndpoint(org.apache.camel.component.mock.MockEndpoint) ExecutorService(java.util.concurrent.ExecutorService) MockEndpoint(org.apache.camel.component.mock.MockEndpoint)

Example 99 with ExecutorService

use of java.util.concurrent.ExecutorService in project camel by apache.

the class LevelDBAggregateConcurrentSameGroupTest method doSendMessages.

private void doSendMessages(int files, int poolSize) throws Exception {
    MockEndpoint mock = getMockEndpoint("mock:aggregated");
    mock.setResultWaitTime(30 * 1000L);
    mock.expectedMessageCount(1);
    ExecutorService executor = Executors.newFixedThreadPool(poolSize);
    for (int i = 0; i < files; i++) {
        final int index = i;
        executor.submit(new Callable<Object>() {

            public Object call() throws Exception {
                template.sendBodyAndHeader("direct:start", index, "id", 123);
                // simulate a little delay
                Thread.sleep(3);
                return null;
            }
        });
    }
    assertMockEndpointsSatisfied();
    executor.shutdownNow();
}
Also used : MockEndpoint(org.apache.camel.component.mock.MockEndpoint) ExecutorService(java.util.concurrent.ExecutorService) MockEndpoint(org.apache.camel.component.mock.MockEndpoint)

Example 100 with ExecutorService

use of java.util.concurrent.ExecutorService in project camel by apache.

the class MailProducerConcurrentTest method doSendMessages.

private void doSendMessages(int files, int poolSize) throws Exception {
    Mailbox.clearAll();
    NotifyBuilder builder = new NotifyBuilder(context).whenDone(files).create();
    getMockEndpoint("mock:result").expectedMessageCount(files);
    getMockEndpoint("mock:result").expectsNoDuplicates(body());
    final CountDownLatch latch = new CountDownLatch(files);
    ExecutorService executor = Executors.newFixedThreadPool(poolSize);
    for (int i = 0; i < files; i++) {
        final int index = i;
        executor.submit(new Callable<Object>() {

            public Object call() throws Exception {
                template.sendBodyAndHeader("direct:start", "Message " + index, "To", "someone@localhost");
                latch.countDown();
                return null;
            }
        });
    }
    // wait first for all the exchanges above to be thoroughly sent asynchronously
    assertTrue(latch.await(5, TimeUnit.SECONDS));
    assertMockEndpointsSatisfied();
    assertTrue(builder.matchesMockWaitTime());
    Mailbox box = Mailbox.get("someone@localhost");
    assertEquals(files, box.size());
    // as we use concurrent producers the mails can arrive out of order
    Set<Object> bodies = new HashSet<Object>();
    for (int i = 0; i < files; i++) {
        bodies.add(box.get(i).getContent());
    }
    assertEquals("There should be " + files + " unique mails", files, bodies.size());
    executor.shutdownNow();
}
Also used : NotifyBuilder(org.apache.camel.builder.NotifyBuilder) Mailbox(org.jvnet.mock_javamail.Mailbox) ExecutorService(java.util.concurrent.ExecutorService) CountDownLatch(java.util.concurrent.CountDownLatch) HashSet(java.util.HashSet)

Aggregations

ExecutorService (java.util.concurrent.ExecutorService)4945 Test (org.junit.Test)1738 ArrayList (java.util.ArrayList)1223 Future (java.util.concurrent.Future)1121 CountDownLatch (java.util.concurrent.CountDownLatch)757 IOException (java.io.IOException)734 Callable (java.util.concurrent.Callable)634 ExecutionException (java.util.concurrent.ExecutionException)564 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)366 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)322 HashMap (java.util.HashMap)275 List (java.util.List)270 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)249 Test (org.testng.annotations.Test)244 TimeoutException (java.util.concurrent.TimeoutException)223 Map (java.util.Map)217 File (java.io.File)213 HashSet (java.util.HashSet)190 Test (org.junit.jupiter.api.Test)174 ThreadPoolExecutor (java.util.concurrent.ThreadPoolExecutor)170