Examples with ThreadPoolExecutor java.util.concurrent.ThreadPoolExecutor used on opensource projects

Example 21 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hadoop by apache.

the class TestFileSystemOperationsWithThreads method testDeleteThreadPoolTerminationFailure.

/*
   * Test case for delete operation with multiple threads and flat listing enabled.
   */
@Test
public void testDeleteThreadPoolTerminationFailure() throws Exception {
    // Spy azure file system object and return mocked thread pool
    NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
    // Spy a thread pool executor and link it to azure file system object.
    String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
    AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(((NativeAzureFileSystem) fs).getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
    // Create a thread executor and link it to mocked thread pool executor object.
    // Mock thread executor to throw exception for terminating threads.
    ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
    Mockito.doNothing().when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
    Mockito.when(mockThreadExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS)).thenThrow(new InterruptedException());
    Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
    // With single iteration, we would have created 7 blobs resulting 7 threads.
    Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
    createFolder(mockFs, "root");
    Path sourceFolder = new Path("root");
    boolean exception = false;
    try {
        mockFs.delete(sourceFolder, true);
    } catch (IOException e) {
        exception = true;
    }
    assertTrue(exception);
    assertTrue(mockFs.exists(sourceFolder));
    // Validate from logs that threads are enabled and delete operation is failed.
    String content = logs.getOutput();
    assertTrue(content.contains("Using thread pool for Delete operation with threads"));
    assertTrue(content.contains("Threads got interrupted Delete blob operation"));
    assertTrue(content.contains("Delete failed as operation on subfolders and files failed."));
}

Example 22 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hadoop by apache.

the class TestFileSystemOperationsWithThreads method testRenameThreadPoolExecuteSingleThreadFailure.

/*
   * Test case for rename operation with multiple threads and flat listing enabled.
   */
@Test
public void testRenameThreadPoolExecuteSingleThreadFailure() throws Exception {
    // Spy azure file system object and return mocked thread pool
    NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
    // Spy a thread pool executor and link it to azure file system object.
    String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
    AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", path, NativeAzureFileSystem.AZURE_RENAME_THREADS));
    // With single iteration, we would have created 7 blobs resulting 7 threads.
    Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor);
    // Create a thread executor and link it to mocked thread pool executor object.
    ThreadPoolExecutor mockThreadExecutor = Mockito.spy(mockThreadPoolExecutor.getThreadPool(7));
    Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
    // Mock thread executor to throw exception for all requests.
    Mockito.doCallRealMethod().doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
    validateRenameFolder(mockFs, "root", "rootnew");
    // Validate from logs that threads are enabled and unused threads exists.
    String content = logs.getOutput();
    assertTrue(content.contains("Using thread pool for Rename operation with threads 7"));
    assertTrue(content.contains("6 threads not used for Rename operation on blob"));
}

Example 23 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hadoop by apache.

the class TestFileSystemOperationsWithThreads method testDeleteThreadPoolExecuteSingleThreadFailure.

/*
   * Test case for delete operation with multiple threads and flat listing enabled.
   */
@Test
public void testDeleteThreadPoolExecuteSingleThreadFailure() throws Exception {
    // Spy azure file system object and return mocked thread pool
    NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
    // Spy a thread pool executor and link it to azure file system object.
    String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
    AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
    // With single iteration, we would have created 7 blobs resulting 7 threads.
    Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
    // Create a thread executor and link it to mocked thread pool executor object.
    ThreadPoolExecutor mockThreadExecutor = Mockito.spy(mockThreadPoolExecutor.getThreadPool(7));
    Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
    // Mock thread executor to throw exception for all requests.
    Mockito.doCallRealMethod().doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
    validateDeleteFolder(mockFs, "root");
    // Validate from logs that threads are enabled and unused threads.
    String content = logs.getOutput();
    assertTrue(content.contains("Using thread pool for Delete operation with threads 7"));
    assertTrue(content.contains("6 threads not used for Delete operation on blob"));
}

Example 24 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hadoop by apache.

the class TestFileSystemOperationsWithThreads method testDeleteThreadPoolExecuteFailure.

/*
   * Test case for delete operation with multiple threads and flat listing enabled.
   */
@Test
public void testDeleteThreadPoolExecuteFailure() throws Exception {
    // Mock thread pool executor to throw exception for all requests.
    ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
    Mockito.doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
    // Spy azure file system object and return mocked thread pool
    NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
    String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
    AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
    Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
    // With single iteration, we would have created 7 blobs resulting 7 threads.
    Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
    validateDeleteFolder(mockFs, "root");
    // Validate from logs that threads are disabled.
    String content = logs.getOutput();
    assertTrue(content.contains("Rejected execution of thread for Delete operation on blob"));
    assertTrue(content.contains("Serializing the Delete operation"));
}

Example 25 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hadoop by apache.

the class TestLogAggregationService method testInvalidThreadPoolSizeValue.

private void testInvalidThreadPoolSizeValue(final String threadPoolSize) throws IOException {
    Supplier<Boolean> isInputInvalid = new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            try {
                int value = Integer.parseInt(threadPoolSize);
                return value <= 0;
            } catch (NumberFormatException ex) {
                return true;
            }
        }
    };
    assertTrue("The thread pool size must be invalid to use with this " + "method", isInputInvalid.get());
    // store configured thread pool size temporarily for restoration
    int initThreadPoolSize = conf.getInt(YarnConfiguration.NM_LOG_AGGREGATION_THREAD_POOL_SIZE, YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE);
    conf.set(YarnConfiguration.NM_LOG_AGGREGATION_THREAD_POOL_SIZE, threadPoolSize);
    DeletionService delSrvc = mock(DeletionService.class);
    LocalDirsHandlerService dirSvc = mock(LocalDirsHandlerService.class);
    when(dirSvc.getLogDirs()).thenThrow(new RuntimeException());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, delSrvc, dirSvc);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ThreadPoolExecutor executorService = (ThreadPoolExecutor) logAggregationService.threadPool;
    assertEquals("The thread pool size should be set to the value of YARN" + ".DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE because the configured " + " thread pool size is " + "invalid.", YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE, executorService.getMaximumPoolSize());
    logAggregationService.stop();
    logAggregationService.close();
    // retore original configuration to aviod side effects
    conf.setInt(YarnConfiguration.NM_LOG_AGGREGATION_THREAD_POOL_SIZE, initThreadPoolSize);
}