Examples with ThreadPoolExecutor java.util.concurrent.ThreadPoolExecutor used on opensource projects

Example 16 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hadoop by apache.

the class FsDatasetAsyncDiskService method addExecutorForVolume.

private void addExecutorForVolume(final FsVolumeImpl volume) {
    ThreadFactory threadFactory = new ThreadFactory() {

        int counter = 0;

        @Override
        public Thread newThread(Runnable r) {
            int thisIndex;
            synchronized (this) {
                thisIndex = counter++;
            }
            Thread t = new Thread(threadGroup, r);
            t.setName("Async disk worker #" + thisIndex + " for volume " + volume);
            return t;
        }
    };
    ThreadPoolExecutor executor = new ThreadPoolExecutor(CORE_THREADS_PER_VOLUME, MAXIMUM_THREADS_PER_VOLUME, THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), threadFactory);
    // This can reduce the number of running threads
    executor.allowCoreThreadTimeOut(true);
    executors.put(volume.getStorageID(), executor);
}

Example 17 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hadoop by apache.

the class TestContainerLauncher method testPoolSize.

@Test(timeout = 10000)
public void testPoolSize() throws InterruptedException {
    ApplicationId appId = ApplicationId.newInstance(12345, 67);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 3);
    JobId jobId = MRBuilderUtils.newJobId(appId, 8);
    TaskId taskId = MRBuilderUtils.newTaskId(jobId, 9, TaskType.MAP);
    AppContext context = mock(AppContext.class);
    CustomContainerLauncher containerLauncher = new CustomContainerLauncher(context);
    containerLauncher.init(new Configuration());
    containerLauncher.start();
    ThreadPoolExecutor threadPool = containerLauncher.getThreadPool();
    // No events yet
    Assert.assertEquals(containerLauncher.initialPoolSize, MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE);
    Assert.assertEquals(0, threadPool.getPoolSize());
    Assert.assertEquals(containerLauncher.initialPoolSize, threadPool.getCorePoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
    for (int i = 0; i < 10; i++) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, i);
        TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, i);
        containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host" + i + ":1234", null, ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH));
    }
    waitForEvents(containerLauncher, 10);
    Assert.assertEquals(10, threadPool.getPoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    // Same set of hosts, so no change
    containerLauncher.finishEventHandling = true;
    int timeOut = 0;
    while (containerLauncher.numEventsProcessed.get() < 10 && timeOut++ < 200) {
        LOG.info("Waiting for number of events processed to become " + 10 + ". It is now " + containerLauncher.numEventsProcessed.get() + ". Timeout is " + timeOut);
        Thread.sleep(1000);
    }
    Assert.assertEquals(10, containerLauncher.numEventsProcessed.get());
    containerLauncher.finishEventHandling = false;
    for (int i = 0; i < 10; i++) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, i + 10);
        TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, i + 10);
        containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host" + i + ":1234", null, ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH));
    }
    waitForEvents(containerLauncher, 20);
    Assert.assertEquals(10, threadPool.getPoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    // Different hosts, there should be an increase in core-thread-pool size to
    // 21(11hosts+10buffer)
    // Core pool size should be 21 but the live pool size should be only 11.
    containerLauncher.expectedCorePoolSize = 11 + containerLauncher.initialPoolSize;
    containerLauncher.finishEventHandling = false;
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 21);
    TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, 21);
    containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host11:1234", null, ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH));
    waitForEvents(containerLauncher, 21);
    Assert.assertEquals(11, threadPool.getPoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    containerLauncher.stop();
    // change configuration MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE
    // and verify initialPoolSize value.
    Configuration conf = new Configuration();
    conf.setInt(MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE, 20);
    containerLauncher = new CustomContainerLauncher(context);
    containerLauncher.init(conf);
    Assert.assertEquals(containerLauncher.initialPoolSize, 20);
}

Example 18 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hadoop by apache.

the class DFSClient method initThreadsNumForHedgedReads.

/**
   * Create hedged reads thread pool, HEDGED_READ_THREAD_POOL, if
   * it does not already exist.
   * @param num Number of threads for hedged reads thread pool.
   * If zero, skip hedged reads thread pool creation.
   */
private synchronized void initThreadsNumForHedgedReads(int num) {
    if (num <= 0 || HEDGED_READ_THREAD_POOL != null)
        return;
    HEDGED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new Daemon.DaemonFactory() {

        private final AtomicInteger threadIndex = new AtomicInteger(0);

        @Override
        public Thread newThread(Runnable r) {
            Thread t = super.newThread(r);
            t.setName("hedgedRead-" + threadIndex.getAndIncrement());
            return t;
        }
    }, new ThreadPoolExecutor.CallerRunsPolicy() {

        @Override
        public void rejectedExecution(Runnable runnable, ThreadPoolExecutor e) {
            LOG.info("Execution rejected, Executing in current thread");
            HEDGED_READ_METRIC.incHedgedReadOpsInCurThread();
            // will run in the current thread
            super.rejectedExecution(runnable, e);
        }
    });
    HEDGED_READ_THREAD_POOL.allowCoreThreadTimeOut(true);
    LOG.debug("Using hedged reads; pool threads={}", num);
}

Example 19 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hadoop by apache.

the class DFSUtilClient method getThreadPoolExecutor.

/**
   * Utility to create a {@link ThreadPoolExecutor}.
   *
   * @param corePoolSize - min threads in the pool, even if idle
   * @param maxPoolSize - max threads in the pool
   * @param keepAliveTimeSecs - max seconds beyond which excess idle threads
   *        will be terminated
   * @param threadNamePrefix - name prefix for the pool threads
   * @param runRejectedExec - when true, rejected tasks from
   *        ThreadPoolExecutor are run in the context of calling thread
   * @return ThreadPoolExecutor
   */
public static ThreadPoolExecutor getThreadPoolExecutor(int corePoolSize, int maxPoolSize, long keepAliveTimeSecs, String threadNamePrefix, boolean runRejectedExec) {
    Preconditions.checkArgument(corePoolSize > 0);
    ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(corePoolSize, maxPoolSize, keepAliveTimeSecs, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new Daemon.DaemonFactory() {

        private final AtomicInteger threadIndex = new AtomicInteger(0);

        @Override
        public Thread newThread(Runnable r) {
            Thread t = super.newThread(r);
            t.setName(threadNamePrefix + threadIndex.getAndIncrement());
            return t;
        }
    });
    if (runRejectedExec) {
        threadPoolExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy() {

            @Override
            public void rejectedExecution(Runnable runnable, ThreadPoolExecutor e) {
                LOG.info(threadNamePrefix + " task is rejected by " + "ThreadPoolExecutor. Executing it in current thread.");
                // will run in the current thread
                super.rejectedExecution(runnable, e);
            }
        });
    }
    return threadPoolExecutor;
}

Example 20 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hadoop by apache.

the class AzureFileSystemThreadPoolExecutor method executeParallel.

/**
   * Execute the file operation parallel using threads. All threads works on a
   * single working set of files stored in input 'contents'. The synchronization
   * between multiple threads is achieved through retrieving atomic index value
   * from the array. Once thread gets the index, it retrieves the file and initiates
   * the file operation. The advantage with this method is that file operations
   * doesn't get serialized due to any thread. Also, the input copy is not changed
   * such that caller can reuse the list for other purposes.
   *
   * This implementation also considers that failure of operation on single file
   * is considered as overall operation failure. All threads bail out their execution
   * as soon as they detect any single thread either got exception or operation is failed.
   *
   * @param contents
   *        List of blobs on which operation to be done.
   * @param threadOperation
   *        The actual operation to be executed by each thread on a file.
   *
   * @param operationStatus
   *        Returns true if the operation is success, false if operation is failed.
   * @throws IOException
   *
   */
boolean executeParallel(FileMetadata[] contents, AzureFileSystemThreadTask threadOperation) throws IOException {
    boolean operationStatus = false;
    boolean threadsEnabled = false;
    int threadCount = this.threadCount;
    ThreadPoolExecutor ioThreadPool = null;
    // Start time for file operation
    long start = Time.monotonicNow();
    // If number of files  are less then reduce threads to file count.
    threadCount = Math.min(contents.length, threadCount);
    if (threadCount > 1) {
        try {
            ioThreadPool = getThreadPool(threadCount);
            threadsEnabled = true;
        } catch (Exception e) {
            // The possibility of this scenario is very remote. Added this code as safety net.
            LOG.warn("Failed to create thread pool with threads {} for operation {} on blob {}." + " Use config {} to set less number of threads. Setting config value to <= 1 will disable threads.", threadCount, operation, key, config);
        }
    } else {
        LOG.warn("Disabling threads for {} operation as thread count {} is <= 1", operation, threadCount);
    }
    if (threadsEnabled) {
        LOG.debug("Using thread pool for {} operation with threads {}", operation, threadCount);
        boolean started = false;
        AzureFileSystemThreadRunnable runnable = new AzureFileSystemThreadRunnable(contents, threadOperation, operation);
        // Don't start any new requests if there is an exception from any one thread.
        for (int i = 0; i < threadCount && runnable.lastException == null && runnable.operationStatus; i++) {
            try {
                ioThreadPool.execute(runnable);
                started = true;
            } catch (RejectedExecutionException ex) {
                // If threads can't be scheduled then report error and move ahead with next thread.
                // Don't fail operation due to this issue.
                LOG.error("Rejected execution of thread for {} operation on blob {}." + " Continuing with existing threads. Use config {} to set less number of threads" + " to avoid this error", operation, key, config);
            }
        }
        // Stop accepting any new execute requests.
        ioThreadPool.shutdown();
        try {
            // Wait for threads to terminate. Keep time out as large value
            ioThreadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
        } catch (InterruptedException intrEx) {
            // If current thread got interrupted then shutdown all threads now.
            ioThreadPool.shutdownNow();
            // Restore the interrupted status
            Thread.currentThread().interrupt();
            LOG.error("Threads got interrupted {} blob operation for {} ", operation, key);
        }
        int threadsNotUsed = threadCount - runnable.threadsUsed.get();
        if (threadsNotUsed > 0) {
            LOG.warn("{} threads not used for {} operation on blob {}", threadsNotUsed, operation, key);
        }
        if (!started) {
            // No threads started. Fall back to serial mode.
            threadsEnabled = false;
            LOG.info("Not able to schedule threads to {} blob {}. Fall back to {} blob serially.", operation, key, operation);
        } else {
            IOException lastException = runnable.lastException;
            // as failure only if file operations are not done on all files.
            if (lastException == null && runnable.operationStatus && runnable.filesProcessed.get() < contents.length) {
                LOG.error("{} failed as operation on subfolders and files failed.", operation);
                lastException = new IOException(operation + " failed as operation on subfolders and files failed.");
            }
            if (lastException != null) {
                // Raise the same exception.
                throw lastException;
            }
            operationStatus = runnable.operationStatus;
        }
    }
    if (!threadsEnabled) {
        // No threads. Serialize the operation. Clear any last exceptions.
        LOG.debug("Serializing the {} operation", operation);
        for (int i = 0; i < contents.length; i++) {
            if (!threadOperation.execute(contents[i])) {
                LOG.warn("Failed to {} file {}", operation, contents[i]);
                return false;
            }
        }
        // Operation is success
        operationStatus = true;
    }
    // Find the duration of time taken for file operation
    long end = Time.monotonicNow();
    LOG.info("Time taken for {} operation is: {} ms with threads: {}", operation, (end - start), threadCount);
    return operationStatus;
}