Examples with ListeningExecutorService com.google.common.util.concurrent.ListeningExecutorService used on opensource projects

Example 41 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project hmftools by hartwigmedical.

the class FastqStats method processFastqs.

/**
 * Counts yield and q30 of fastqs in the fastqsPerSample multimap, using 1 thread per file.
 * The yield and q30 of the Undetermined sample will count towards the total yield and q30 of the flowcell.
 *
 * @param fastqsPerSample multimap of sampleName and fastqs to process
 * @param threadCount     number of maximum threads
 * @return FastqTracker with yield and q30 stats for the fastqs processed.
 */
@NotNull
static FastqTracker processFastqs(@NotNull final Multimap<String, File> fastqsPerSample, final int threadCount) throws InterruptedException {
    LOGGER.info("Using " + threadCount + " threads. Processing " + fastqsPerSample.size() + " fastQ files.");
    final FastqTrackerWrapper tracker = new FastqTrackerWrapper();
    final ListeningExecutorService threadPool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threadCount));
    for (final String sampleName : fastqsPerSample.keySet()) {
        final Collection<File> fastqs = fastqsPerSample.get(sampleName);
        for (final File fastq : fastqs) {
            final String laneName = getLaneName(fastq);
            final ListenableFuture<FastqData> futureResult = threadPool.submit(() -> processFile(fastq));
            addCallback(futureResult, (data) -> tracker.addDataFromSampleFile(sampleName, laneName, data), (error) -> LOGGER.error("Failed to process file: " + fastq.getName(), error));
        }
    }
    threadPool.shutdown();
    threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
    return tracker.tracker();
}

Example 42 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project Singularity by HubSpot.

the class SingularityExecutorMonitor method getShellCommandExecutorServiceForTask.

public ListeningExecutorService getShellCommandExecutorServiceForTask(String taskId) {
    if (!taskToShellCommandPool.containsKey(taskId)) {
        ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat(taskId + "-shellCommandPool-%d").build()));
        taskToShellCommandPool.put(taskId, executorService);
    }
    return taskToShellCommandPool.get(taskId);
}

Example 43 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project xtext-eclipse by eclipse.

the class ParallelBuilderParticipant method doBuild.

@Override
protected void doBuild(List<Delta> deltas, Map<String, OutputConfiguration> outputConfigurations, Map<OutputConfiguration, Iterable<IMarker>> generatorMarkers, IBuildContext context, EclipseResourceFileSystemAccess2 access, IProgressMonitor progressMonitor) throws CoreException {
    BlockingQueue<FileSystemAccessRequest> requestQueue = newBlockingQueue(QUEUE_CAPACITY);
    // This queue is only used from the current thread
    // thus there is no need for a blocking queue. The add operation should also not block the
    // builder thread
    Queue<ParallelBuildContext> afterGenerateQueue = new ArrayDeque<ParallelBuildContext>(QUEUE_CAPACITY);
    FileSystemAccessQueue fileSystemAccessQueue = new FileSystemAccessQueue(requestQueue, progressMonitor);
    Tripwire tripwire = new Tripwire();
    EList<Adapter> adapters = context.getResourceSet().eAdapters();
    EObservableAdapterList observableAdapters = (EObservableAdapterList) adapters;
    adapters.add(fileSystemAccessQueue);
    observableAdapters.addListener(tripwire);
    try {
        SubMonitor subMonitor = SubMonitor.convert(progressMonitor, 1);
        subMonitor.subTask("Compiling...");
        access.setMonitor(subMonitor.split(1));
        List<ListenableFuture<?>> tasks = Lists.newArrayList();
        ListeningExecutorService executor = executors.getExecutor();
        for (IResourceDescription.Delta delta : deltas) {
            if (getResourceServiceProvider().canHandle(delta.getUri())) {
                try {
                    ParallelBuildContext parallelBuildContext = new ParallelBuildContext(delta, context, outputConfigurations, generatorMarkers, fileSystemAccessQueue, afterGenerateQueue, access, progressMonitor);
                    Runnable runnable = createRunnable(parallelBuildContext);
                    tasks.add(executor.submit(runnable));
                } catch (Exception e) {
                    addMarkerAndLogError(delta.getUri(), e);
                }
            }
        }
        ListenableFuture<List<Object>> generatorResult = Futures.successfulAsList(tasks);
        List<SimpleEntry<URI, Throwable>> exceptions = Lists.newArrayList();
        boolean interrupted = false;
        try {
            /*
				 * it is important to check generatorResult and requestQueue second
				 * as it can happen that if you use !requestQueue.isEmpty() || !generatorResult.isDone()
				 * that the generatorResult.isDone() becomes true after requestQueue.isEmpty() was checked
				 * and thus requestQueue.isEmpty() changes back to false
				 * but we stop the while loop anyway and thus miss generated files.
				 */
            while (!generatorResult.isDone() || !requestQueue.isEmpty()) {
                if (subMonitor.isCanceled()) {
                    cancelProcessing(requestQueue, afterGenerateQueue, generatorResult);
                    throw new OperationCanceledException();
                }
                FileSystemAccessRequest request = null;
                try {
                    request = requestQueue.poll(QUEUE_POLL_TIMEOUT, TimeUnit.MILLISECONDS);
                } catch (InterruptedException e) {
                    interrupted = true;
                }
                if (request != null) {
                    try {
                        request.run();
                    } catch (OperationCanceledException e) {
                        cancelProcessing(requestQueue, afterGenerateQueue, generatorResult);
                        throw e;
                    } catch (Exception e) {
                        Throwable cause = e;
                        if (cause instanceof CoreException) {
                            cause = cause.getCause();
                        }
                        exceptions.add(new SimpleEntry<URI, Throwable>(request.getUri(), cause));
                    }
                }
            }
        } finally {
            if (interrupted) {
                Thread.currentThread().interrupt();
            }
            for (SimpleEntry<URI, Throwable> exception : exceptions) {
                addMarkerAndLogError(exception.getKey(), exception.getValue());
            }
        }
    } finally {
        observableAdapters.removeListener(tripwire);
        adapters.remove(fileSystemAccessQueue);
    }
}

Example 44 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project jackrabbit-oak by apache.

the class FileCacheTest method getInvalidateConcurrent.

/**
 * Retrieve and invalidate concurrently.
 * @throws Exception
 */
@Test
public void getInvalidateConcurrent() throws Exception {
    LOG.info("Started getInvalidateConcurrent");
    // Create load
    for (int i = 0; i < 15; i++) {
        if (i != 4) {
            File f = createFile(i, loader, cache, folder);
            assertCache(i, cache, f);
        }
    }
    LOG.info("Finished creating load");
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
    closer.register(new ExecutorCloser(executorService, 5, TimeUnit.MILLISECONDS));
    CountDownLatch thread1Start = new CountDownLatch(1);
    SettableFuture<File> future1 = retrieveThread(executorService, ID_PREFIX + 10, cache, thread1Start);
    thread1Start.countDown();
    File f = createFile(4, loader, cache, folder);
    CountDownLatch thread2Start = new CountDownLatch(1);
    SettableFuture<File> future2 = retrieveThread(executorService, ID_PREFIX + 4, cache, thread2Start);
    thread2Start.countDown();
    File f10 = future1.get();
    File f4 = future2.get();
    LOG.info("Async tasks finished");
    if (f10.exists()) {
        assertCacheIfPresent(10, cache, f10);
    }
    if (f4.exists()) {
        assertCacheIfPresent(4, cache, f4);
    }
    LOG.info("Finished getInvalidateConcurrent");
}

Example 45 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project jackrabbit-oak by apache.

the class FileCacheTest method retrievePutConcurrent.

/**
 * Retrieve and put different files concurrently.
 * @throws Exception
 */
@Test
public void retrievePutConcurrent() throws Exception {
    LOG.info("Started retrievePutConcurrent");
    // Create load
    final File f = createFile(0, loader, cache, folder);
    File f2 = copyToFile(randomStream(1, 4 * 1024), folder.newFile());
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
    closer.register(new ExecutorCloser(executorService, 5, TimeUnit.MILLISECONDS));
    CountDownLatch thread1Start = new CountDownLatch(1);
    SettableFuture<File> future1 = retrieveThread(executorService, ID_PREFIX + 0, cache, thread1Start);
    CountDownLatch thread2Start = new CountDownLatch(1);
    SettableFuture<Boolean> future2 = putThread(executorService, 1, f2, cache, thread2Start);
    thread1Start.countDown();
    thread2Start.countDown();
    future1.get();
    future2.get();
    LOG.info("Async tasks finished");
    assertCacheIfPresent(0, cache, f);
    assertCacheIfPresent(1, cache, copyToFile(randomStream(1, 4 * 1024), folder.newFile()));
    assertCacheStats(cache, 2, 8 * 1024, 1, 1);
    LOG.info("Finished retrievePutConcurrent");
}