Examples with ExecutorCompletionService java.util.concurrent.ExecutorCompletionService used on opensource projects

Example 31 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project hadoop by apache.

the class TestBatchIbr method createExecutor.

static ExecutorService createExecutor() throws Exception {
    final ExecutorService executor = Executors.newFixedThreadPool(NUM_THREADS);
    final ExecutorCompletionService<Path> completion = new ExecutorCompletionService<>(executor);
    // initialize all threads and buffers
    for (int i = 0; i < NUM_THREADS; i++) {
        completion.submit(new Callable<Path>() {

            @Override
            public Path call() throws Exception {
                IO_BUF.get();
                VERIFY_BUF.get();
                return null;
            }
        });
    }
    for (int i = 0; i < NUM_THREADS; i++) {
        completion.take().get();
    }
    return executor;
}

Example 32 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project camel by apache.

the class JmsRouteRequestReplyTest method runRequestReplyThreaded.

protected void runRequestReplyThreaded(String fromUri) throws Exception {
    // start template
    template.start();
    ExecutorService executor = context.getExecutorServiceManager().newFixedThreadPool(this, "Task", maxTasks);
    CompletionService<Task> completionService = new ExecutorCompletionService<Task>(executor);
    final AtomicInteger counter = new AtomicInteger(-1);
    for (int i = 0; i < maxTasks; i++) {
        Task task = new Task(counter, fromUri);
        completionService.submit(task);
    }
    for (int i = 0; i < maxTasks; i++) {
        Future<Task> future = completionService.take();
        Task task = future.get(60, TimeUnit.SECONDS);
        assertNotNull("Should complete the task", task);
        task.assertSuccess();
    }
    context.getExecutorServiceManager().shutdownNow(executor);
}

Example 33 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project graphhopper by graphhopper.

the class LMAlgoFactoryDecorator method loadOrDoWork.

/**
 * This method calculates the landmark data for all weightings (optionally in parallel) or if already existent loads it.
 *
 * @return true if the preparation data for at least one weighting was calculated.
 * @see com.graphhopper.routing.ch.CHAlgoFactoryDecorator#prepare(StorableProperties) for a very similar method
 */
public boolean loadOrDoWork(final StorableProperties properties) {
    ExecutorCompletionService completionService = new ExecutorCompletionService<>(threadPool);
    int counter = 0;
    final AtomicBoolean prepared = new AtomicBoolean(false);
    for (final PrepareLandmarks plm : preparations) {
        counter++;
        final int tmpCounter = counter;
        final String name = AbstractWeighting.weightingToFileName(plm.getWeighting());
        completionService.submit(new Runnable() {

            @Override
            public void run() {
                if (plm.loadExisting())
                    return;
                LOGGER.info(tmpCounter + "/" + getPreparations().size() + " calling LM prepare.doWork for " + plm.getWeighting() + " ... (" + getMemInfo() + ")");
                prepared.set(true);
                Thread.currentThread().setName(name);
                plm.doWork();
                properties.put(Landmark.PREPARE + "date." + name, createFormatter().format(new Date()));
            }
        }, name);
    }
    threadPool.shutdown();
    try {
        for (int i = 0; i < preparations.size(); i++) {
            completionService.take().get();
        }
    } catch (Exception e) {
        threadPool.shutdownNow();
        throw new RuntimeException(e);
    }
    return prepared.get();
}

Example 34 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project hbase by apache.

the class SnapshotManifestV2 method loadRegionManifests.

static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf, final Executor executor, final FileSystem fs, final Path snapshotDir, final SnapshotDescription desc, final int manifestSizeLimit) throws IOException {
    FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {

        @Override
        public boolean accept(Path path) {
            return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
        }
    });
    if (manifestFiles == null || manifestFiles.length == 0)
        return null;
    final ExecutorCompletionService<SnapshotRegionManifest> completionService = new ExecutorCompletionService<>(executor);
    for (final FileStatus st : manifestFiles) {
        completionService.submit(new Callable<SnapshotRegionManifest>() {

            @Override
            public SnapshotRegionManifest call() throws IOException {
                FSDataInputStream stream = fs.open(st.getPath());
                CodedInputStream cin = CodedInputStream.newInstance(stream);
                cin.setSizeLimit(manifestSizeLimit);
                try {
                    return SnapshotRegionManifest.parseFrom(cin);
                } finally {
                    stream.close();
                }
            }
        });
    }
    ArrayList<SnapshotRegionManifest> regionsManifest = new ArrayList<>(manifestFiles.length);
    try {
        for (int i = 0; i < manifestFiles.length; ++i) {
            regionsManifest.add(completionService.take().get());
        }
    } catch (InterruptedException e) {
        throw new InterruptedIOException(e.getMessage());
    } catch (ExecutionException e) {
        Throwable t = e.getCause();
        if (t instanceof InvalidProtocolBufferException) {
            throw (InvalidProtocolBufferException) t;
        } else {
            IOException ex = new IOException("ExecutionException");
            ex.initCause(e.getCause());
            throw ex;
        }
    }
    return regionsManifest;
}

Example 35 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project hbase by apache.

the class SnapshotReferenceUtil method concurrentVisitReferencedFiles.

public static void concurrentVisitReferencedFiles(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest, final ExecutorService exec, final StoreFileVisitor visitor) throws IOException {
    final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
    final Path snapshotDir = manifest.getSnapshotDir();
    List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
    if (regionManifests == null || regionManifests.isEmpty()) {
        LOG.debug("No manifest files present: " + snapshotDir);
        return;
    }
    final ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<>(exec);
    for (final SnapshotRegionManifest regionManifest : regionManifests) {
        completionService.submit(new Callable<Void>() {

            @Override
            public Void call() throws IOException {
                visitRegionStoreFiles(regionManifest, visitor);
                return null;
            }
        });
    }
    try {
        for (int i = 0; i < regionManifests.size(); ++i) {
            completionService.take().get();
        }
    } catch (InterruptedException e) {
        throw new InterruptedIOException(e.getMessage());
    } catch (ExecutionException e) {
        if (e.getCause() instanceof CorruptedSnapshotException) {
            throw new CorruptedSnapshotException(e.getCause().getMessage(), ProtobufUtil.createSnapshotDesc(snapshotDesc));
        } else {
            IOException ex = new IOException();
            ex.initCause(e.getCause());
            throw ex;
        }
    }
}