Examples with ExecutorCompletionService java.util.concurrent.ExecutorCompletionService used on opensource projects

Example 51 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project tika by apache.

the class TestMetadata method testMultithreadedDates.

@Test
public void testMultithreadedDates() throws Exception {
    int numThreads = 10;
    ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
    ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<Integer>(executorService);
    for (int i = 0; i < numThreads; i++) {
        executorCompletionService.submit(new MetadataDateAdder());
    }
    int finished = 0;
    while (finished < numThreads) {
        Future<Integer> future = executorCompletionService.take();
        if (future != null && future.isDone()) {
            Integer retVal = future.get();
            finished++;
        }
    }
}

Example 52 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project tika by apache.

the class HtmlParserTest method testDetector.

private void testDetector(EncodingDetector detector) throws Exception {
    Path testDocs = Paths.get(this.getClass().getResource("/test-documents").toURI());
    List<Path> tmp = new ArrayList<>();
    Map<Path, String> encodings = new ConcurrentHashMap<>();
    File[] testDocArray = testDocs.toFile().listFiles();
    assertNotNull("no test docs??", testDocArray);
    for (File file : testDocArray) {
        if (file.getName().endsWith(".txt") || file.getName().endsWith(".html")) {
            String encoding = getEncoding(detector, file.toPath());
            tmp.add(file.toPath());
            encodings.put(file.toPath(), encoding);
        }
    }
    ArrayBlockingQueue<Path> paths = new ArrayBlockingQueue<>(tmp.size());
    paths.addAll(tmp);
    int numThreads = paths.size() + 1;
    ExecutorService ex = Executors.newFixedThreadPool(numThreads);
    CompletionService<String> completionService = new ExecutorCompletionService<>(ex);
    for (int i = 0; i < numThreads; i++) {
        completionService.submit(new EncodingDetectorRunner(paths, encodings, detector));
    }
    int completed = 0;
    while (completed < numThreads) {
        Future<String> future = completionService.take();
        if (future.isDone() && //was thrown during call
        EncodingDetectorRunner.DONE.equals(future.get())) {
            completed++;
        }
    }
}

Example 53 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project tika by apache.

the class AbstractBufferTest method runTest.

@Test(timeout = 30000)
public void runTest() throws InterruptedException, ExecutionException {
    List<String> keys = new ArrayList<>();
    Collections.addAll(keys, new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k" });
    int numGets = 100;
    int numTesters = 20;
    AbstractDBBuffer b = new TestBuffer();
    ExecutorService ex = Executors.newFixedThreadPool(numTesters);
    CompletionService<MyTestResult> completionService = new ExecutorCompletionService<>(ex);
    for (int i = 0; i < numTesters; i++) {
        completionService.submit(new Tester(keys, b, numGets));
    }
    int results = 0;
    Map<String, Integer> combined = new HashMap<>();
    while (results < numTesters) {
        Future<MyTestResult> futureResult = completionService.poll(1, TimeUnit.SECONDS);
        if (futureResult != null) {
            results++;
            assertEquals(keys.size(), futureResult.get().getMap().keySet().size());
            for (Map.Entry<String, Integer> e : futureResult.get().getMap().entrySet()) {
                if (!combined.containsKey(e.getKey())) {
                    combined.put(e.getKey(), e.getValue());
                } else {
                    assertEquals(combined.get(e.getKey()), e.getValue());
                }
            }
        }
    }
    assertEquals(keys.size(), b.getNumWrites());
}

Example 54 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project tika by apache.

the class StrawManTikaAppDriver method main.

public static void main(String[] args) {
    long start = new Date().getTime();
    if (args.length < 6) {
        System.err.println(StrawManTikaAppDriver.usage());
    }
    Path inputDir = Paths.get(args[0]);
    Path outputDir = Paths.get(args[1]);
    int totalThreads = Integer.parseInt(args[2]);
    List<String> commandLine = new ArrayList<>();
    commandLine.addAll(Arrays.asList(args).subList(3, args.length));
    totalThreads = (totalThreads < 1) ? 1 : totalThreads;
    ExecutorService ex = Executors.newFixedThreadPool(totalThreads);
    ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(ex);
    for (int i = 0; i < totalThreads; i++) {
        StrawManTikaAppDriver driver = new StrawManTikaAppDriver(inputDir, outputDir, totalThreads, commandLine.toArray(new String[commandLine.size()]));
        completionService.submit(driver);
    }
    int totalFilesProcessed = 0;
    for (int i = 0; i < totalThreads; i++) {
        try {
            Future<Integer> future = completionService.take();
            if (future != null) {
                totalFilesProcessed += future.get();
            }
        } catch (InterruptedException | ExecutionException e) {
            LOG.error(e.getMessage(), e);
        }
    }
    double elapsedSeconds = (double) (new Date().getTime() - start) / (double) 1000;
    LOG.info("Processed {} in {} seconds", totalFilesProcessed, elapsedSeconds);
}

Example 55 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project cdap by caskdata.

the class WorkflowDriver method executeFork.

private void executeFork(final ApplicationSpecification appSpec, WorkflowForkNode fork, final InstantiatorFactory instantiator, final ClassLoader classLoader, final WorkflowToken token) throws Exception {
    CountDownLatch executorTerminateLatch = new CountDownLatch(1);
    ExecutorService executorService = createExecutor(fork.getBranches().size(), executorTerminateLatch, "fork-" + fork.getNodeId() + "-%d");
    CompletionService<Map.Entry<String, WorkflowToken>> completionService = new ExecutorCompletionService<>(executorService);
    try {
        for (final List<WorkflowNode> branch : fork.getBranches()) {
            completionService.submit(new Callable<Map.Entry<String, WorkflowToken>>() {

                @Override
                public Map.Entry<String, WorkflowToken> call() throws Exception {
                    WorkflowToken copiedToken = ((BasicWorkflowToken) token).deepCopy();
                    executeAll(branch.iterator(), appSpec, instantiator, classLoader, copiedToken);
                    return Maps.immutableEntry(branch.toString(), copiedToken);
                }
            });
        }
        for (int i = 0; i < fork.getBranches().size(); i++) {
            try {
                Future<Map.Entry<String, WorkflowToken>> forkBranchResult = completionService.take();
                Map.Entry<String, WorkflowToken> retValue = forkBranchResult.get();
                String branchInfo = retValue.getKey();
                WorkflowToken branchToken = retValue.getValue();
                ((BasicWorkflowToken) token).mergeToken(branchToken);
                LOG.trace("Execution of branch {} for fork {} completed.", branchInfo, fork);
            } catch (InterruptedException e) {
                // Due to workflow abortion, so just break the loop
                break;
            } catch (ExecutionException e) {
                // Unwrap the cause
                Throwables.propagateIfPossible(e.getCause(), Exception.class);
                throw Throwables.propagate(e.getCause());
            }
        }
    } finally {
        // Update the WorkflowToken after the execution of the FORK node completes.
        runtimeStore.updateWorkflowToken(workflowRunId, token);
        executorService.shutdownNow();
        // Wait for the executor termination
        executorTerminateLatch.await();
    }
}