Examples with FutureTask java.util.concurrent.FutureTask used on opensource projects

Example 66 with FutureTask

use of java.util.concurrent.FutureTask in project hive by apache.

the class CLIServiceTest method testExecuteStatementParallel.

@Test
public void testExecuteStatementParallel() throws Exception {
    Map<String, String> confOverlay = new HashMap<String, String>();
    String tableName = "TEST_EXEC_PARALLEL";
    String columnDefinitions = "(ID STRING)";
    // Open a session and set up the test data
    SessionHandle sessionHandle = setupTestData(tableName, columnDefinitions, confOverlay);
    assertNotNull(sessionHandle);
    long longPollingTimeout = HiveConf.getTimeVar(new HiveConf(), HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT, TimeUnit.MILLISECONDS);
    confOverlay.put(HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT.varname, longPollingTimeout + "ms");
    int THREAD_COUNT = 10, QUERY_COUNT = 10;
    // TODO: refactor this into an utility, LLAP tests use this pattern a lot
    ExecutorService executor = Executors.newFixedThreadPool(THREAD_COUNT);
    CountDownLatch cdlIn = new CountDownLatch(THREAD_COUNT), cdlOut = new CountDownLatch(1);
    @SuppressWarnings("unchecked") Callable<Void>[] cs = new Callable[3];
    // Create callables with different queries.
    String query = "SELECT ID + %1$d FROM " + tableName;
    cs[0] = createQueryCallable(query, confOverlay, longPollingTimeout, QUERY_COUNT, OperationState.FINISHED, true, cdlIn, cdlOut);
    query = "SELECT t1.ID, SUM(t2.ID) + %1$d FROM  " + tableName + " t1 CROSS JOIN " + tableName + " t2 GROUP BY t1.ID HAVING t1.ID > 1";
    cs[1] = createQueryCallable(query, confOverlay, longPollingTimeout, QUERY_COUNT, OperationState.FINISHED, true, cdlIn, cdlOut);
    query = "SELECT b.a FROM (SELECT (t1.ID + %1$d) as a , t2.* FROM  " + tableName + " t1 INNER JOIN " + tableName + " t2 ON t1.ID = t2.ID WHERE t2.ID > 2) b";
    cs[2] = createQueryCallable(query, confOverlay, longPollingTimeout, QUERY_COUNT, OperationState.FINISHED, true, cdlIn, cdlOut);
    @SuppressWarnings("unchecked") FutureTask<Void>[] tasks = new FutureTask[THREAD_COUNT];
    for (int i = 0; i < THREAD_COUNT; ++i) {
        tasks[i] = new FutureTask<Void>(cs[i % cs.length]);
        executor.execute(tasks[i]);
    }
    try {
        // Wait for all threads to be ready.
        cdlIn.await();
        // Release them at the same time.
        cdlOut.countDown();
        for (int i = 0; i < THREAD_COUNT; ++i) {
            tasks[i].get();
        }
    } catch (Throwable t) {
        throw new RuntimeException(t);
    }
    // Cleanup
    client.executeStatement(sessionHandle, "DROP TABLE " + tableName, confOverlay);
    client.closeSession(sessionHandle);
}

Example 67 with FutureTask

use of java.util.concurrent.FutureTask in project buck by facebook.

the class DefaultProcessExecutor method execute.

/**
   * Executes the specified already-launched process.
   * <p>
   * If {@code options} contains {@link Option#PRINT_STD_OUT}, then the stdout of the process will
   * be written directly to the stdout passed to the constructor of this executor. Otherwise,
   * the stdout of the process will be made available via {@link Result#getStdout()}.
   * <p>
   * If {@code options} contains {@link Option#PRINT_STD_ERR}, then the stderr of the process will
   * be written directly to the stderr passed to the constructor of this executor. Otherwise,
   * the stderr of the process will be made available via {@link Result#getStderr()}.
   * @param timeOutHandler If present, this method will be called before the process is killed.
   */
public Result execute(LaunchedProcess launchedProcess, Set<Option> options, Optional<String> stdin, Optional<Long> timeOutMs, Optional<Consumer<Process>> timeOutHandler) throws InterruptedException {
    Preconditions.checkState(launchedProcess instanceof LaunchedProcessImpl);
    Process process = ((LaunchedProcessImpl) launchedProcess).process;
    // Read stdout/stderr asynchronously while running a Process.
    // See http://stackoverflow.com/questions/882772/capturing-stdout-when-calling-runtime-exec
    boolean shouldPrintStdOut = options.contains(Option.PRINT_STD_OUT);
    boolean expectingStdOut = options.contains(Option.EXPECTING_STD_OUT);
    PrintStream stdOutToWriteTo = shouldPrintStdOut ? stdOutStream : new CapturingPrintStream();
    InputStreamConsumer stdOut = new InputStreamConsumer(process.getInputStream(), InputStreamConsumer.createAnsiHighlightingHandler(/* flagOutputWrittenToStream */
    !shouldPrintStdOut && !expectingStdOut, stdOutToWriteTo, ansi));
    boolean shouldPrintStdErr = options.contains(Option.PRINT_STD_ERR);
    boolean expectingStdErr = options.contains(Option.EXPECTING_STD_ERR);
    PrintStream stdErrToWriteTo = shouldPrintStdErr ? stdErrStream : new CapturingPrintStream();
    InputStreamConsumer stdErr = new InputStreamConsumer(process.getErrorStream(), InputStreamConsumer.createAnsiHighlightingHandler(/* flagOutputWrittenToStream */
    !shouldPrintStdErr && !expectingStdErr, stdErrToWriteTo, ansi));
    // Consume the streams so they do not deadlock.
    FutureTask<Void> stdOutTerminationFuture = new FutureTask<>(stdOut);
    Thread stdOutConsumer = Threads.namedThread("ProcessExecutor (stdOut)", stdOutTerminationFuture);
    stdOutConsumer.start();
    FutureTask<Void> stdErrTerminationFuture = new FutureTask<>(stdErr);
    Thread stdErrConsumer = Threads.namedThread("ProcessExecutor (stdErr)", stdErrTerminationFuture);
    stdErrConsumer.start();
    boolean timedOut = false;
    // Block until the Process completes.
    try {
        // deadlocks, as the stdout/stderr consumers are running in separate threads.
        if (stdin.isPresent()) {
            try (OutputStreamWriter stdinWriter = new OutputStreamWriter(process.getOutputStream())) {
                stdinWriter.write(stdin.get());
            }
        }
        // the regular `waitFor` method.
        if (timeOutMs.isPresent()) {
            timedOut = waitForTimeoutInternal(process, timeOutMs.get(), timeOutHandler);
            if (!processHelper.hasProcessFinished(process)) {
                process.destroy();
            }
        } else {
            process.waitFor();
        }
        stdOutTerminationFuture.get();
        stdErrTerminationFuture.get();
    } catch (ExecutionException | IOException e) {
        // situation.
        return new Result(1);
    } finally {
        process.destroy();
        process.waitFor();
    }
    Optional<String> stdoutText = getDataIfNotPrinted(stdOutToWriteTo, shouldPrintStdOut);
    Optional<String> stderrText = getDataIfNotPrinted(stdErrToWriteTo, shouldPrintStdErr);
    // Report the exit code of the Process.
    int exitCode = process.exitValue();
    // printed.
    if (exitCode != 0 && !options.contains(Option.IS_SILENT)) {
        if (!shouldPrintStdOut && !stdoutText.get().isEmpty()) {
            LOG.verbose("Writing captured stdout text to stream: [%s]", stdoutText.get());
            stdOutStream.print(stdoutText.get());
        }
        if (!shouldPrintStdErr && !stderrText.get().isEmpty()) {
            LOG.verbose("Writing captured stderr text to stream: [%s]", stderrText.get());
            stdErrStream.print(stderrText.get());
        }
    }
    return new Result(exitCode, timedOut, stdoutText, stderrText);
}

Example 68 with FutureTask

use of java.util.concurrent.FutureTask in project buck by facebook.

the class ProjectBuildFileParser method init.

/**
   * Initialize the parser, starting buck.py.
   */
private void init() throws IOException {
    projectBuildFileParseEventStarted = new ProjectBuildFileParseEvents.Started();
    buckEventBus.post(projectBuildFileParseEventStarted);
    try (SimplePerfEvent.Scope scope = SimplePerfEvent.scope(buckEventBus, PerfEventId.of("ParserInit"))) {
        ImmutableMap.Builder<String, String> pythonEnvironmentBuilder = ImmutableMap.builder();
        // Strip out PYTHONPATH. buck.py manually sets this to include only nailgun. We don't want
        // to inject nailgun into the parser's PYTHONPATH, so strip that value out.
        // If we wanted to pass on some environmental PYTHONPATH, we would have to do some actual
        // merging of this and the BuckConfig's python module search path.
        pythonEnvironmentBuilder.putAll(Maps.filterKeys(environment, k -> !PYTHONPATH_ENV_VAR_NAME.equals(k)));
        if (options.getPythonModuleSearchPath().isPresent()) {
            pythonEnvironmentBuilder.put(PYTHONPATH_ENV_VAR_NAME, options.getPythonModuleSearchPath().get());
        }
        ImmutableMap<String, String> pythonEnvironment = pythonEnvironmentBuilder.build();
        ProcessExecutorParams params = ProcessExecutorParams.builder().setCommand(buildArgs()).setEnvironment(pythonEnvironment).build();
        LOG.debug("Starting buck.py command: %s environment: %s", params.getCommand(), params.getEnvironment());
        buckPyProcess = processExecutor.launchProcess(params);
        LOG.debug("Started process %s successfully", buckPyProcess);
        OutputStream stdin = buckPyProcess.getOutputStream();
        InputStream stderr = buckPyProcess.getErrorStream();
        InputStreamConsumer stderrConsumer = new InputStreamConsumer(stderr, (InputStreamConsumer.Handler) line -> buckEventBus.post(ConsoleEvent.warning("Warning raised by BUCK file parser: %s", line)));
        stderrConsumerTerminationFuture = new FutureTask<>(stderrConsumer);
        stderrConsumerThread = Threads.namedThread(ProjectBuildFileParser.class.getSimpleName(), stderrConsumerTerminationFuture);
        stderrConsumerThread.start();
        buckPyStdinWriter = new BufferedOutputStream(stdin);
    }
}

Example 69 with FutureTask

use of java.util.concurrent.FutureTask in project hibernate-orm by hibernate.

the class ServiceRegistryTest method execute.

private FutureTask<SlowInitializationService>[] execute() throws InterruptedException, ExecutionException {
    FutureTask<SlowInitializationService>[] results = new FutureTask[NUMBER_OF_THREADS];
    ExecutorService executor = Executors.newFixedThreadPool(NUMBER_OF_THREADS);
    for (int i = 0; i < NUMBER_OF_THREADS; i++) {
        results[i] = new FutureTask<>(new ServiceCallable(registry));
        executor.execute(results[i]);
    }
    return results;
}

Example 70 with FutureTask

use of java.util.concurrent.FutureTask in project flink by apache.

the class FileCache method createTmpFile.

// ------------------------------------------------------------------------
/**
	 * If the file doesn't exists locally, it will copy the file to the temp directory.
	 *
	 * @param name  The name under which the file is registered.
	 * @param entry The cache entry descriptor (path, executable flag)
	 * @param jobID The ID of the job for which the file is copied.
	 * @return The handle to the task that copies the file.
	 */
public Future<Path> createTmpFile(String name, DistributedCacheEntry entry, JobID jobID) {
    synchronized (lock) {
        Map<String, Tuple4<Integer, File, Path, Future<Path>>> jobEntries = entries.get(jobID);
        if (jobEntries == null) {
            jobEntries = new HashMap<String, Tuple4<Integer, File, Path, Future<Path>>>();
            entries.put(jobID, jobEntries);
        }
        // tuple is (ref-count, parent-temp-dir, cached-file-path, copy-process)
        Tuple4<Integer, File, Path, Future<Path>> fileEntry = jobEntries.get(name);
        if (fileEntry != null) {
            // file is already in the cache. return a future that
            // immediately returns the file
            fileEntry.f0 = fileEntry.f0 + 1;
            // return the future. may be that the copy is still in progress
            return fileEntry.f3;
        } else {
            // need to copy the file
            // create the target path
            File tempDirToUse = new File(storageDirectories[nextDirectory++], jobID.toString());
            if (nextDirectory >= storageDirectories.length) {
                nextDirectory = 0;
            }
            String sourceFile = entry.filePath;
            int posOfSep = sourceFile.lastIndexOf("/");
            if (posOfSep > 0) {
                sourceFile = sourceFile.substring(posOfSep + 1);
            }
            Path target = new Path(tempDirToUse.getAbsolutePath() + "/" + sourceFile);
            // kick off the copying
            CopyProcess cp = new CopyProcess(entry, target);
            FutureTask<Path> copyTask = new FutureTask<Path>(cp);
            executorService.submit(copyTask);
            // store our entry
            jobEntries.put(name, new Tuple4<Integer, File, Path, Future<Path>>(1, tempDirToUse, target, copyTask));
            return copyTask;
        }
    }
}