Examples with FutureTask java.util.concurrent.FutureTask used on opensource projects

Example 61 with FutureTask

use of java.util.concurrent.FutureTask in project bazel by bazelbuild.

the class SkyframeActionExecutor method executeAction.

/**
   * Executes the provided action on the current thread. Returns the ActionExecutionValue with the
   * result, either computed here or already computed on another thread.
   *
   * <p>For use from {@link ArtifactFunction} only.
   */
ActionExecutionValue executeAction(Action action, ActionMetadataHandler metadataHandler, long actionStartTime, ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException {
    Exception exception = badActionMap.get(action);
    if (exception != null) {
        // If action had a conflict with some other action in the graph, report it now.
        reportError(exception.getMessage(), exception, action, null);
    }
    Artifact primaryOutput = action.getPrimaryOutput();
    FutureTask<ActionExecutionValue> actionTask = new FutureTask<>(new ActionRunner(action, metadataHandler, actionStartTime, actionExecutionContext));
    // Check to see if another action is already executing/has executed this value.
    Pair<Action, FutureTask<ActionExecutionValue>> oldAction = buildActionMap.putIfAbsent(primaryOutput, Pair.of(action, actionTask));
    if (oldAction == null) {
        actionTask.run();
    } else {
        Preconditions.checkState(action != oldAction.first, action);
        Preconditions.checkState(Actions.canBeShared(oldAction.first, action), "Actions cannot be shared: %s %s", oldAction.first, action);
        // Wait for other action to finish, so any actions that depend on its outputs can execute.
        actionTask = oldAction.second;
    }
    try {
        return actionTask.get();
    } catch (ExecutionException e) {
        Throwables.propagateIfPossible(e.getCause(), ActionExecutionException.class, InterruptedException.class);
        throw new IllegalStateException(e);
    } finally {
        String message = action.getProgressMessage();
        if (message != null) {
            // completion messages are enabled (--show_task_finish).
            if (completionReceiver != null) {
                completionReceiver.actionCompleted(action);
            }
            reporter.finishTask(null, prependExecPhaseStats(message));
        }
    }
}

Example 62 with FutureTask

use of java.util.concurrent.FutureTask in project hive by apache.

the class TezSessionState method openInternal.

protected void openInternal(final HiveConf conf, Collection<String> additionalFiles, boolean isAsync, LogHelper console, Path scratchDir) throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException {
    this.conf = conf;
    // TODO Why is the queue name set again. It has already been setup via setQueueName. Do only one of the two.
    String confQueueName = conf.get(TezConfiguration.TEZ_QUEUE_NAME);
    if (queueName != null && !queueName.equals(confQueueName)) {
        LOG.warn("Resetting a queue name that was already set: was " + queueName + ", now " + confQueueName);
    }
    this.queueName = confQueueName;
    this.doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS);
    final boolean llapMode = "llap".equalsIgnoreCase(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE));
    // TODO This - at least for the session pool - will always be the hive user. How does doAs above this affect things ?
    UserGroupInformation ugi = Utils.getUGI();
    user = ugi.getShortUserName();
    LOG.info("User of session id " + sessionId + " is " + user);
    // create the tez tmp dir
    tezScratchDir = scratchDir == null ? createTezDir(sessionId) : scratchDir;
    additionalFilesNotFromConf.clear();
    if (additionalFiles != null) {
        additionalFilesNotFromConf.addAll(additionalFiles);
    }
    refreshLocalResourcesFromConf(conf);
    // unless already installed on all the cluster nodes, we'll have to
    // localize hive-exec.jar as well.
    appJarLr = createJarLocalResource(utils.getExecJarPathLocal());
    // configuration for the application master
    final Map<String, LocalResource> commonLocalResources = new HashMap<String, LocalResource>();
    commonLocalResources.put(utils.getBaseName(appJarLr), appJarLr);
    for (LocalResource lr : localizedResources) {
        commonLocalResources.put(utils.getBaseName(lr), lr);
    }
    if (llapMode) {
        // localize llap client jars
        addJarLRByClass(LlapTaskSchedulerService.class, commonLocalResources);
        addJarLRByClass(LlapProtocolClientImpl.class, commonLocalResources);
        addJarLRByClass(LlapProtocolClientProxy.class, commonLocalResources);
        addJarLRByClassName("org.apache.hadoop.registry.client.api.RegistryOperations", commonLocalResources);
    }
    // Create environment for AM.
    Map<String, String> amEnv = new HashMap<String, String>();
    MRHelpers.updateEnvBasedOnMRAMEnv(conf, amEnv);
    // and finally we're ready to create and start the session
    // generate basic tez config
    final TezConfiguration tezConfig = new TezConfiguration(conf);
    // set up the staging directory to use
    tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString());
    conf.stripHiddenConfigurations(tezConfig);
    ServicePluginsDescriptor servicePluginsDescriptor;
    Credentials llapCredentials = null;
    if (llapMode) {
        if (UserGroupInformation.isSecurityEnabled()) {
            llapCredentials = new Credentials();
            llapCredentials.addToken(LlapTokenIdentifier.KIND_NAME, getLlapToken(user, tezConfig));
        }
        // TODO Change this to not serialize the entire Configuration - minor.
        UserPayload servicePluginPayload = TezUtils.createUserPayloadFromConf(tezConfig);
        // we need plugins to handle llap and uber mode
        servicePluginsDescriptor = ServicePluginsDescriptor.create(true, new TaskSchedulerDescriptor[] { TaskSchedulerDescriptor.create(LLAP_SERVICE, LLAP_SCHEDULER).setUserPayload(servicePluginPayload) }, new ContainerLauncherDescriptor[] { ContainerLauncherDescriptor.create(LLAP_SERVICE, LLAP_LAUNCHER) }, new TaskCommunicatorDescriptor[] { TaskCommunicatorDescriptor.create(LLAP_SERVICE, LLAP_TASK_COMMUNICATOR).setUserPayload(servicePluginPayload) });
    } else {
        servicePluginsDescriptor = ServicePluginsDescriptor.create(true);
    }
    // container prewarming. tell the am how many containers we need
    if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
        int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS);
        n = Math.max(tezConfig.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT), n);
        tezConfig.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, n);
    }
    setupSessionAcls(tezConfig, conf);
    final TezClient session = TezClient.newBuilder("HIVE-" + sessionId, tezConfig).setIsSession(true).setLocalResources(commonLocalResources).setCredentials(llapCredentials).setServicePluginDescriptor(servicePluginsDescriptor).build();
    LOG.info("Opening new Tez Session (id: " + sessionId + ", scratch dir: " + tezScratchDir + ")");
    TezJobMonitor.initShutdownHook();
    if (!isAsync) {
        startSessionAndContainers(session, conf, commonLocalResources, tezConfig, false);
        this.session = session;
    } else {
        FutureTask<TezClient> sessionFuture = new FutureTask<>(new Callable<TezClient>() {

            @Override
            public TezClient call() throws Exception {
                try {
                    return startSessionAndContainers(session, conf, commonLocalResources, tezConfig, true);
                } catch (Throwable t) {
                    LOG.error("Failed to start Tez session", t);
                    throw (t instanceof Exception) ? (Exception) t : new Exception(t);
                }
            }
        });
        new Thread(sessionFuture, "Tez session start thread").start();
        // We assume here nobody will try to get session before open() returns.
        this.console = console;
        this.sessionFuture = sessionFuture;
    }
}

Example 63 with FutureTask

use of java.util.concurrent.FutureTask in project hive by apache.

the class TestTezWorkConcurrency method ensureDagIdIsUnique.

@Test
public void ensureDagIdIsUnique() throws Exception {
    final int threadCount = 5;
    final CountDownLatch threadReadyToStartSignal = new CountDownLatch(threadCount);
    final CountDownLatch startThreadSignal = new CountDownLatch(1);
    final int numberOfTezWorkToCreatePerThread = 100;
    List<FutureTask<Set<String>>> tasks = Lists.newArrayList();
    for (int i = 0; i < threadCount; i++) {
        tasks.add(new FutureTask<>(new Callable<Set<String>>() {

            @Override
            public Set<String> call() throws Exception {
                threadReadyToStartSignal.countDown();
                startThreadSignal.await();
                return generateTezWorkDagIds(numberOfTezWorkToCreatePerThread);
            }
        }));
    }
    ExecutorService executor = Executors.newFixedThreadPool(threadCount);
    for (FutureTask<Set<String>> task : tasks) {
        executor.execute(task);
    }
    threadReadyToStartSignal.await();
    startThreadSignal.countDown();
    Set<String> allTezWorkDagIds = getAllTezWorkDagIds(tasks);
    assertEquals(threadCount * numberOfTezWorkToCreatePerThread, allTezWorkDagIds.size());
}

Example 64 with FutureTask

use of java.util.concurrent.FutureTask in project hive by apache.

the class TestFixedSizedObjectPool method testMTTImpl.

public void testMTTImpl(int size, int takerCount, int giverCount) {
    final int TASK_COUNT = takerCount + giverCount, GIVECOUNT = 15000, TAKECOUNT = 15000;
    ExecutorService executor = Executors.newFixedThreadPool(TASK_COUNT);
    final CountDownLatch cdlIn = new CountDownLatch(TASK_COUNT), cdlOut = new CountDownLatch(1);
    final FixedSizedObjectPool<Object> pool = new FixedSizedObjectPool<>(size, new OneObjHelper(), true);
    // Pre-fill the pool halfway.
    HashSet<Object> allGiven = new HashSet<>();
    for (int i = 0; i < (size >> 1); ++i) {
        Object o = new Object();
        allGiven.add(o);
        assertTrue(pool.tryOffer(o));
    }
    @SuppressWarnings("unchecked") FutureTask<Object>[] tasks = new FutureTask[TASK_COUNT];
    TakeRunnable[] takers = new TakeRunnable[takerCount];
    OfferRunnable[] givers = new OfferRunnable[giverCount];
    int ti = 0;
    for (int i = 0; i < takerCount; ++i, ++ti) {
        takers[i] = new TakeRunnable(pool, cdlIn, cdlOut, TAKECOUNT);
        tasks[ti] = new FutureTask<Object>(takers[i], null);
        executor.execute(tasks[ti]);
    }
    for (int i = 0; i < giverCount; ++i, ++ti) {
        givers[i] = new OfferRunnable(pool, cdlIn, cdlOut, GIVECOUNT);
        tasks[ti] = new FutureTask<Object>(givers[i], null);
        executor.execute(tasks[ti]);
    }
    long time = 0;
    try {
        // Wait for all threads to be ready.
        cdlIn.await();
        time = System.nanoTime();
        // Release them at the same time.
        cdlOut.countDown();
        for (int i = 0; i < TASK_COUNT; ++i) {
            tasks[i].get();
        }
        time = (System.nanoTime() - time);
    } catch (Throwable t) {
        throw new RuntimeException(t);
    }
    int given = allGiven.size(), takenOld = 0;
    for (OfferRunnable g : givers) {
        for (Object o : g.objects) {
            assertTrue(allGiven.add(o));
            ++given;
        }
    }
    for (TakeRunnable t : takers) {
        for (Object o : t.objects) {
            assertTrue(allGiven.remove(o));
            ++takenOld;
        }
    }
    LOG.info("MTT test - size " + size + ", takers/givers " + takerCount + "/" + giverCount + "; offered " + (given - (size >> 1)) + " (attempted " + (GIVECOUNT * giverCount) + "); reused " + takenOld + ", allocated " + ((TAKECOUNT * takerCount) - takenOld) + " (took " + time / 1000000L + "ms including thread sync)");
    // Verify that we can drain the pool, then cycle it, i.e. the state is not corrupted.
    while (pool.take() != OneObjHelper.THE_OBJECT) ;
    for (int i = 0; i < size; ++i) {
        assertTrue(pool.tryOffer(new Object()));
    }
    assertFalse(pool.tryOffer(new Object()));
    for (int i = 0; i < size; ++i) {
        assertTrue(OneObjHelper.THE_OBJECT != pool.take());
    }
    assertTrue(OneObjHelper.THE_OBJECT == pool.take());
}

Example 65 with FutureTask

use of java.util.concurrent.FutureTask in project hive by apache.

the class TestIOContextMap method testTezLlapAttemptMap.

@Test
public void testTezLlapAttemptMap() throws Exception {
    // Tests that different threads get the same object per attempt per input, and different
    // between attempts/inputs; that attempt is inherited between threads; and that clearing
    // the attempt produces a different result.
    final int THREAD_COUNT = 2, ITER_COUNT = 1000, ATTEMPT_COUNT = 3;
    final AtomicInteger countdown = new AtomicInteger(ITER_COUNT);
    final IOContext[] results = new IOContext[ITER_COUNT * ATTEMPT_COUNT];
    ExecutorService executor = Executors.newFixedThreadPool(THREAD_COUNT);
    final CountDownLatch cdlIn = new CountDownLatch(THREAD_COUNT), cdlOut = new CountDownLatch(1);
    @SuppressWarnings("unchecked") FutureTask<Void>[] tasks = new FutureTask[THREAD_COUNT];
    for (int i = 0; i < tasks.length; ++i) {
        tasks[i] = new FutureTask<Void>(new Callable<Void>() {

            public Void call() throws Exception {
                final Configuration conf = new Configuration(), conf2 = new Configuration();
                syncThreadStart(cdlIn, cdlOut);
                while (true) {
                    int nextIx = countdown.decrementAndGet();
                    if (nextIx < 0)
                        break;
                    String input1 = "Input " + nextIx;
                    conf.set(Utilities.INPUT_NAME, input1);
                    for (int j = 0; j < ATTEMPT_COUNT; ++j) {
                        String attemptId = "Attempt " + nextIx + ":" + j;
                        IOContextMap.setThreadAttemptId(attemptId);
                        final IOContext r1 = results[(nextIx * ATTEMPT_COUNT) + j] = IOContextMap.get(conf);
                        // For some attempts, check inheritance.
                        if ((nextIx % (ITER_COUNT / 10)) == 0) {
                            String input2 = "Input2 " + nextIx;
                            conf2.set(Utilities.INPUT_NAME, input2);
                            final AtomicReference<IOContext> ref2 = new AtomicReference<>();
                            Thread t = new Thread(new Runnable() {

                                public void run() {
                                    assertSame(r1, IOContextMap.get(conf));
                                    ref2.set(IOContextMap.get(conf2));
                                }
                            });
                            t.start();
                            t.join();
                            assertSame(ref2.get(), IOContextMap.get(conf2));
                        }
                    // Don't clear the attempt ID, or the stuff will be cleared.
                    }
                    if (nextIx == 0)
                        break;
                }
                return null;
            }
        });
        executor.execute(tasks[i]);
    }
    // Wait for all threads to be ready.
    cdlIn.await();
    // Release them at the same time.
    cdlOut.countDown();
    for (int i = 0; i < tasks.length; ++i) {
        tasks[i].get();
    }
    Configuration conf = new Configuration();
    Set<IOContext> resultSet = Sets.newIdentityHashSet();
    for (int i = 0; i < ITER_COUNT; ++i) {
        conf.set(Utilities.INPUT_NAME, "Input " + i);
        for (int j = 0; j < ATTEMPT_COUNT; ++j) {
            String attemptId = "Attempt " + i + ":" + j;
            IOContext result = results[(i * ATTEMPT_COUNT) + j];
            // All the objects must be different.
            assertTrue(resultSet.add(result));
            IOContextMap.setThreadAttemptId(attemptId);
            // Matching result for attemptId + input.
            assertSame(result, IOContextMap.get(conf));
            IOContextMap.clearThreadAttempt(attemptId);
            IOContextMap.setThreadAttemptId(attemptId);
            // Different result after clearing.
            assertNotSame(result, IOContextMap.get(conf));
        }
    }
}