Examples with FutureTask java.util.concurrent.FutureTask used on opensource projects

Example 56 with FutureTask

use of java.util.concurrent.FutureTask in project hive by apache.

the class TestFixedSizedObjectPool method testMTTImpl.

public void testMTTImpl(int size, int takerCount, int giverCount) {
    final int TASK_COUNT = takerCount + giverCount, GIVECOUNT = 15000, TAKECOUNT = 15000;
    ExecutorService executor = Executors.newFixedThreadPool(TASK_COUNT);
    final CountDownLatch cdlIn = new CountDownLatch(TASK_COUNT), cdlOut = new CountDownLatch(1);
    final FixedSizedObjectPool<Object> pool = new FixedSizedObjectPool<>(size, new OneObjHelper(), true);
    // Pre-fill the pool halfway.
    HashSet<Object> allGiven = new HashSet<>();
    for (int i = 0; i < (size >> 1); ++i) {
        Object o = new Object();
        allGiven.add(o);
        assertTrue(pool.tryOffer(o));
    }
    @SuppressWarnings("unchecked") FutureTask<Object>[] tasks = new FutureTask[TASK_COUNT];
    TakeRunnable[] takers = new TakeRunnable[takerCount];
    OfferRunnable[] givers = new OfferRunnable[giverCount];
    int ti = 0;
    for (int i = 0; i < takerCount; ++i, ++ti) {
        takers[i] = new TakeRunnable(pool, cdlIn, cdlOut, TAKECOUNT);
        tasks[ti] = new FutureTask<Object>(takers[i], null);
        executor.execute(tasks[ti]);
    }
    for (int i = 0; i < giverCount; ++i, ++ti) {
        givers[i] = new OfferRunnable(pool, cdlIn, cdlOut, GIVECOUNT);
        tasks[ti] = new FutureTask<Object>(givers[i], null);
        executor.execute(tasks[ti]);
    }
    long time = 0;
    try {
        // Wait for all threads to be ready.
        cdlIn.await();
        time = System.nanoTime();
        // Release them at the same time.
        cdlOut.countDown();
        for (int i = 0; i < TASK_COUNT; ++i) {
            tasks[i].get();
        }
        time = (System.nanoTime() - time);
    } catch (Throwable t) {
        throw new RuntimeException(t);
    }
    int given = allGiven.size(), takenOld = 0;
    for (OfferRunnable g : givers) {
        for (Object o : g.objects) {
            assertTrue(allGiven.add(o));
            ++given;
        }
    }
    for (TakeRunnable t : takers) {
        for (Object o : t.objects) {
            assertTrue(allGiven.remove(o));
            ++takenOld;
        }
    }
    LOG.info("MTT test - size " + size + ", takers/givers " + takerCount + "/" + giverCount + "; offered " + (given - (size >> 1)) + " (attempted " + (GIVECOUNT * giverCount) + "); reused " + takenOld + ", allocated " + ((TAKECOUNT * takerCount) - takenOld) + " (took " + time / 1000000L + "ms including thread sync)");
    // Verify that we can drain the pool, then cycle it, i.e. the state is not corrupted.
    while (pool.take() != OneObjHelper.THE_OBJECT) ;
    for (int i = 0; i < size; ++i) {
        assertTrue(pool.tryOffer(new Object()));
    }
    assertFalse(pool.tryOffer(new Object()));
    for (int i = 0; i < size; ++i) {
        assertTrue(OneObjHelper.THE_OBJECT != pool.take());
    }
    assertTrue(OneObjHelper.THE_OBJECT == pool.take());
}

Example 57 with FutureTask

use of java.util.concurrent.FutureTask in project hive by apache.

the class TestBuddyAllocator method testMTTArenas.

@Test
public void testMTTArenas() {
    final int min = 3, max = 4, maxAlloc = 1 << max, minAllocCount = 2048, threadCount = 4;
    final BuddyAllocator a = new BuddyAllocator(isDirect, isMapped, 1 << min, maxAlloc, maxAlloc, (1 << min) * minAllocCount, tmpDir, new DummyMemoryManager(), LlapDaemonCacheMetrics.create("test", "1"));
    ExecutorService executor = Executors.newFixedThreadPool(threadCount);
    final CountDownLatch cdlIn = new CountDownLatch(threadCount), cdlOut = new CountDownLatch(1);
    Callable<Void> testCallable = new Callable<Void>() {

        public Void call() throws Exception {
            syncThreadStart(cdlIn, cdlOut);
            allocSameSize(a, minAllocCount / threadCount, min);
            return null;
        }
    };
    @SuppressWarnings("unchecked") FutureTask<Void>[] allocTasks = new FutureTask[threadCount];
    for (int i = 0; i < threadCount; ++i) {
        allocTasks[i] = new FutureTask<>(testCallable);
        executor.execute(allocTasks[i]);
    }
    try {
        // Wait for all threads to be ready.
        cdlIn.await();
        // Release them at the same time.
        cdlOut.countDown();
        for (int i = 0; i < threadCount; ++i) {
            allocTasks[i].get();
        }
    } catch (Throwable t) {
        throw new RuntimeException(t);
    }
}

Example 58 with FutureTask

use of java.util.concurrent.FutureTask in project hive by apache.

the class TestBuddyAllocator method testMTT.

@Test
public void testMTT() {
    final int min = 3, max = 8, maxAlloc = 1 << max, allocsPerSize = 3;
    final BuddyAllocator a = new BuddyAllocator(isDirect, isMapped, 1 << min, maxAlloc, maxAlloc * 8, maxAlloc * 24, tmpDir, new DummyMemoryManager(), LlapDaemonCacheMetrics.create("test", "1"));
    ExecutorService executor = Executors.newFixedThreadPool(3);
    final CountDownLatch cdlIn = new CountDownLatch(3), cdlOut = new CountDownLatch(1);
    FutureTask<Void> upTask = new FutureTask<Void>(new Callable<Void>() {

        public Void call() throws Exception {
            syncThreadStart(cdlIn, cdlOut);
            allocateUp(a, min, max, allocsPerSize, false);
            allocateUp(a, min, max, allocsPerSize, true);
            return null;
        }
    }), downTask = new FutureTask<Void>(new Callable<Void>() {

        public Void call() throws Exception {
            syncThreadStart(cdlIn, cdlOut);
            allocateDown(a, min, max, allocsPerSize, false);
            allocateDown(a, min, max, allocsPerSize, true);
            return null;
        }
    }), sameTask = new FutureTask<Void>(new Callable<Void>() {

        public Void call() throws Exception {
            syncThreadStart(cdlIn, cdlOut);
            for (int i = min; i <= max; ++i) {
                allocSameSize(a, (1 << (max - i)) * allocsPerSize, i);
            }
            return null;
        }
    });
    executor.execute(sameTask);
    executor.execute(upTask);
    executor.execute(downTask);
    try {
        // Wait for all threads to be ready.
        cdlIn.await();
        // Release them at the same time.
        cdlOut.countDown();
        upTask.get();
        downTask.get();
        sameTask.get();
    } catch (Throwable t) {
        throw new RuntimeException(t);
    }
}

Example 59 with FutureTask

use of java.util.concurrent.FutureTask in project hive by apache.

the class TezSessionState method openInternal.

protected void openInternal(final HiveConf conf, Collection<String> additionalFiles, boolean isAsync, LogHelper console, Path scratchDir) throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException {
    this.conf = conf;
    // TODO Why is the queue name set again. It has already been setup via setQueueName. Do only one of the two.
    String confQueueName = conf.get(TezConfiguration.TEZ_QUEUE_NAME);
    if (queueName != null && !queueName.equals(confQueueName)) {
        LOG.warn("Resetting a queue name that was already set: was " + queueName + ", now " + confQueueName);
    }
    this.queueName = confQueueName;
    this.doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS);
    final boolean llapMode = "llap".equalsIgnoreCase(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE));
    // TODO This - at least for the session pool - will always be the hive user. How does doAs above this affect things ?
    UserGroupInformation ugi = Utils.getUGI();
    user = ugi.getShortUserName();
    LOG.info("User of session id " + sessionId + " is " + user);
    // create the tez tmp dir
    tezScratchDir = scratchDir == null ? createTezDir(sessionId) : scratchDir;
    additionalFilesNotFromConf.clear();
    if (additionalFiles != null) {
        additionalFilesNotFromConf.addAll(additionalFiles);
    }
    refreshLocalResourcesFromConf(conf);
    // unless already installed on all the cluster nodes, we'll have to
    // localize hive-exec.jar as well.
    appJarLr = createJarLocalResource(utils.getExecJarPathLocal());
    // configuration for the application master
    final Map<String, LocalResource> commonLocalResources = new HashMap<String, LocalResource>();
    commonLocalResources.put(utils.getBaseName(appJarLr), appJarLr);
    for (LocalResource lr : localizedResources) {
        commonLocalResources.put(utils.getBaseName(lr), lr);
    }
    if (llapMode) {
        // localize llap client jars
        addJarLRByClass(LlapTaskSchedulerService.class, commonLocalResources);
        addJarLRByClass(LlapProtocolClientImpl.class, commonLocalResources);
        addJarLRByClass(LlapProtocolClientProxy.class, commonLocalResources);
        addJarLRByClassName("org.apache.hadoop.registry.client.api.RegistryOperations", commonLocalResources);
    }
    // Create environment for AM.
    Map<String, String> amEnv = new HashMap<String, String>();
    MRHelpers.updateEnvBasedOnMRAMEnv(conf, amEnv);
    // and finally we're ready to create and start the session
    // generate basic tez config
    final TezConfiguration tezConfig = new TezConfiguration(conf);
    // set up the staging directory to use
    tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString());
    conf.stripHiddenConfigurations(tezConfig);
    ServicePluginsDescriptor servicePluginsDescriptor;
    Credentials llapCredentials = null;
    if (llapMode) {
        if (UserGroupInformation.isSecurityEnabled()) {
            llapCredentials = new Credentials();
            llapCredentials.addToken(LlapTokenIdentifier.KIND_NAME, getLlapToken(user, tezConfig));
        }
        // TODO Change this to not serialize the entire Configuration - minor.
        UserPayload servicePluginPayload = TezUtils.createUserPayloadFromConf(tezConfig);
        // we need plugins to handle llap and uber mode
        servicePluginsDescriptor = ServicePluginsDescriptor.create(true, new TaskSchedulerDescriptor[] { TaskSchedulerDescriptor.create(LLAP_SERVICE, LLAP_SCHEDULER).setUserPayload(servicePluginPayload) }, new ContainerLauncherDescriptor[] { ContainerLauncherDescriptor.create(LLAP_SERVICE, LLAP_LAUNCHER) }, new TaskCommunicatorDescriptor[] { TaskCommunicatorDescriptor.create(LLAP_SERVICE, LLAP_TASK_COMMUNICATOR).setUserPayload(servicePluginPayload) });
    } else {
        servicePluginsDescriptor = ServicePluginsDescriptor.create(true);
    }
    // container prewarming. tell the am how many containers we need
    if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
        int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS);
        n = Math.max(tezConfig.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT), n);
        tezConfig.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, n);
    }
    setupSessionAcls(tezConfig, conf);
    final TezClient session = TezClient.newBuilder("HIVE-" + sessionId, tezConfig).setIsSession(true).setLocalResources(commonLocalResources).setCredentials(llapCredentials).setServicePluginDescriptor(servicePluginsDescriptor).build();
    LOG.info("Opening new Tez Session (id: " + sessionId + ", scratch dir: " + tezScratchDir + ")");
    TezJobMonitor.initShutdownHook();
    if (!isAsync) {
        startSessionAndContainers(session, conf, commonLocalResources, tezConfig, false);
        this.session = session;
    } else {
        FutureTask<TezClient> sessionFuture = new FutureTask<>(new Callable<TezClient>() {

            @Override
            public TezClient call() throws Exception {
                try {
                    return startSessionAndContainers(session, conf, commonLocalResources, tezConfig, true);
                } catch (Throwable t) {
                    LOG.error("Failed to start Tez session", t);
                    throw (t instanceof Exception) ? (Exception) t : new Exception(t);
                }
            }
        });
        new Thread(sessionFuture, "Tez session start thread").start();
        // We assume here nobody will try to get session before open() returns.
        this.console = console;
        this.sessionFuture = sessionFuture;
    }
}

Example 60 with FutureTask

use of java.util.concurrent.FutureTask in project bazel by bazelbuild.

the class SkyframeActionExecutor method executeAction.

/**
   * Executes the provided action on the current thread. Returns the ActionExecutionValue with the
   * result, either computed here or already computed on another thread.
   *
   * <p>For use from {@link ArtifactFunction} only.
   */
ActionExecutionValue executeAction(Action action, ActionMetadataHandler metadataHandler, long actionStartTime, ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException {
    Exception exception = badActionMap.get(action);
    if (exception != null) {
        // If action had a conflict with some other action in the graph, report it now.
        reportError(exception.getMessage(), exception, action, null);
    }
    Artifact primaryOutput = action.getPrimaryOutput();
    FutureTask<ActionExecutionValue> actionTask = new FutureTask<>(new ActionRunner(action, metadataHandler, actionStartTime, actionExecutionContext));
    // Check to see if another action is already executing/has executed this value.
    Pair<Action, FutureTask<ActionExecutionValue>> oldAction = buildActionMap.putIfAbsent(primaryOutput, Pair.of(action, actionTask));
    if (oldAction == null) {
        actionTask.run();
    } else {
        Preconditions.checkState(action != oldAction.first, action);
        Preconditions.checkState(Actions.canBeShared(oldAction.first, action), "Actions cannot be shared: %s %s", oldAction.first, action);
        // Wait for other action to finish, so any actions that depend on its outputs can execute.
        actionTask = oldAction.second;
    }
    try {
        return actionTask.get();
    } catch (ExecutionException e) {
        Throwables.propagateIfPossible(e.getCause(), ActionExecutionException.class, InterruptedException.class);
        throw new IllegalStateException(e);
    } finally {
        String message = action.getProgressMessage();
        if (message != null) {
            // completion messages are enabled (--show_task_finish).
            if (completionReceiver != null) {
                completionReceiver.actionCompleted(action);
            }
            reporter.finishTask(null, prependExecPhaseStats(message));
        }
    }
}