Examples with Callable java.util.concurrent.Callable used on opensource projects

Example 31 with Callable

use of java.util.concurrent.Callable in project hbase by apache.

the class TestFastFail method testFastFail.

@Ignore("Can go zombie -- see HBASE-14421; FIX")
@Test
public void testFastFail() throws IOException, InterruptedException {
    Admin admin = TEST_UTIL.getAdmin();
    final String tableName = name.getMethodName();
    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes(tableName)));
    desc.addFamily(new HColumnDescriptor(FAMILY));
    admin.createTable(desc, Bytes.toBytes("aaaa"), Bytes.toBytes("zzzz"), 32);
    final long numRows = 1000;
    Configuration conf = TEST_UTIL.getConfiguration();
    conf.setLong(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, SLEEPTIME * 100);
    conf.setInt(HConstants.HBASE_CLIENT_PAUSE, SLEEPTIME / 10);
    conf.setBoolean(HConstants.HBASE_CLIENT_FAST_FAIL_MODE_ENABLED, true);
    conf.setLong(HConstants.HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS, 0);
    conf.setClass(HConstants.HBASE_CLIENT_FAST_FAIL_INTERCEPTOR_IMPL, MyPreemptiveFastFailInterceptor.class, PreemptiveFastFailInterceptor.class);
    final Connection connection = ConnectionFactory.createConnection(conf);
    /**
     * Write numRows worth of data, so that the workers can arbitrarily read.
     */
    List<Put> puts = new ArrayList<>();
    for (long i = 0; i < numRows; i++) {
        byte[] rowKey = longToByteArrayKey(i);
        Put put = new Put(rowKey);
        // value is the same as the row key
        byte[] value = rowKey;
        put.addColumn(FAMILY, QUALIFIER, value);
        puts.add(put);
    }
    try (Table table = connection.getTable(TableName.valueOf(tableName))) {
        table.put(puts);
        LOG.info("Written all puts.");
    }
    /**
     * The number of threads that are going to perform actions against the test
     * table.
     */
    int nThreads = 100;
    ExecutorService service = Executors.newFixedThreadPool(nThreads);
    final CountDownLatch continueOtherHalf = new CountDownLatch(1);
    final CountDownLatch doneHalfway = new CountDownLatch(nThreads);
    final AtomicInteger numSuccessfullThreads = new AtomicInteger(0);
    final AtomicInteger numFailedThreads = new AtomicInteger(0);
    // The total time taken for the threads to perform the second put;
    final AtomicLong totalTimeTaken = new AtomicLong(0);
    final AtomicInteger numBlockedWorkers = new AtomicInteger(0);
    final AtomicInteger numPreemptiveFastFailExceptions = new AtomicInteger(0);
    List<Future<Boolean>> futures = new ArrayList<>();
    for (int i = 0; i < nThreads; i++) {
        futures.add(service.submit(new Callable<Boolean>() {

            /**
         * The workers are going to perform a couple of reads. The second read
         * will follow the killing of a regionserver so that we make sure that
         * some of threads go into PreemptiveFastFailExcception
         */
            public Boolean call() throws Exception {
                try (Table table = connection.getTable(TableName.valueOf(tableName))) {
                    // Add some jitter here
                    Thread.sleep(Math.abs(random.nextInt()) % 250);
                    byte[] row = longToByteArrayKey(Math.abs(random.nextLong()) % numRows);
                    Get g = new Get(row);
                    g.addColumn(FAMILY, QUALIFIER);
                    try {
                        table.get(g);
                    } catch (Exception e) {
                        LOG.debug("Get failed : ", e);
                        doneHalfway.countDown();
                        return false;
                    }
                    // Done with one get, proceeding to do the next one.
                    doneHalfway.countDown();
                    continueOtherHalf.await();
                    long startTime = System.currentTimeMillis();
                    g = new Get(row);
                    g.addColumn(FAMILY, QUALIFIER);
                    try {
                        table.get(g);
                        // The get was successful
                        numSuccessfullThreads.addAndGet(1);
                    } catch (Exception e) {
                        if (e instanceof PreemptiveFastFailException) {
                            // We were issued a PreemptiveFastFailException
                            numPreemptiveFastFailExceptions.addAndGet(1);
                        }
                        // Irrespective of PFFE, the request failed.
                        numFailedThreads.addAndGet(1);
                        return false;
                    } finally {
                        long enTime = System.currentTimeMillis();
                        totalTimeTaken.addAndGet(enTime - startTime);
                        if ((enTime - startTime) >= SLEEPTIME) {
                            // Considering the slow workers as the blockedWorkers.
                            // This assumes that the threads go full throttle at performing
                            // actions. In case the thread scheduling itself is as slow as
                            // SLEEPTIME, then this test might fail and so, we might have
                            // set it to a higher number on slower machines.
                            numBlockedWorkers.addAndGet(1);
                        }
                    }
                    return true;
                } catch (Exception e) {
                    LOG.error("Caught unknown exception", e);
                    doneHalfway.countDown();
                    return false;
                }
            }
        }));
    }
    doneHalfway.await();
    // Kill a regionserver
    TEST_UTIL.getHBaseCluster().getRegionServer(0).getRpcServer().stop();
    TEST_UTIL.getHBaseCluster().getRegionServer(0).stop("Testing");
    // Let the threads continue going
    continueOtherHalf.countDown();
    Thread.sleep(2 * SLEEPTIME);
    // Start a RS in the cluster
    TEST_UTIL.getHBaseCluster().startRegionServer();
    int numThreadsReturnedFalse = 0;
    int numThreadsReturnedTrue = 0;
    int numThreadsThrewExceptions = 0;
    for (Future<Boolean> f : futures) {
        try {
            numThreadsReturnedTrue += f.get() ? 1 : 0;
            numThreadsReturnedFalse += f.get() ? 0 : 1;
        } catch (Exception e) {
            numThreadsThrewExceptions++;
        }
    }
    LOG.debug("numThreadsReturnedFalse:" + numThreadsReturnedFalse + " numThreadsReturnedTrue:" + numThreadsReturnedTrue + " numThreadsThrewExceptions:" + numThreadsThrewExceptions + " numFailedThreads:" + numFailedThreads.get() + " numSuccessfullThreads:" + numSuccessfullThreads.get() + " numBlockedWorkers:" + numBlockedWorkers.get() + " totalTimeWaited: " + totalTimeTaken.get() / (numBlockedWorkers.get() == 0 ? Long.MAX_VALUE : numBlockedWorkers.get()) + " numPFFEs: " + numPreemptiveFastFailExceptions.get());
    assertEquals("The expected number of all the successfull and the failed " + "threads should equal the total number of threads that we spawned", nThreads, numFailedThreads.get() + numSuccessfullThreads.get());
    assertEquals("All the failures should be coming from the secondput failure", numFailedThreads.get(), numThreadsReturnedFalse);
    assertEquals("Number of threads that threw execution exceptions " + "otherwise should be 0", numThreadsThrewExceptions, 0);
    assertEquals("The regionservers that returned true should equal to the" + " number of successful threads", numThreadsReturnedTrue, numSuccessfullThreads.get());
    assertTrue("There will be atleast one thread that retried instead of failing", MyPreemptiveFastFailInterceptor.numBraveSouls.get() > 0);
    assertTrue("There will be atleast one PreemptiveFastFail exception," + " otherwise, the test makes little sense." + "numPreemptiveFastFailExceptions: " + numPreemptiveFastFailExceptions.get(), numPreemptiveFastFailExceptions.get() > 0);
    assertTrue("Only few thread should ideally be waiting for the dead " + "regionserver to be coming back. numBlockedWorkers:" + numBlockedWorkers.get() + " threads that retried : " + MyPreemptiveFastFailInterceptor.numBraveSouls.get(), numBlockedWorkers.get() <= MyPreemptiveFastFailInterceptor.numBraveSouls.get());
}

Example 32 with Callable

use of java.util.concurrent.Callable in project hive by apache.

the class TestIOContextMap method testSparkThreadLocal.

@Test
public void testSparkThreadLocal() throws Exception {
    // Test that input name does not change IOContext returned, and that each thread gets its own.
    final Configuration conf1 = new Configuration();
    conf1.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, "spark");
    final Configuration conf2 = new Configuration(conf1);
    conf2.set(Utilities.INPUT_NAME, "Other input");
    final int THREAD_COUNT = 2;
    ExecutorService executor = Executors.newFixedThreadPool(THREAD_COUNT);
    final CountDownLatch cdlIn = new CountDownLatch(THREAD_COUNT), cdlOut = new CountDownLatch(1);
    @SuppressWarnings("unchecked") FutureTask<IOContext>[] tasks = new FutureTask[THREAD_COUNT];
    for (int i = 0; i < tasks.length; ++i) {
        tasks[i] = new FutureTask<IOContext>(new Callable<IOContext>() {

            public IOContext call() throws Exception {
                syncThreadStart(cdlIn, cdlOut);
                IOContext c1 = IOContextMap.get(conf1), c2 = IOContextMap.get(conf2);
                assertSame(c1, c2);
                return c1;
            }
        });
        executor.execute(tasks[i]);
    }
    // Wait for all threads to be ready.
    cdlIn.await();
    // Release them at the same time.
    cdlOut.countDown();
    Set<IOContext> results = Sets.newIdentityHashSet();
    for (int i = 0; i < tasks.length; ++i) {
        // All the objects must be different.
        assertTrue(results.add(tasks[i].get()));
    }
}

Example 33 with Callable

use of java.util.concurrent.Callable in project hive by apache.

the class TestIOContextMap method testMRTezGlobalMap.

@Test
public void testMRTezGlobalMap() throws Exception {
    // Tests concurrent modification, and that results are the same per input across threads
    // but different between inputs.
    final int THREAD_COUNT = 2, ITER_COUNT = 1000;
    final AtomicInteger countdown = new AtomicInteger(ITER_COUNT);
    final CountDownLatch phase1End = new CountDownLatch(THREAD_COUNT);
    final IOContext[] results = new IOContext[ITER_COUNT];
    ExecutorService executor = Executors.newFixedThreadPool(THREAD_COUNT);
    final CountDownLatch cdlIn = new CountDownLatch(THREAD_COUNT), cdlOut = new CountDownLatch(1);
    @SuppressWarnings("unchecked") FutureTask<Void>[] tasks = new FutureTask[THREAD_COUNT];
    for (int i = 0; i < tasks.length; ++i) {
        tasks[i] = new FutureTask<Void>(new Callable<Void>() {

            public Void call() throws Exception {
                Configuration conf = new Configuration();
                syncThreadStart(cdlIn, cdlOut);
                // Phase 1 - create objects.
                while (true) {
                    int nextIx = countdown.decrementAndGet();
                    if (nextIx < 0)
                        break;
                    conf.set(Utilities.INPUT_NAME, "Input " + nextIx);
                    results[nextIx] = IOContextMap.get(conf);
                    if (nextIx == 0)
                        break;
                }
                phase1End.countDown();
                phase1End.await();
                // Phase 2 - verify we get the expected objects created by all threads.
                for (int i = 0; i < ITER_COUNT; ++i) {
                    conf.set(Utilities.INPUT_NAME, "Input " + i);
                    IOContext ctx = IOContextMap.get(conf);
                    assertSame(results[i], ctx);
                }
                return null;
            }
        });
        executor.execute(tasks[i]);
    }
    // Wait for all threads to be ready.
    cdlIn.await();
    // Release them at the same time.
    cdlOut.countDown();
    for (int i = 0; i < tasks.length; ++i) {
        tasks[i].get();
    }
    Set<IOContext> resultSet = Sets.newIdentityHashSet();
    for (int i = 0; i < results.length; ++i) {
        // All the objects must be different.
        assertTrue(resultSet.add(results[i]));
    }
}

Example 34 with Callable

use of java.util.concurrent.Callable in project kafka by apache.

the class DistributedHerder method halt.

// public for testing
public void halt() {
    synchronized (this) {
        // Clean up any connectors and tasks that are still running.
        log.info("Stopping connectors and tasks that are still assigned to this worker.");
        List<Callable<Void>> callables = new ArrayList<>();
        for (String connectorName : new ArrayList<>(worker.connectorNames())) {
            callables.add(getConnectorStoppingCallable(connectorName));
        }
        for (ConnectorTaskId taskId : new ArrayList<>(worker.taskIds())) {
            callables.add(getTaskStoppingCallable(taskId));
        }
        startAndStop(callables);
        member.stop();
        // Explicitly fail any outstanding requests so they actually get a response and get an
        // understandable reason for their failure.
        HerderRequest request = requests.pollFirst();
        while (request != null) {
            request.callback().onCompletion(new ConnectException("Worker is shutting down"), null);
            request = requests.pollFirst();
        }
        stopServices();
    }
}

Example 35 with Callable

use of java.util.concurrent.Callable in project hive by apache.

the class HostExecutor method execInstances.

private List<ListenableFuture<RemoteCommandResult>> execInstances(List<Drone> drones, final String cmd) throws InterruptedException, IOException {
    List<ListenableFuture<RemoteCommandResult>> result = Lists.newArrayList();
    for (final Drone drone : ImmutableList.copyOf(drones)) {
        result.add(mExecutor.submit(new Callable<RemoteCommandResult>() {

            @Override
            public RemoteCommandResult call() throws Exception {
                Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
                templateVariables.put("instanceName", drone.getInstanceName());
                templateVariables.put("localDir", drone.getLocalDirectory());
                String command = Templates.getTemplateResult(cmd, templateVariables);
                SSHResult result = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(), drone.getHost(), drone.getInstance(), command, true).call();
                if (result.getExitCode() != Constants.EXIT_CODE_SUCCESS) {
                    // return value not checked due to concurrent access
                    mDrones.remove(drone);
                    mLogger.error("Aborting drone during exec " + command, new AbortDroneException("Drone " + drone + " exited with " + result.getExitCode() + ": " + result));
                    return null;
                } else {
                    return result;
                }
            }
        }));
    }
    return result;
}