Examples with ExecutorService java.util.concurrent.ExecutorService used on opensource projects

Example 81 with ExecutorService

use of java.util.concurrent.ExecutorService in project hadoop by apache.

the class ITestBlockingThreadPoolExecutorService method testChainedQueue.

@Test
public void testChainedQueue() throws Throwable {
    ensureCreated();
    int size = 2;
    ExecutorService wrapper = new SemaphoredDelegatingExecutor(tpe, size, true);
    verifyQueueSize(wrapper, size);
}

Example 82 with ExecutorService

use of java.util.concurrent.ExecutorService in project hadoop by apache.

the class TestNMTimelineCollectorManager method testMultithreadedAddAndRemove.

@Test
public void testMultithreadedAddAndRemove() throws Exception {
    final int numApps = 5;
    List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
    for (int i = 0; i < numApps; i++) {
        final ApplicationId appId = ApplicationId.newInstance(0L, i);
        Callable<Boolean> task = new Callable<Boolean>() {

            public Boolean call() {
                AppLevelTimelineCollector collector = new AppLevelTimelineCollector(appId);
                boolean successPut = (collectorManager.putIfAbsent(appId, collector) == collector);
                return successPut && collectorManager.remove(appId);
            }
        };
        tasks.add(task);
    }
    ExecutorService executor = Executors.newFixedThreadPool(numApps);
    try {
        List<Future<Boolean>> futures = executor.invokeAll(tasks);
        for (Future<Boolean> future : futures) {
            assertTrue(future.get());
        }
    } finally {
        executor.shutdownNow();
    }
    // check the keys
    for (int i = 0; i < numApps; i++) {
        final ApplicationId appId = ApplicationId.newInstance(0L, i);
        assertFalse(collectorManager.containsTimelineCollector(appId));
    }
}

Example 83 with ExecutorService

use of java.util.concurrent.ExecutorService in project hbase by apache.

the class ProcedureWALPerformanceEvaluation method doWork.

@Override
public int doWork() {
    try {
        setupProcedureStore();
        ExecutorService executor = Executors.newFixedThreadPool(numThreads);
        Future<?>[] futures = new Future<?>[numThreads];
        // Start worker threads.
        long start = System.currentTimeMillis();
        for (int i = 0; i < numThreads; i++) {
            futures[i] = executor.submit(new Worker(start));
        }
        boolean failure = false;
        try {
            for (Future<?> future : futures) {
                long timeout = start + WORKER_THREADS_TIMEOUT_SEC * 1000 - System.currentTimeMillis();
                failure |= (future.get(timeout, TimeUnit.MILLISECONDS).equals(EXIT_FAILURE));
            }
        } catch (Exception e) {
            System.err.println("Exception in worker thread.");
            e.printStackTrace();
            return EXIT_FAILURE;
        }
        executor.shutdown();
        if (failure) {
            return EXIT_FAILURE;
        }
        long timeTaken = System.currentTimeMillis() - start;
        System.out.println("******************************************");
        System.out.println("Num threads    : " + numThreads);
        System.out.println("Num procedures : " + numProcs);
        System.out.println("Sync type      : " + syncType);
        System.out.println("Time taken     : " + (timeTaken / 1000.0f) + "sec");
        System.out.println("******************************************");
        System.out.println("Raw format for scripts");
        System.out.println(String.format("RESULT [%s=%s, %s=%s, %s=%s, %s=%s, %s=%s, " + "total_time_ms=%s]", NUM_PROCS_OPTION.getOpt(), numProcs, STATE_SIZE_OPTION.getOpt(), stateSize, SYNC_OPTION.getOpt(), syncType, NUM_THREADS_OPTION.getOpt(), numThreads, NUM_WALS_OPTION.getOpt(), numWals, timeTaken));
        return EXIT_SUCCESS;
    } catch (IOException e) {
        e.printStackTrace();
        return EXIT_FAILURE;
    } finally {
        tearDownProcedureStore();
    }
}

Example 84 with ExecutorService

use of java.util.concurrent.ExecutorService in project hbase by apache.

the class LoadIncrementalHFiles method doBulkLoad.

/**
   * Perform a bulk load of the given directory into the given
   * pre-existing table.  This method is not threadsafe.
   *
   * @param hfofDir the directory that was provided as the output path
   *   of a job using HFileOutputFormat
   * @param admin the Admin
   * @param table the table to load into
   * @param regionLocator region locator
   * @param silence true to ignore unmatched column families
   * @param copyFile always copy hfiles if true
   * @throws TableNotFoundException if table does not yet exist
   */
public void doBulkLoad(Path hfofDir, final Admin admin, Table table, RegionLocator regionLocator, boolean silence, boolean copyFile) throws TableNotFoundException, IOException {
    if (!admin.isTableAvailable(regionLocator.getName())) {
        throw new TableNotFoundException("Table " + table.getName() + " is not currently available.");
    }
    /*
     * Checking hfile format is a time-consuming operation, we should have an option to skip
     * this step when bulkloading millions of HFiles. See HBASE-13985.
     */
    boolean validateHFile = getConf().getBoolean("hbase.loadincremental.validate.hfile", true);
    if (!validateHFile) {
        LOG.warn("You are skipping HFiles validation, it might cause some data loss if files " + "are not correct. If you fail to read data from your table after using this " + "option, consider removing the files and bulkload again without this option. " + "See HBASE-13985");
    }
    // LQI queue does not need to be threadsafe -- all operations on this queue
    // happen in this thread
    Deque<LoadQueueItem> queue = new LinkedList<>();
    ExecutorService pool = null;
    SecureBulkLoadClient secureClient = null;
    try {
        prepareHFileQueue(hfofDir, table, queue, validateHFile, silence);
        if (queue.isEmpty()) {
            LOG.warn("Bulk load operation did not find any files to load in " + "directory " + hfofDir != null ? hfofDir.toUri() : "" + ".  Does it contain files in " + "subdirectories that correspond to column family names?");
            return;
        }
        pool = createExecutorService();
        secureClient = new SecureBulkLoadClient(table.getConfiguration(), table);
        retValue = performBulkLoad(admin, table, regionLocator, queue, pool, secureClient, copyFile);
    } finally {
        cleanup(admin, queue, pool, secureClient);
    }
}

Example 85 with ExecutorService

use of java.util.concurrent.ExecutorService in project hbase by apache.

the class LoadIncrementalHFiles method doBulkLoad.

/**
   * Perform a bulk load of the given directory into the given
   * pre-existing table.  This method is not threadsafe.
   *
   * @param map map of family to List of hfiles
   * @param admin the Admin
   * @param table the table to load into
   * @param regionLocator region locator
   * @param silence true to ignore unmatched column families
   * @param copyFile always copy hfiles if true
   * @throws TableNotFoundException if table does not yet exist
   */
public void doBulkLoad(Map<byte[], List<Path>> map, final Admin admin, Table table, RegionLocator regionLocator, boolean silence, boolean copyFile) throws TableNotFoundException, IOException {
    if (!admin.isTableAvailable(regionLocator.getName())) {
        throw new TableNotFoundException("Table " + table.getName() + " is not currently available.");
    }
    // LQI queue does not need to be threadsafe -- all operations on this queue
    // happen in this thread
    Deque<LoadQueueItem> queue = new LinkedList<>();
    ExecutorService pool = null;
    SecureBulkLoadClient secureClient = null;
    try {
        prepareHFileQueue(map, table, queue, silence);
        if (queue.isEmpty()) {
            LOG.warn("Bulk load operation did not get any files to load");
            return;
        }
        pool = createExecutorService();
        secureClient = new SecureBulkLoadClient(table.getConfiguration(), table);
        for (Map.Entry<byte[], List<Path>> entry : map.entrySet()) {
            for (Path p : entry.getValue()) {
                fs = p.getFileSystem(table.getConfiguration());
                break;
            }
        }
        retValue = performBulkLoad(admin, table, regionLocator, queue, pool, secureClient, copyFile);
    } finally {
        cleanup(admin, queue, pool, secureClient);
    }
}