Examples with ExecutorService java.util.concurrent.ExecutorService used on opensource projects

Example 51 with ExecutorService

use of java.util.concurrent.ExecutorService in project jersey by jersey.

the class AsyncResourceTest method testLongRunningResource.

@Test
public void testLongRunningResource() throws InterruptedException {
    final WebTarget resourceTarget = target().path(App.ASYNC_LONG_RUNNING_OP_PATH);
    final String expectedResponse = SimpleLongRunningResource.NOTIFICATION_RESPONSE;
    final int MAX_MESSAGES = 100;
    final int LATCH_WAIT_TIMEOUT = 25 * getAsyncTimeoutMultiplier();
    final boolean debugMode = false;
    final boolean sequentialGet = false;
    final Object sequentialGetLock = new Object();
    final ExecutorService executor = Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat("async-resource-test-%02d").setUncaughtExceptionHandler(new JerseyProcessingUncaughtExceptionHandler()).build());
    final Map<Integer, String> getResponses = new ConcurrentHashMap<Integer, String>();
    final CountDownLatch getRequestLatch = new CountDownLatch(MAX_MESSAGES);
    try {
        for (int i = 0; i < MAX_MESSAGES; i++) {
            final int requestId = i;
            executor.submit(new Runnable() {

                @Override
                public void run() {
                    if (debugMode || sequentialGet) {
                        synchronized (sequentialGetLock) {
                            get();
                        }
                    } else {
                        get();
                    }
                }

                private void get() {
                    try {
                        int attemptCounter = 0;
                        while (true) {
                            attemptCounter++;
                            try {
                                final String response = resourceTarget.request().get(String.class);
                                getResponses.put(requestId, response);
                                break;
                            } catch (Throwable t) {
                                LOGGER.log(Level.SEVERE, String.format("Error sending GET request <%s> for %d. time.", requestId, attemptCounter), t);
                            }
                            if (attemptCounter > 3) {
                                break;
                            }
                            Thread.sleep(10);
                        }
                    } catch (InterruptedException ignored) {
                        LOGGER.log(Level.WARNING, String.format("Error sending GET message <%s>: Interrupted", requestId), ignored);
                    } finally {
                        getRequestLatch.countDown();
                    }
                }
            });
        }
        if (debugMode) {
            getRequestLatch.await();
        } else {
            if (!getRequestLatch.await(LATCH_WAIT_TIMEOUT, TimeUnit.SECONDS)) {
                LOGGER.log(Level.SEVERE, "Waiting for all GET requests to complete has timed out.");
            }
        }
    } finally {
        executor.shutdownNow();
    }
    final ArrayList<Map.Entry<Integer, String>> responseEntryList = new ArrayList<Map.Entry<Integer, String>>(getResponses.entrySet());
    Collections.sort(responseEntryList, new Comparator<Map.Entry<Integer, String>>() {

        @Override
        public int compare(Map.Entry<Integer, String> o1, Map.Entry<Integer, String> o2) {
            return o1.getKey().compareTo(o2.getKey());
        }
    });
    StringBuilder messageBuilder = new StringBuilder("GET responses received: ").append(responseEntryList.size()).append("\n");
    for (Map.Entry<Integer, String> getResponseEntry : responseEntryList) {
        messageBuilder.append(String.format("GET response for message %02d: ", getResponseEntry.getKey())).append(getResponseEntry.getValue()).append('\n');
    }
    LOGGER.info(messageBuilder.toString());
    for (Map.Entry<Integer, String> entry : responseEntryList) {
        assertEquals(String.format("Unexpected GET notification response for message %02d", entry.getKey()), expectedResponse, entry.getValue());
    }
    assertEquals(MAX_MESSAGES, getResponses.size());
}

Example 52 with ExecutorService

use of java.util.concurrent.ExecutorService in project languagetool by languagetool-org.

the class MultiThreadingTest1 method test.

@Test
@Ignore("for interactive use only")
public void test() throws Exception {
    List<Language> languages1 = new ArrayList<>(Languages.get());
    initExpectedResults(languages1);
    List<Language> languages2 = new ArrayList<>(Languages.get());
    ExecutorService executor = Executors.newFixedThreadPool(THREADS);
    for (int i = 0; i < RUNS; i++) {
        System.out.println("Run #" + i);
        Collections.shuffle(languages1, rnd);
        Collections.shuffle(languages2, rnd);
        List<Future> futures = new ArrayList<>();
        for (int j = 0; j < languages1.size(); j++) {
            Language lang1 = languages1.get(j);
            Language lang2 = languages2.get(j);
            //System.out.println("Checking " + lang1 + " and " + lang2);
            futures.add(executor.submit(new Handler(lang1)));
            futures.add(executor.submit(new Handler(lang2)));
        }
        for (Future future : futures) {
            // wait for all results or exception
            future.get();
        }
    }
}

Example 53 with ExecutorService

use of java.util.concurrent.ExecutorService in project pinot by linkedin.

the class CreateSegmentCommand method execute.

@Override
public boolean execute() throws Exception {
    LOGGER.info("Executing command: {}", toString());
    // Load generator config if exist.
    final SegmentGeneratorConfig segmentGeneratorConfig;
    if (_generatorConfigFile != null) {
        segmentGeneratorConfig = new ObjectMapper().readValue(new File(_generatorConfigFile), SegmentGeneratorConfig.class);
    } else {
        segmentGeneratorConfig = new SegmentGeneratorConfig();
    }
    // Load config from segment generator config.
    String configDataDir = segmentGeneratorConfig.getDataDir();
    if (_dataDir == null) {
        if (configDataDir == null) {
            throw new RuntimeException("Must specify dataDir.");
        }
        _dataDir = configDataDir;
    } else {
        if (configDataDir != null && !configDataDir.equals(_dataDir)) {
            LOGGER.warn("Find dataDir conflict in command line and config file, use config in command line: {}", _dataDir);
        }
    }
    FileFormat configFormat = segmentGeneratorConfig.getFormat();
    if (_format == null) {
        if (configFormat == null) {
            throw new RuntimeException("Format cannot be null in config file.");
        }
        _format = configFormat;
    } else {
        if (configFormat != _format && configFormat != FileFormat.AVRO) {
            LOGGER.warn("Find format conflict in command line and config file, use config in command line: {}", _format);
        }
    }
    String configOutDir = segmentGeneratorConfig.getOutDir();
    if (_outDir == null) {
        if (configOutDir == null) {
            throw new RuntimeException("Must specify outDir.");
        }
        _outDir = configOutDir;
    } else {
        if (configOutDir != null && !configOutDir.equals(_outDir)) {
            LOGGER.warn("Find outDir conflict in command line and config file, use config in command line: {}", _outDir);
        }
    }
    if (segmentGeneratorConfig.isOverwrite()) {
        _overwrite = true;
    }
    String configTableName = segmentGeneratorConfig.getTableName();
    if (_tableName == null) {
        if (configTableName == null) {
            throw new RuntimeException("Must specify tableName.");
        }
        _tableName = configTableName;
    } else {
        if (configTableName != null && !configTableName.equals(_tableName)) {
            LOGGER.warn("Find tableName conflict in command line and config file, use config in command line: {}", _tableName);
        }
    }
    String configSegmentName = segmentGeneratorConfig.getSegmentName();
    if (_segmentName == null) {
        if (configSegmentName == null) {
            throw new RuntimeException("Must specify segmentName.");
        }
        _segmentName = configSegmentName;
    } else {
        if (configSegmentName != null && !configSegmentName.equals(_segmentName)) {
            LOGGER.warn("Find segmentName conflict in command line and config file, use config in command line: {}", _segmentName);
        }
    }
    // Filter out all input files.
    File dir = new File(_dataDir);
    if (!dir.exists() || !dir.isDirectory()) {
        throw new RuntimeException("Data directory " + _dataDir + " not found.");
    }
    File[] files = dir.listFiles(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return name.toLowerCase().endsWith(_format.toString().toLowerCase());
        }
    });
    if ((files == null) || (files.length == 0)) {
        throw new RuntimeException("Data directory " + _dataDir + " does not contain " + _format.toString().toUpperCase() + " files.");
    }
    // Make sure output directory does not already exist, or can be overwritten.
    File outDir = new File(_outDir);
    if (outDir.exists()) {
        if (!_overwrite) {
            throw new IOException("Output directory " + _outDir + " already exists.");
        } else {
            FileUtils.deleteDirectory(outDir);
        }
    }
    // Set other generator configs from command line.
    segmentGeneratorConfig.setDataDir(_dataDir);
    segmentGeneratorConfig.setFormat(_format);
    segmentGeneratorConfig.setOutDir(_outDir);
    segmentGeneratorConfig.setOverwrite(_overwrite);
    segmentGeneratorConfig.setTableName(_tableName);
    segmentGeneratorConfig.setSegmentName(_segmentName);
    if (_schemaFile != null) {
        if (segmentGeneratorConfig.getSchemaFile() != null && !segmentGeneratorConfig.getSchemaFile().equals(_schemaFile)) {
            LOGGER.warn("Find schemaFile conflict in command line and config file, use config in command line: {}", _schemaFile);
        }
        segmentGeneratorConfig.setSchemaFile(_schemaFile);
    }
    if (_readerConfigFile != null) {
        if (segmentGeneratorConfig.getReaderConfigFile() != null && !segmentGeneratorConfig.getReaderConfigFile().equals(_readerConfigFile)) {
            LOGGER.warn("Find readerConfigFile conflict in command line and config file, use config in command line: {}", _readerConfigFile);
        }
        segmentGeneratorConfig.setReaderConfigFile(_readerConfigFile);
    }
    if (_enableStarTreeIndex) {
        segmentGeneratorConfig.setEnableStarTreeIndex(true);
    }
    if (_starTreeIndexSpecFile != null) {
        if (segmentGeneratorConfig.getStarTreeIndexSpecFile() != null && !segmentGeneratorConfig.getStarTreeIndexSpecFile().equals(_starTreeIndexSpecFile)) {
            LOGGER.warn("Find starTreeIndexSpecFile conflict in command line and config file, use config in command line: {}", _starTreeIndexSpecFile);
        }
        segmentGeneratorConfig.setStarTreeIndexSpecFile(_starTreeIndexSpecFile);
    }
    ExecutorService executor = Executors.newFixedThreadPool(_numThreads);
    int cnt = 0;
    for (final File file : files) {
        final int segCnt = cnt;
        executor.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    SegmentGeneratorConfig config = new SegmentGeneratorConfig(segmentGeneratorConfig);
                    config.setInputFilePath(file.getAbsolutePath());
                    config.setSegmentName(_segmentName + "_" + segCnt);
                    config.loadConfigFiles();
                    final SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
                    driver.init(config);
                    driver.build();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        });
        cnt += 1;
    }
    executor.shutdown();
    return executor.awaitTermination(1, TimeUnit.HOURS);
}

Example 54 with ExecutorService

use of java.util.concurrent.ExecutorService in project pinot by linkedin.

the class QueryRunner method targetQPSQueryRunner.

/**
   * Use multiple threads to run query at a target QPS.
   * <p>Use a concurrent linked queue to buffer the queries to be sent. Use the main thread to insert queries into the
   * queue at the target QPS, and start <code>numThreads</code> worker threads to fetch queries from the queue and send
   * them.
   * <p>The main thread is responsible for collecting and logging the statistic information periodically.
   * <p>Queries are picked sequentially from the query file.
   * <p>Query runner will stop when all queries in the query file has been executed number of times configured.
   *
   * @param conf perf benchmark driver config.
   * @param queryFile query file.
   * @param numTimesToRunQueries number of times to run all queries in the query file, 0 means infinite times.
   * @param numThreads number of threads sending queries.
   * @param startQPS start QPS (target QPS).
   * @param reportIntervalMs report interval in milliseconds.
   * @param numIntervalsToReportAndClearStatistics number of report intervals to report detailed statistics and clear
   *                                               them, 0 means never.
   * @throws Exception
   */
public static void targetQPSQueryRunner(PerfBenchmarkDriverConf conf, String queryFile, int numTimesToRunQueries, int numThreads, double startQPS, int reportIntervalMs, int numIntervalsToReportAndClearStatistics) throws Exception {
    List<String> queries;
    try (FileInputStream input = new FileInputStream(new File(queryFile))) {
        queries = IOUtils.readLines(input);
    }
    PerfBenchmarkDriver driver = new PerfBenchmarkDriver(conf);
    ConcurrentLinkedQueue<String> queryQueue = new ConcurrentLinkedQueue<>();
    AtomicInteger numQueriesExecuted = new AtomicInteger(0);
    AtomicLong totalBrokerTime = new AtomicLong(0L);
    AtomicLong totalClientTime = new AtomicLong(0L);
    List<Statistics> statisticsList = Collections.singletonList(new Statistics(CLIENT_TIME_STATISTICS));
    ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
    for (int i = 0; i < numThreads; i++) {
        executorService.submit(new Worker(driver, queryQueue, numQueriesExecuted, totalBrokerTime, totalClientTime, statisticsList));
    }
    executorService.shutdown();
    int queryIntervalMs = (int) (MILLIS_PER_SECOND / startQPS);
    long startTime = System.currentTimeMillis();
    long reportStartTime = startTime;
    int numReportIntervals = 0;
    int numTimesExecuted = 0;
    while (numTimesToRunQueries == 0 || numTimesExecuted < numTimesToRunQueries) {
        if (executorService.isTerminated()) {
            LOGGER.error("All threads got exception and already dead.");
            return;
        }
        for (String query : queries) {
            queryQueue.add(query);
            Thread.sleep(queryIntervalMs);
            long currentTime = System.currentTimeMillis();
            if (currentTime - reportStartTime >= reportIntervalMs) {
                long timePassed = currentTime - startTime;
                int numQueriesExecutedInt = numQueriesExecuted.get();
                LOGGER.info("Target QPS: {}, Time Passed: {}ms, Queries Executed: {}, Average QPS: {}, " + "Average Broker Time: {}ms, Average Client Time: {}ms, Queries Queued: {}.", startQPS, timePassed, numQueriesExecutedInt, numQueriesExecutedInt / ((double) timePassed / MILLIS_PER_SECOND), totalBrokerTime.get() / (double) numQueriesExecutedInt, totalClientTime.get() / (double) numQueriesExecutedInt, queryQueue.size());
                reportStartTime = currentTime;
                numReportIntervals++;
                if ((numIntervalsToReportAndClearStatistics != 0) && (numReportIntervals == numIntervalsToReportAndClearStatistics)) {
                    numReportIntervals = 0;
                    startTime = currentTime;
                    reportAndClearStatistics(numQueriesExecuted, totalBrokerTime, totalClientTime, statisticsList);
                }
            }
        }
        numTimesExecuted++;
    }
    // Wait for all queries getting executed.
    while (queryQueue.size() != 0) {
        Thread.sleep(1);
    }
    executorService.shutdownNow();
    while (!executorService.isTerminated()) {
        Thread.sleep(1);
    }
    long timePassed = System.currentTimeMillis() - startTime;
    int numQueriesExecutedInt = numQueriesExecuted.get();
    LOGGER.info("--------------------------------------------------------------------------------");
    LOGGER.info("FINAL REPORT:");
    LOGGER.info("Target QPS: {}, Time Passed: {}ms, Queries Executed: {}, Average QPS: {}, " + "Average Broker Time: {}ms, Average Client Time: {}ms.", startQPS, timePassed, numQueriesExecutedInt, numQueriesExecutedInt / ((double) timePassed / MILLIS_PER_SECOND), totalBrokerTime.get() / (double) numQueriesExecutedInt, totalClientTime.get() / (double) numQueriesExecutedInt);
    for (Statistics statistics : statisticsList) {
        statistics.report();
    }
}

Example 55 with ExecutorService

use of java.util.concurrent.ExecutorService in project pinot by linkedin.

the class PerfBenchmarkRunner method startServerWithPreLoadedSegments.

private void startServerWithPreLoadedSegments() throws Exception {
    PerfBenchmarkDriverConf perfBenchmarkDriverConf = new PerfBenchmarkDriverConf();
    perfBenchmarkDriverConf.setStartZookeeper(false);
    perfBenchmarkDriverConf.setStartController(false);
    perfBenchmarkDriverConf.setStartBroker(false);
    perfBenchmarkDriverConf.setServerInstanceDataDir(_dataDir);
    final PerfBenchmarkDriver driver = new PerfBenchmarkDriver(perfBenchmarkDriverConf, _tempDir, _loadMode, _segmentFormatVersion, false);
    driver.run();
    if (_isBatchLoad) {
        String[] tableNames = new File(_dataDir).list();
        Preconditions.checkNotNull(tableNames);
        ExecutorService executorService = Executors.newFixedThreadPool(_numThreads);
        for (final String tableName : tableNames) {
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    try {
                        loadTable(driver, _dataDir, tableName, null);
                    } catch (Exception e) {
                        LOGGER.error("Caught exception while loading table: {}", tableName, e);
                    }
                }
            });
        }
        executorService.shutdown();
        executorService.awaitTermination(_timeoutInSeconds, TimeUnit.SECONDS);
    } else {
        List<String> invertedIndexColumns = null;
        if (_invertedIndexColumns != null) {
            invertedIndexColumns = Arrays.asList(_invertedIndexColumns.split(","));
        }
        for (String tableName : _tableNames.split(",")) {
            loadTable(driver, _dataDir, tableName, invertedIndexColumns);
        }
    }
}