Examples with ListenableFuture com.google.common.util.concurrent.ListenableFuture used on opensource projects

Example 11 with ListenableFuture

use of com.google.common.util.concurrent.ListenableFuture in project druid by druid-io.

the class PrioritizedExecutorServiceTest method testOrderedExecutionEqualPriorityCallable.

@Test
public void testOrderedExecutionEqualPriorityCallable() throws ExecutionException, InterruptedException {
    final int numTasks = 1_000;
    final List<ListenableFuture<?>> futures = Lists.newArrayListWithExpectedSize(numTasks);
    final AtomicInteger hasRun = new AtomicInteger(0);
    for (int i = 0; i < numTasks; ++i) {
        futures.add(exec.submit(getCheckingPrioritizedCallable(i, hasRun)));
    }
    latch.countDown();
    checkFutures(futures);
}

Example 12 with ListenableFuture

use of com.google.common.util.concurrent.ListenableFuture in project druid by druid-io.

the class PrioritizedExecutorServiceTest method testOrderedExecutionMultiplePriorityMix.

@Test
public void testOrderedExecutionMultiplePriorityMix() throws ExecutionException, InterruptedException {
    final int DEFAULT = 0;
    final int MIN = -1;
    final int MAX = 1;
    exec = new PrioritizedExecutorService(exec.threadPoolExecutor, true, DEFAULT, config);
    final int numTasks = 999;
    final int[] priorities = new int[] { MAX, DEFAULT, MIN };
    final int tasksPerPriority = numTasks / priorities.length;
    final int[] priorityOffsets = new int[] { 0, tasksPerPriority, tasksPerPriority * 2 };
    final List<ListenableFuture<?>> futures = Lists.newArrayListWithExpectedSize(numTasks);
    final AtomicInteger hasRun = new AtomicInteger(0);
    final Random random = new Random(789401);
    for (int i = 0; i < numTasks; ++i) {
        final int priorityBucket = i % priorities.length;
        final int myPriority = priorities[priorityBucket];
        final int priorityOffset = priorityOffsets[priorityBucket];
        final int expectedPriorityOrder = i / priorities.length;
        if (random.nextBoolean()) {
            futures.add(exec.submit(getCheckingPrioritizedCallable(priorityOffset + expectedPriorityOrder, hasRun, myPriority)));
        } else {
            futures.add(exec.submit(getCheckingPrioritizedRunnable(priorityOffset + expectedPriorityOrder, hasRun, myPriority)));
        }
    }
    latch.countDown();
    checkFutures(futures);
}

Example 13 with ListenableFuture

use of com.google.common.util.concurrent.ListenableFuture in project druid by druid-io.

the class PrioritizedExecutorServiceTest method testOrderedExecutionEqualPriorityRunnable.

// Make sure entries are processed FIFO
@Test
public void testOrderedExecutionEqualPriorityRunnable() throws ExecutionException, InterruptedException {
    final int numTasks = 100;
    final List<ListenableFuture<?>> futures = Lists.newArrayListWithExpectedSize(numTasks);
    final AtomicInteger hasRun = new AtomicInteger(0);
    for (int i = 0; i < numTasks; ++i) {
        futures.add(exec.submit(getCheckingPrioritizedRunnable(i, hasRun)));
    }
    latch.countDown();
    checkFutures(futures);
}

Example 14 with ListenableFuture

use of com.google.common.util.concurrent.ListenableFuture in project druid by druid-io.

the class ChainedExecutionQueryRunnerTest method testQueryTimeout.

@Test(timeout = 60000)
public void testQueryTimeout() throws Exception {
    ExecutorService exec = PrioritizedExecutorService.create(new Lifecycle(), new DruidProcessingConfig() {

        @Override
        public String getFormatString() {
            return "test";
        }

        @Override
        public int getNumThreads() {
            return 2;
        }
    });
    final CountDownLatch queriesStarted = new CountDownLatch(2);
    final CountDownLatch queriesInterrupted = new CountDownLatch(2);
    final CountDownLatch queryIsRegistered = new CountDownLatch(1);
    Capture<ListenableFuture> capturedFuture = new Capture<>();
    QueryWatcher watcher = EasyMock.createStrictMock(QueryWatcher.class);
    watcher.registerQuery(EasyMock.<Query>anyObject(), EasyMock.and(EasyMock.<ListenableFuture>anyObject(), EasyMock.capture(capturedFuture)));
    EasyMock.expectLastCall().andAnswer(new IAnswer<Void>() {

        @Override
        public Void answer() throws Throwable {
            queryIsRegistered.countDown();
            return null;
        }
    }).once();
    EasyMock.replay(watcher);
    ArrayBlockingQueue<DyingQueryRunner> interrupted = new ArrayBlockingQueue<>(3);
    Set<DyingQueryRunner> runners = Sets.newHashSet(new DyingQueryRunner(queriesStarted, queriesInterrupted, interrupted), new DyingQueryRunner(queriesStarted, queriesInterrupted, interrupted), new DyingQueryRunner(queriesStarted, queriesInterrupted, interrupted));
    ChainedExecutionQueryRunner chainedRunner = new ChainedExecutionQueryRunner<>(exec, watcher, Lists.<QueryRunner<Integer>>newArrayList(runners));
    HashMap<String, Object> context = new HashMap<String, Object>();
    final Sequence seq = chainedRunner.run(Druids.newTimeseriesQueryBuilder().dataSource("test").intervals("2014/2015").aggregators(Lists.<AggregatorFactory>newArrayList(new CountAggregatorFactory("count"))).context(ImmutableMap.<String, Object>of(QueryContextKeys.TIMEOUT, 100, "queryId", "test")).build(), context);
    Future resultFuture = Executors.newFixedThreadPool(1).submit(new Runnable() {

        @Override
        public void run() {
            Sequences.toList(seq, Lists.newArrayList());
        }
    });
    // wait for query to register and start
    queryIsRegistered.await();
    queriesStarted.await();
    Assert.assertTrue(capturedFuture.hasCaptured());
    ListenableFuture future = capturedFuture.getValue();
    // wait for query to time out
    QueryInterruptedException cause = null;
    try {
        resultFuture.get();
    } catch (ExecutionException e) {
        Assert.assertTrue(e.getCause() instanceof QueryInterruptedException);
        Assert.assertEquals("Query timeout", ((QueryInterruptedException) e.getCause()).getErrorCode());
        cause = (QueryInterruptedException) e.getCause();
    }
    queriesInterrupted.await();
    Assert.assertNotNull(cause);
    Assert.assertTrue(future.isCancelled());
    DyingQueryRunner interrupted1 = interrupted.poll();
    synchronized (interrupted1) {
        Assert.assertTrue("runner 1 started", interrupted1.hasStarted);
        Assert.assertTrue("runner 1 interrupted", interrupted1.interrupted);
    }
    DyingQueryRunner interrupted2 = interrupted.poll();
    synchronized (interrupted2) {
        Assert.assertTrue("runner 2 started", interrupted2.hasStarted);
        Assert.assertTrue("runner 2 interrupted", interrupted2.interrupted);
    }
    runners.remove(interrupted1);
    runners.remove(interrupted2);
    DyingQueryRunner remainingRunner = runners.iterator().next();
    synchronized (remainingRunner) {
        Assert.assertTrue("runner 3 should be interrupted or not have started", !remainingRunner.hasStarted || remainingRunner.interrupted);
    }
    Assert.assertFalse("runner 1 not completed", interrupted1.hasCompleted);
    Assert.assertFalse("runner 2 not completed", interrupted2.hasCompleted);
    Assert.assertFalse("runner 3 not completed", remainingRunner.hasCompleted);
    EasyMock.verify(watcher);
}

Example 15 with ListenableFuture

use of com.google.common.util.concurrent.ListenableFuture in project hive by apache.

the class ExecutionPhase method execute.

@Override
public void execute() throws Throwable {
    long start = System.currentTimeMillis();
    List<TestBatch> testBatches = Lists.newArrayList();
    for (TestBatch batch : testBatchSupplier.get()) {
        testBatches.add(batch);
        if (batch.isParallel()) {
            parallelWorkQueue.add(batch);
        } else {
            isolatedWorkQueue.add(batch);
        }
    }
    logger.info("ParallelWorkQueueSize={}, IsolatedWorkQueueSize={}", parallelWorkQueue.size(), isolatedWorkQueue.size());
    if (logger.isDebugEnabled()) {
        for (TestBatch testBatch : parallelWorkQueue) {
            logger.debug("PBatch: {}", testBatch);
        }
        for (TestBatch testBatch : isolatedWorkQueue) {
            logger.debug("IBatch: {}", testBatch);
        }
    }
    try {
        int expectedNumHosts = hostExecutors.size();
        initalizeHosts();
        do {
            replaceBadHosts(expectedNumHosts);
            List<ListenableFuture<Void>> results = Lists.newArrayList();
            for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
                results.add(hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults));
            }
            Futures.allAsList(results).get();
        } while (!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty()));
        for (TestBatch batch : testBatches) {
            File batchLogDir;
            if (failedTestResults.contains(batch)) {
                batchLogDir = new File(failedLogDir, batch.getName());
            } else {
                batchLogDir = new File(succeededLogDir, batch.getName());
            }
            JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir);
            executedTests.addAll(parser.getAllExecutedTests());
            for (String failedTest : parser.getAllFailedTests()) {
                failedTests.add(failedTest + " (batchId=" + batch.getBatchId() + ")");
            }
            // if the TEST*.xml was not generated or was corrupt, let someone know
            if (parser.getTestClassesWithReportAvailable().size() < batch.getTestClasses().size()) {
                Set<String> expTestClasses = new HashSet<>(batch.getTestClasses());
                expTestClasses.removeAll(parser.getTestClassesWithReportAvailable());
                for (String testClass : expTestClasses) {
                    StringBuilder messageBuilder = new StringBuilder();
                    messageBuilder.append(testClass).append(" - did not produce a TEST-*.xml file (likely timed out)").append(" (batchId=").append(batch.getBatchId()).append(")");
                    if (batch instanceof QFileTestBatch) {
                        Collection<String> tests = ((QFileTestBatch) batch).getTests();
                        if (tests.size() != 0) {
                            messageBuilder.append("\n\t[");
                            messageBuilder.append(Joiner.on(",").join(tests));
                            messageBuilder.append("]");
                        }
                    }
                    failedTests.add(messageBuilder.toString());
                }
            }
        }
    } finally {
        long elapsed = System.currentTimeMillis() - start;
        logger.info("PERF: exec phase " + TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS) + " minutes");
    }
}