Examples with ListenableFuture com.google.common.util.concurrent.ListenableFuture used on opensource projects

Example 1 with ListenableFuture

use of com.google.common.util.concurrent.ListenableFuture in project buck by facebook.

the class AdbHelper method adbCall.

/**
   * Execute an {@link AdbCallable} for all matching devices. This functions performs device
   * filtering based on three possible arguments:
   *
   *  -e (emulator-only) - only emulators are passing the filter
   *  -d (device-only) - only real devices are passing the filter
   *  -s (serial) - only device/emulator with specific serial number are passing the filter
   *
   *  If more than one device matches the filter this function will fail unless multi-install
   *  mode is enabled (-x). This flag is used as a marker that user understands that multiple
   *  devices will be used to install the apk if needed.
   */
@SuppressWarnings("PMD.EmptyCatchBlock")
@SuppressForbidden
public boolean adbCall(AdbCallable adbCallable, boolean quiet) throws InterruptedException {
    List<IDevice> devices;
    try (SimplePerfEvent.Scope ignored = SimplePerfEvent.scope(buckEventBus, "set_up_adb_call")) {
        devices = getDevices(quiet);
        if (devices.size() == 0) {
            return false;
        }
    }
    int adbThreadCount = options.getAdbThreadCount();
    if (adbThreadCount <= 0) {
        adbThreadCount = devices.size();
    }
    // Start executions on all matching devices.
    List<ListenableFuture<Boolean>> futures = Lists.newArrayList();
    ListeningExecutorService executorService = listeningDecorator(newMultiThreadExecutor(new CommandThreadFactory(getClass().getSimpleName()), adbThreadCount));
    for (final IDevice device : devices) {
        futures.add(executorService.submit(adbCallable.forDevice(device)));
    }
    // Wait for all executions to complete or fail.
    List<Boolean> results = null;
    try {
        results = Futures.allAsList(futures).get();
    } catch (ExecutionException ex) {
        console.printBuildFailure("Failed: " + adbCallable);
        ex.printStackTrace(console.getStdErr());
        return false;
    } catch (InterruptedException e) {
        try {
            Futures.allAsList(futures).cancel(true);
        } catch (CancellationException ignored) {
        // Rethrow original InterruptedException instead.
        }
        Thread.currentThread().interrupt();
        throw e;
    } finally {
        MostExecutors.shutdownOrThrow(executorService, 10, TimeUnit.MINUTES, new InterruptionFailedException("Failed to shutdown ExecutorService."));
    }
    int successCount = 0;
    for (Boolean result : results) {
        if (result) {
            successCount++;
        }
    }
    int failureCount = results.size() - successCount;
    // Report results.
    if (successCount > 0 && !quiet) {
        console.printSuccess(String.format("Successfully ran %s on %d device(s)", adbCallable, successCount));
    }
    if (failureCount > 0) {
        console.printBuildFailure(String.format("Failed to %s on %d device(s).", adbCallable, failureCount));
    }
    return failureCount == 0;
}

Example 2 with ListenableFuture

use of com.google.common.util.concurrent.ListenableFuture in project buck by facebook.

the class WorkspaceAndProjectGenerator method generateProject.

private void generateProject(final Map<Path, ProjectGenerator> projectGenerators, ListeningExecutorService listeningExecutorService, WorkspaceGenerator workspaceGenerator, ImmutableSet<BuildTarget> targetsInRequiredProjects, ImmutableMultimap.Builder<BuildTarget, PBXTarget> buildTargetToPbxTargetMapBuilder, ImmutableMap.Builder<PBXTarget, Path> targetToProjectPathMapBuilder, final Optional<BuildTarget> targetToBuildWithBuck) throws IOException, InterruptedException {
    ImmutableMultimap.Builder<Cell, BuildTarget> projectCellToBuildTargetsBuilder = ImmutableMultimap.builder();
    for (TargetNode<?, ?> targetNode : projectGraph.getNodes()) {
        BuildTarget buildTarget = targetNode.getBuildTarget();
        projectCellToBuildTargetsBuilder.put(rootCell.getCell(buildTarget), buildTarget);
    }
    ImmutableMultimap<Cell, BuildTarget> projectCellToBuildTargets = projectCellToBuildTargetsBuilder.build();
    List<ListenableFuture<GenerationResult>> projectGeneratorFutures = new ArrayList<>();
    for (final Cell projectCell : projectCellToBuildTargets.keySet()) {
        ImmutableMultimap.Builder<Path, BuildTarget> projectDirectoryToBuildTargetsBuilder = ImmutableMultimap.builder();
        final ImmutableSet<BuildTarget> cellRules = ImmutableSet.copyOf(projectCellToBuildTargets.get(projectCell));
        for (BuildTarget buildTarget : cellRules) {
            projectDirectoryToBuildTargetsBuilder.put(buildTarget.getBasePath(), buildTarget);
        }
        ImmutableMultimap<Path, BuildTarget> projectDirectoryToBuildTargets = projectDirectoryToBuildTargetsBuilder.build();
        final Path relativeTargetCell = rootCell.getRoot().relativize(projectCell.getRoot());
        for (final Path projectDirectory : projectDirectoryToBuildTargets.keySet()) {
            final ImmutableSet<BuildTarget> rules = filterRulesForProjectDirectory(projectGraph, ImmutableSet.copyOf(projectDirectoryToBuildTargets.get(projectDirectory)));
            if (Sets.intersection(targetsInRequiredProjects, rules).isEmpty()) {
                continue;
            }
            final boolean isMainProject = workspaceArguments.srcTarget.isPresent() && rules.contains(workspaceArguments.srcTarget.get());
            projectGeneratorFutures.add(listeningExecutorService.submit(() -> {
                GenerationResult result = generateProjectForDirectory(projectGenerators, targetToBuildWithBuck, projectCell, projectDirectory, rules, isMainProject, targetsInRequiredProjects);
                // convert the projectPath to relative to the target cell here
                result = GenerationResult.of(relativeTargetCell.resolve(result.getProjectPath()), result.isProjectGenerated(), result.getRequiredBuildTargets(), result.getBuildTargetToGeneratedTargetMap());
                return result;
            }));
        }
    }
    List<GenerationResult> generationResults;
    try {
        generationResults = Futures.allAsList(projectGeneratorFutures).get();
    } catch (ExecutionException e) {
        Throwables.throwIfInstanceOf(e.getCause(), IOException.class);
        Throwables.throwIfUnchecked(e.getCause());
        throw new IllegalStateException("Unexpected exception: ", e);
    }
    for (GenerationResult result : generationResults) {
        if (!result.isProjectGenerated()) {
            continue;
        }
        workspaceGenerator.addFilePath(result.getProjectPath());
        processGenerationResult(buildTargetToPbxTargetMapBuilder, targetToProjectPathMapBuilder, result);
    }
}

Example 3 with ListenableFuture

use of com.google.common.util.concurrent.ListenableFuture in project buck by facebook.

the class TestRunning method runTests.

@SuppressWarnings("PMD.EmptyCatchBlock")
public static int runTests(final CommandRunnerParams params, Iterable<TestRule> tests, ExecutionContext executionContext, final TestRunningOptions options, ListeningExecutorService service, BuildEngine buildEngine, final StepRunner stepRunner, SourcePathResolver sourcePathResolver, SourcePathRuleFinder ruleFinder) throws IOException, ExecutionException, InterruptedException {
    ImmutableSet<JavaLibrary> rulesUnderTestForCoverage;
    // If needed, we first run instrumentation on the class files.
    if (options.isCodeCoverageEnabled()) {
        rulesUnderTestForCoverage = getRulesUnderTest(tests);
        if (!rulesUnderTestForCoverage.isEmpty()) {
            try {
                // We'll use the filesystem of the first rule under test. This will fail if there are any
                // tests from a different repo, but it'll help us bootstrap ourselves to being able to
                // support multiple repos
                // TODO(t8220837): Support tests in multiple repos
                JavaLibrary library = rulesUnderTestForCoverage.iterator().next();
                stepRunner.runStepForBuildTarget(executionContext, new MakeCleanDirectoryStep(library.getProjectFilesystem(), JacocoConstants.getJacocoOutputDir(library.getProjectFilesystem())), Optional.empty());
            } catch (StepFailedException e) {
                params.getBuckEventBus().post(ConsoleEvent.severe(Throwables.getRootCause(e).getLocalizedMessage()));
                return 1;
            }
        }
    } else {
        rulesUnderTestForCoverage = ImmutableSet.of();
    }
    final ImmutableSet<String> testTargets = FluentIterable.from(tests).transform(BuildRule::getBuildTarget).transform(Object::toString).toSet();
    final int totalNumberOfTests = Iterables.size(tests);
    params.getBuckEventBus().post(TestRunEvent.started(options.isRunAllTests(), options.getTestSelectorList(), options.shouldExplainTestSelectorList(), testTargets));
    // Start running all of the tests. The result of each java_test() rule is represented as a
    // ListenableFuture.
    List<ListenableFuture<TestResults>> results = Lists.newArrayList();
    TestRuleKeyFileHelper testRuleKeyFileHelper = new TestRuleKeyFileHelper(buildEngine);
    final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger();
    final List<TestRun> separateTestRuns = Lists.newArrayList();
    List<TestRun> parallelTestRuns = Lists.newArrayList();
    for (final TestRule test : tests) {
        // Determine whether the test needs to be executed.
        final Callable<TestResults> resultsInterpreter = getCachingCallable(test.interpretTestResults(executionContext, /*isUsingTestSelectors*/
        !options.getTestSelectorList().isEmpty()));
        boolean isTestRunRequired;
        isTestRunRequired = isTestRunRequiredForTest(test, buildEngine, executionContext, testRuleKeyFileHelper, options.getTestResultCacheMode(), resultsInterpreter, !options.getTestSelectorList().isEmpty(), !options.getEnvironmentOverrides().isEmpty());
        final Map<String, UUID> testUUIDMap = new HashMap<>();
        final AtomicReference<TestStatusMessageEvent.Started> currentTestStatusMessageEvent = new AtomicReference<>();
        TestRule.TestReportingCallback testReportingCallback = new TestRule.TestReportingCallback() {

            @Override
            public void testsDidBegin() {
                LOG.debug("Tests for rule %s began", test.getBuildTarget());
            }

            @Override
            public void statusDidBegin(TestStatusMessage didBeginMessage) {
                LOG.debug("Test status did begin: %s", didBeginMessage);
                TestStatusMessageEvent.Started startedEvent = TestStatusMessageEvent.started(didBeginMessage);
                TestStatusMessageEvent.Started previousEvent = currentTestStatusMessageEvent.getAndSet(startedEvent);
                Preconditions.checkState(previousEvent == null, "Received begin status before end status (%s)", previousEvent);
                params.getBuckEventBus().post(startedEvent);
                String message = didBeginMessage.getMessage();
                if (message.toLowerCase().contains("debugger")) {
                    executionContext.getStdErr().println(executionContext.getAnsi().asWarningText(message));
                }
            }

            @Override
            public void statusDidEnd(TestStatusMessage didEndMessage) {
                LOG.debug("Test status did end: %s", didEndMessage);
                TestStatusMessageEvent.Started previousEvent = currentTestStatusMessageEvent.getAndSet(null);
                Preconditions.checkState(previousEvent != null, "Received end status before begin status (%s)", previousEvent);
                params.getBuckEventBus().post(TestStatusMessageEvent.finished(previousEvent, didEndMessage));
            }

            @Override
            public void testDidBegin(String testCaseName, String testName) {
                LOG.debug("Test rule %s test case %s test name %s began", test.getBuildTarget(), testCaseName, testName);
                UUID testUUID = UUID.randomUUID();
                // UUID is immutable and thread-safe as of Java 7, so it's
                // safe to stash in a map and use later:
                //
                // http://bugs.java.com/view_bug.do?bug_id=6611830
                testUUIDMap.put(testCaseName + ":" + testName, testUUID);
                params.getBuckEventBus().post(TestSummaryEvent.started(testUUID, testCaseName, testName));
            }

            @Override
            public void testDidEnd(TestResultSummary testResultSummary) {
                LOG.debug("Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary);
                UUID testUUID = testUUIDMap.get(testResultSummary.getTestCaseName() + ":" + testResultSummary.getTestName());
                Preconditions.checkNotNull(testUUID);
                params.getBuckEventBus().post(TestSummaryEvent.finished(testUUID, testResultSummary));
            }

            @Override
            public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) {
                LOG.debug("Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries);
            }
        };
        List<Step> steps;
        if (isTestRunRequired) {
            params.getBuckEventBus().post(IndividualTestEvent.started(testTargets));
            ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder();
            Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget()));
            List<Step> testSteps = test.runTests(executionContext, options, sourcePathResolver, testReportingCallback);
            if (!testSteps.isEmpty()) {
                stepsBuilder.addAll(testSteps);
                stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test));
            }
            steps = stepsBuilder.build();
        } else {
            steps = ImmutableList.of();
        }
        TestRun testRun = TestRun.of(test, steps, getStatusTransformingCallable(isTestRunRequired, resultsInterpreter), testReportingCallback);
        // commands because the rule is cached, but its results must still be processed.
        if (test.runTestSeparately()) {
            LOG.debug("Running test %s in serial", test);
            separateTestRuns.add(testRun);
        } else {
            LOG.debug("Running test %s in parallel", test);
            parallelTestRuns.add(testRun);
        }
    }
    for (TestRun testRun : parallelTestRuns) {
        ListenableFuture<TestResults> testResults = runStepsAndYieldResult(stepRunner, executionContext, testRun.getSteps(), testRun.getTestResultsCallable(), testRun.getTest().getBuildTarget(), params.getBuckEventBus(), service);
        results.add(transformTestResults(params, testResults, testRun.getTest(), testRun.getTestReportingCallback(), testTargets, lastReportedTestSequenceNumber, totalNumberOfTests));
    }
    ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results);
    final List<TestResults> completedResults = Lists.newArrayList();
    final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService();
    ListenableFuture<Void> uberFuture = MoreFutures.addListenableCallback(parallelTestStepsFuture, new FutureCallback<List<TestResults>>() {

        @Override
        public void onSuccess(List<TestResults> parallelTestResults) {
            LOG.debug("Parallel tests completed, running separate tests...");
            completedResults.addAll(parallelTestResults);
            List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList();
            for (TestRun testRun : separateTestRuns) {
                separateResultsList.add(transformTestResults(params, runStepsAndYieldResult(stepRunner, executionContext, testRun.getSteps(), testRun.getTestResultsCallable(), testRun.getTest().getBuildTarget(), params.getBuckEventBus(), directExecutorService), testRun.getTest(), testRun.getTestReportingCallback(), testTargets, lastReportedTestSequenceNumber, totalNumberOfTests));
            }
            ListenableFuture<List<TestResults>> serialResults = Futures.allAsList(separateResultsList);
            try {
                completedResults.addAll(serialResults.get());
            } catch (ExecutionException e) {
                LOG.error(e, "Error fetching serial test results");
                throw new HumanReadableException(e, "Error fetching serial test results");
            } catch (InterruptedException e) {
                LOG.error(e, "Interrupted fetching serial test results");
                try {
                    serialResults.cancel(true);
                } catch (CancellationException ignored) {
                // Rethrow original InterruptedException instead.
                }
                Thread.currentThread().interrupt();
                throw new HumanReadableException(e, "Test cancelled");
            }
            LOG.debug("Done running serial tests.");
        }

        @Override
        public void onFailure(Throwable e) {
            LOG.error(e, "Parallel tests failed, not running serial tests");
            throw new HumanReadableException(e, "Parallel tests failed");
        }
    }, directExecutorService);
    try {
        // Block until all the tests have finished running.
        uberFuture.get();
    } catch (ExecutionException e) {
        e.printStackTrace(params.getConsole().getStdErr());
        return 1;
    } catch (InterruptedException e) {
        try {
            uberFuture.cancel(true);
        } catch (CancellationException ignored) {
        // Rethrow original InterruptedException instead.
        }
        Thread.currentThread().interrupt();
        throw e;
    }
    params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults));
    // Write out the results as XML, if requested.
    Optional<String> path = options.getPathToXmlTestOutput();
    if (path.isPresent()) {
        try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) {
            writeXmlOutput(completedResults, writer);
        }
    }
    // Generate the code coverage report.
    if (options.isCodeCoverageEnabled() && !rulesUnderTestForCoverage.isEmpty()) {
        try {
            JavaBuckConfig javaBuckConfig = params.getBuckConfig().getView(JavaBuckConfig.class);
            DefaultJavaPackageFinder defaultJavaPackageFinder = javaBuckConfig.createDefaultJavaPackageFinder();
            stepRunner.runStepForBuildTarget(executionContext, getReportCommand(rulesUnderTestForCoverage, defaultJavaPackageFinder, javaBuckConfig.getDefaultJavaOptions().getJavaRuntimeLauncher(), params.getCell().getFilesystem(), sourcePathResolver, ruleFinder, JacocoConstants.getJacocoOutputDir(params.getCell().getFilesystem()), options.getCoverageReportFormat(), options.getCoverageReportTitle(), javaBuckConfig.getDefaultJavacOptions().getSpoolMode() == JavacOptions.SpoolMode.INTERMEDIATE_TO_DISK, options.getCoverageIncludes(), options.getCoverageExcludes()), Optional.empty());
        } catch (StepFailedException e) {
            params.getBuckEventBus().post(ConsoleEvent.severe(Throwables.getRootCause(e).getLocalizedMessage()));
            return 1;
        }
    }
    boolean failures = Iterables.any(completedResults, results1 -> {
        LOG.debug("Checking result %s for failure", results1);
        return !results1.isSuccess();
    });
    return failures ? TEST_FAILURES_EXIT_CODE : 0;
}

Example 4 with ListenableFuture

use of com.google.common.util.concurrent.ListenableFuture in project druid by druid-io.

the class KafkaIndexTaskClientTest method testGetStartTimeAsync.

@Test
public void testGetStartTimeAsync() throws Exception {
    final DateTime now = DateTime.now();
    final int numRequests = TEST_IDS.size();
    Capture<Request> captured = Capture.newInstance(CaptureType.ALL);
    expect(responseHolder.getStatus()).andReturn(HttpResponseStatus.OK).anyTimes();
    expect(responseHolder.getContent()).andReturn(String.valueOf(now.getMillis())).anyTimes();
    expect(httpClient.go(capture(captured), anyObject(FullResponseHandler.class), eq(TEST_HTTP_TIMEOUT))).andReturn(Futures.immediateFuture(responseHolder)).times(numRequests);
    replayAll();
    List<URL> expectedUrls = Lists.newArrayList();
    List<ListenableFuture<DateTime>> futures = Lists.newArrayList();
    for (int i = 0; i < numRequests; i++) {
        expectedUrls.add(new URL(String.format(URL_FORMATTER, TEST_HOST, TEST_PORT, TEST_IDS.get(i), "time/start")));
        futures.add(client.getStartTimeAsync(TEST_IDS.get(i)));
    }
    List<DateTime> responses = Futures.allAsList(futures).get();
    verifyAll();
    List<Request> requests = captured.getValues();
    Assert.assertEquals(numRequests, requests.size());
    Assert.assertEquals(numRequests, responses.size());
    for (int i = 0; i < numRequests; i++) {
        Assert.assertEquals(HttpMethod.GET, requests.get(i).getMethod());
        Assert.assertTrue("unexpectedURL", expectedUrls.contains(requests.get(i).getUrl()));
        Assert.assertEquals(now, responses.get(i));
    }
}

Example 5 with ListenableFuture

use of com.google.common.util.concurrent.ListenableFuture in project druid by druid-io.

the class KafkaIndexTaskClientTest method testPauseAsync.

@Test
public void testPauseAsync() throws Exception {
    final int numRequests = TEST_IDS.size();
    Capture<Request> captured = Capture.newInstance(CaptureType.ALL);
    expect(responseHolder.getStatus()).andReturn(HttpResponseStatus.OK).anyTimes();
    expect(responseHolder.getContent()).andReturn("{\"0\":\"1\"}").anyTimes();
    expect(httpClient.go(capture(captured), anyObject(FullResponseHandler.class), eq(TEST_HTTP_TIMEOUT))).andReturn(Futures.immediateFuture(responseHolder)).times(numRequests);
    replayAll();
    List<URL> expectedUrls = Lists.newArrayList();
    List<ListenableFuture<Map<Integer, Long>>> futures = Lists.newArrayList();
    for (int i = 0; i < numRequests; i++) {
        expectedUrls.add(new URL(String.format(URL_FORMATTER, TEST_HOST, TEST_PORT, TEST_IDS.get(i), "pause")));
        futures.add(client.pauseAsync(TEST_IDS.get(i)));
    }
    List<Map<Integer, Long>> responses = Futures.allAsList(futures).get();
    verifyAll();
    List<Request> requests = captured.getValues();
    Assert.assertEquals(numRequests, requests.size());
    Assert.assertEquals(numRequests, responses.size());
    for (int i = 0; i < numRequests; i++) {
        Assert.assertEquals(HttpMethod.POST, requests.get(i).getMethod());
        Assert.assertTrue("unexpectedURL", expectedUrls.contains(requests.get(i).getUrl()));
        Assert.assertEquals(Maps.newLinkedHashMap(ImmutableMap.of(0, 1L)), responses.get(i));
    }
}