use of com.google.common.util.concurrent.ListenableFuture in project buck by facebook.
the class AdbHelper method adbCall.
/**
* Execute an {@link AdbCallable} for all matching devices. This functions performs device
* filtering based on three possible arguments:
*
* -e (emulator-only) - only emulators are passing the filter
* -d (device-only) - only real devices are passing the filter
* -s (serial) - only device/emulator with specific serial number are passing the filter
*
* If more than one device matches the filter this function will fail unless multi-install
* mode is enabled (-x). This flag is used as a marker that user understands that multiple
* devices will be used to install the apk if needed.
*/
@SuppressWarnings("PMD.EmptyCatchBlock")
@SuppressForbidden
public boolean adbCall(AdbCallable adbCallable, boolean quiet) throws InterruptedException {
List<IDevice> devices;
try (SimplePerfEvent.Scope ignored = SimplePerfEvent.scope(buckEventBus, "set_up_adb_call")) {
devices = getDevices(quiet);
if (devices.size() == 0) {
return false;
}
}
int adbThreadCount = options.getAdbThreadCount();
if (adbThreadCount <= 0) {
adbThreadCount = devices.size();
}
// Start executions on all matching devices.
List<ListenableFuture<Boolean>> futures = Lists.newArrayList();
ListeningExecutorService executorService = listeningDecorator(newMultiThreadExecutor(new CommandThreadFactory(getClass().getSimpleName()), adbThreadCount));
for (final IDevice device : devices) {
futures.add(executorService.submit(adbCallable.forDevice(device)));
}
// Wait for all executions to complete or fail.
List<Boolean> results = null;
try {
results = Futures.allAsList(futures).get();
} catch (ExecutionException ex) {
console.printBuildFailure("Failed: " + adbCallable);
ex.printStackTrace(console.getStdErr());
return false;
} catch (InterruptedException e) {
try {
Futures.allAsList(futures).cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw e;
} finally {
MostExecutors.shutdownOrThrow(executorService, 10, TimeUnit.MINUTES, new InterruptionFailedException("Failed to shutdown ExecutorService."));
}
int successCount = 0;
for (Boolean result : results) {
if (result) {
successCount++;
}
}
int failureCount = results.size() - successCount;
// Report results.
if (successCount > 0 && !quiet) {
console.printSuccess(String.format("Successfully ran %s on %d device(s)", adbCallable, successCount));
}
if (failureCount > 0) {
console.printBuildFailure(String.format("Failed to %s on %d device(s).", adbCallable, failureCount));
}
return failureCount == 0;
}
use of com.google.common.util.concurrent.ListenableFuture in project buck by facebook.
the class WorkspaceAndProjectGenerator method generateProject.
private void generateProject(final Map<Path, ProjectGenerator> projectGenerators, ListeningExecutorService listeningExecutorService, WorkspaceGenerator workspaceGenerator, ImmutableSet<BuildTarget> targetsInRequiredProjects, ImmutableMultimap.Builder<BuildTarget, PBXTarget> buildTargetToPbxTargetMapBuilder, ImmutableMap.Builder<PBXTarget, Path> targetToProjectPathMapBuilder, final Optional<BuildTarget> targetToBuildWithBuck) throws IOException, InterruptedException {
ImmutableMultimap.Builder<Cell, BuildTarget> projectCellToBuildTargetsBuilder = ImmutableMultimap.builder();
for (TargetNode<?, ?> targetNode : projectGraph.getNodes()) {
BuildTarget buildTarget = targetNode.getBuildTarget();
projectCellToBuildTargetsBuilder.put(rootCell.getCell(buildTarget), buildTarget);
}
ImmutableMultimap<Cell, BuildTarget> projectCellToBuildTargets = projectCellToBuildTargetsBuilder.build();
List<ListenableFuture<GenerationResult>> projectGeneratorFutures = new ArrayList<>();
for (final Cell projectCell : projectCellToBuildTargets.keySet()) {
ImmutableMultimap.Builder<Path, BuildTarget> projectDirectoryToBuildTargetsBuilder = ImmutableMultimap.builder();
final ImmutableSet<BuildTarget> cellRules = ImmutableSet.copyOf(projectCellToBuildTargets.get(projectCell));
for (BuildTarget buildTarget : cellRules) {
projectDirectoryToBuildTargetsBuilder.put(buildTarget.getBasePath(), buildTarget);
}
ImmutableMultimap<Path, BuildTarget> projectDirectoryToBuildTargets = projectDirectoryToBuildTargetsBuilder.build();
final Path relativeTargetCell = rootCell.getRoot().relativize(projectCell.getRoot());
for (final Path projectDirectory : projectDirectoryToBuildTargets.keySet()) {
final ImmutableSet<BuildTarget> rules = filterRulesForProjectDirectory(projectGraph, ImmutableSet.copyOf(projectDirectoryToBuildTargets.get(projectDirectory)));
if (Sets.intersection(targetsInRequiredProjects, rules).isEmpty()) {
continue;
}
final boolean isMainProject = workspaceArguments.srcTarget.isPresent() && rules.contains(workspaceArguments.srcTarget.get());
projectGeneratorFutures.add(listeningExecutorService.submit(() -> {
GenerationResult result = generateProjectForDirectory(projectGenerators, targetToBuildWithBuck, projectCell, projectDirectory, rules, isMainProject, targetsInRequiredProjects);
// convert the projectPath to relative to the target cell here
result = GenerationResult.of(relativeTargetCell.resolve(result.getProjectPath()), result.isProjectGenerated(), result.getRequiredBuildTargets(), result.getBuildTargetToGeneratedTargetMap());
return result;
}));
}
}
List<GenerationResult> generationResults;
try {
generationResults = Futures.allAsList(projectGeneratorFutures).get();
} catch (ExecutionException e) {
Throwables.throwIfInstanceOf(e.getCause(), IOException.class);
Throwables.throwIfUnchecked(e.getCause());
throw new IllegalStateException("Unexpected exception: ", e);
}
for (GenerationResult result : generationResults) {
if (!result.isProjectGenerated()) {
continue;
}
workspaceGenerator.addFilePath(result.getProjectPath());
processGenerationResult(buildTargetToPbxTargetMapBuilder, targetToProjectPathMapBuilder, result);
}
}
use of com.google.common.util.concurrent.ListenableFuture in project buck by facebook.
the class TestRunning method runTests.
@SuppressWarnings("PMD.EmptyCatchBlock")
public static int runTests(final CommandRunnerParams params, Iterable<TestRule> tests, ExecutionContext executionContext, final TestRunningOptions options, ListeningExecutorService service, BuildEngine buildEngine, final StepRunner stepRunner, SourcePathResolver sourcePathResolver, SourcePathRuleFinder ruleFinder) throws IOException, ExecutionException, InterruptedException {
ImmutableSet<JavaLibrary> rulesUnderTestForCoverage;
// If needed, we first run instrumentation on the class files.
if (options.isCodeCoverageEnabled()) {
rulesUnderTestForCoverage = getRulesUnderTest(tests);
if (!rulesUnderTestForCoverage.isEmpty()) {
try {
// We'll use the filesystem of the first rule under test. This will fail if there are any
// tests from a different repo, but it'll help us bootstrap ourselves to being able to
// support multiple repos
// TODO(t8220837): Support tests in multiple repos
JavaLibrary library = rulesUnderTestForCoverage.iterator().next();
stepRunner.runStepForBuildTarget(executionContext, new MakeCleanDirectoryStep(library.getProjectFilesystem(), JacocoConstants.getJacocoOutputDir(library.getProjectFilesystem())), Optional.empty());
} catch (StepFailedException e) {
params.getBuckEventBus().post(ConsoleEvent.severe(Throwables.getRootCause(e).getLocalizedMessage()));
return 1;
}
}
} else {
rulesUnderTestForCoverage = ImmutableSet.of();
}
final ImmutableSet<String> testTargets = FluentIterable.from(tests).transform(BuildRule::getBuildTarget).transform(Object::toString).toSet();
final int totalNumberOfTests = Iterables.size(tests);
params.getBuckEventBus().post(TestRunEvent.started(options.isRunAllTests(), options.getTestSelectorList(), options.shouldExplainTestSelectorList(), testTargets));
// Start running all of the tests. The result of each java_test() rule is represented as a
// ListenableFuture.
List<ListenableFuture<TestResults>> results = Lists.newArrayList();
TestRuleKeyFileHelper testRuleKeyFileHelper = new TestRuleKeyFileHelper(buildEngine);
final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger();
final List<TestRun> separateTestRuns = Lists.newArrayList();
List<TestRun> parallelTestRuns = Lists.newArrayList();
for (final TestRule test : tests) {
// Determine whether the test needs to be executed.
final Callable<TestResults> resultsInterpreter = getCachingCallable(test.interpretTestResults(executionContext, /*isUsingTestSelectors*/
!options.getTestSelectorList().isEmpty()));
boolean isTestRunRequired;
isTestRunRequired = isTestRunRequiredForTest(test, buildEngine, executionContext, testRuleKeyFileHelper, options.getTestResultCacheMode(), resultsInterpreter, !options.getTestSelectorList().isEmpty(), !options.getEnvironmentOverrides().isEmpty());
final Map<String, UUID> testUUIDMap = new HashMap<>();
final AtomicReference<TestStatusMessageEvent.Started> currentTestStatusMessageEvent = new AtomicReference<>();
TestRule.TestReportingCallback testReportingCallback = new TestRule.TestReportingCallback() {
@Override
public void testsDidBegin() {
LOG.debug("Tests for rule %s began", test.getBuildTarget());
}
@Override
public void statusDidBegin(TestStatusMessage didBeginMessage) {
LOG.debug("Test status did begin: %s", didBeginMessage);
TestStatusMessageEvent.Started startedEvent = TestStatusMessageEvent.started(didBeginMessage);
TestStatusMessageEvent.Started previousEvent = currentTestStatusMessageEvent.getAndSet(startedEvent);
Preconditions.checkState(previousEvent == null, "Received begin status before end status (%s)", previousEvent);
params.getBuckEventBus().post(startedEvent);
String message = didBeginMessage.getMessage();
if (message.toLowerCase().contains("debugger")) {
executionContext.getStdErr().println(executionContext.getAnsi().asWarningText(message));
}
}
@Override
public void statusDidEnd(TestStatusMessage didEndMessage) {
LOG.debug("Test status did end: %s", didEndMessage);
TestStatusMessageEvent.Started previousEvent = currentTestStatusMessageEvent.getAndSet(null);
Preconditions.checkState(previousEvent != null, "Received end status before begin status (%s)", previousEvent);
params.getBuckEventBus().post(TestStatusMessageEvent.finished(previousEvent, didEndMessage));
}
@Override
public void testDidBegin(String testCaseName, String testName) {
LOG.debug("Test rule %s test case %s test name %s began", test.getBuildTarget(), testCaseName, testName);
UUID testUUID = UUID.randomUUID();
// UUID is immutable and thread-safe as of Java 7, so it's
// safe to stash in a map and use later:
//
// http://bugs.java.com/view_bug.do?bug_id=6611830
testUUIDMap.put(testCaseName + ":" + testName, testUUID);
params.getBuckEventBus().post(TestSummaryEvent.started(testUUID, testCaseName, testName));
}
@Override
public void testDidEnd(TestResultSummary testResultSummary) {
LOG.debug("Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary);
UUID testUUID = testUUIDMap.get(testResultSummary.getTestCaseName() + ":" + testResultSummary.getTestName());
Preconditions.checkNotNull(testUUID);
params.getBuckEventBus().post(TestSummaryEvent.finished(testUUID, testResultSummary));
}
@Override
public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) {
LOG.debug("Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries);
}
};
List<Step> steps;
if (isTestRunRequired) {
params.getBuckEventBus().post(IndividualTestEvent.started(testTargets));
ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder();
Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget()));
List<Step> testSteps = test.runTests(executionContext, options, sourcePathResolver, testReportingCallback);
if (!testSteps.isEmpty()) {
stepsBuilder.addAll(testSteps);
stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test));
}
steps = stepsBuilder.build();
} else {
steps = ImmutableList.of();
}
TestRun testRun = TestRun.of(test, steps, getStatusTransformingCallable(isTestRunRequired, resultsInterpreter), testReportingCallback);
// commands because the rule is cached, but its results must still be processed.
if (test.runTestSeparately()) {
LOG.debug("Running test %s in serial", test);
separateTestRuns.add(testRun);
} else {
LOG.debug("Running test %s in parallel", test);
parallelTestRuns.add(testRun);
}
}
for (TestRun testRun : parallelTestRuns) {
ListenableFuture<TestResults> testResults = runStepsAndYieldResult(stepRunner, executionContext, testRun.getSteps(), testRun.getTestResultsCallable(), testRun.getTest().getBuildTarget(), params.getBuckEventBus(), service);
results.add(transformTestResults(params, testResults, testRun.getTest(), testRun.getTestReportingCallback(), testTargets, lastReportedTestSequenceNumber, totalNumberOfTests));
}
ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results);
final List<TestResults> completedResults = Lists.newArrayList();
final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService();
ListenableFuture<Void> uberFuture = MoreFutures.addListenableCallback(parallelTestStepsFuture, new FutureCallback<List<TestResults>>() {
@Override
public void onSuccess(List<TestResults> parallelTestResults) {
LOG.debug("Parallel tests completed, running separate tests...");
completedResults.addAll(parallelTestResults);
List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList();
for (TestRun testRun : separateTestRuns) {
separateResultsList.add(transformTestResults(params, runStepsAndYieldResult(stepRunner, executionContext, testRun.getSteps(), testRun.getTestResultsCallable(), testRun.getTest().getBuildTarget(), params.getBuckEventBus(), directExecutorService), testRun.getTest(), testRun.getTestReportingCallback(), testTargets, lastReportedTestSequenceNumber, totalNumberOfTests));
}
ListenableFuture<List<TestResults>> serialResults = Futures.allAsList(separateResultsList);
try {
completedResults.addAll(serialResults.get());
} catch (ExecutionException e) {
LOG.error(e, "Error fetching serial test results");
throw new HumanReadableException(e, "Error fetching serial test results");
} catch (InterruptedException e) {
LOG.error(e, "Interrupted fetching serial test results");
try {
serialResults.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw new HumanReadableException(e, "Test cancelled");
}
LOG.debug("Done running serial tests.");
}
@Override
public void onFailure(Throwable e) {
LOG.error(e, "Parallel tests failed, not running serial tests");
throw new HumanReadableException(e, "Parallel tests failed");
}
}, directExecutorService);
try {
// Block until all the tests have finished running.
uberFuture.get();
} catch (ExecutionException e) {
e.printStackTrace(params.getConsole().getStdErr());
return 1;
} catch (InterruptedException e) {
try {
uberFuture.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw e;
}
params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults));
// Write out the results as XML, if requested.
Optional<String> path = options.getPathToXmlTestOutput();
if (path.isPresent()) {
try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) {
writeXmlOutput(completedResults, writer);
}
}
// Generate the code coverage report.
if (options.isCodeCoverageEnabled() && !rulesUnderTestForCoverage.isEmpty()) {
try {
JavaBuckConfig javaBuckConfig = params.getBuckConfig().getView(JavaBuckConfig.class);
DefaultJavaPackageFinder defaultJavaPackageFinder = javaBuckConfig.createDefaultJavaPackageFinder();
stepRunner.runStepForBuildTarget(executionContext, getReportCommand(rulesUnderTestForCoverage, defaultJavaPackageFinder, javaBuckConfig.getDefaultJavaOptions().getJavaRuntimeLauncher(), params.getCell().getFilesystem(), sourcePathResolver, ruleFinder, JacocoConstants.getJacocoOutputDir(params.getCell().getFilesystem()), options.getCoverageReportFormat(), options.getCoverageReportTitle(), javaBuckConfig.getDefaultJavacOptions().getSpoolMode() == JavacOptions.SpoolMode.INTERMEDIATE_TO_DISK, options.getCoverageIncludes(), options.getCoverageExcludes()), Optional.empty());
} catch (StepFailedException e) {
params.getBuckEventBus().post(ConsoleEvent.severe(Throwables.getRootCause(e).getLocalizedMessage()));
return 1;
}
}
boolean failures = Iterables.any(completedResults, results1 -> {
LOG.debug("Checking result %s for failure", results1);
return !results1.isSuccess();
});
return failures ? TEST_FAILURES_EXIT_CODE : 0;
}
use of com.google.common.util.concurrent.ListenableFuture in project druid by druid-io.
the class KafkaIndexTaskClientTest method testGetStartTimeAsync.
@Test
public void testGetStartTimeAsync() throws Exception {
final DateTime now = DateTime.now();
final int numRequests = TEST_IDS.size();
Capture<Request> captured = Capture.newInstance(CaptureType.ALL);
expect(responseHolder.getStatus()).andReturn(HttpResponseStatus.OK).anyTimes();
expect(responseHolder.getContent()).andReturn(String.valueOf(now.getMillis())).anyTimes();
expect(httpClient.go(capture(captured), anyObject(FullResponseHandler.class), eq(TEST_HTTP_TIMEOUT))).andReturn(Futures.immediateFuture(responseHolder)).times(numRequests);
replayAll();
List<URL> expectedUrls = Lists.newArrayList();
List<ListenableFuture<DateTime>> futures = Lists.newArrayList();
for (int i = 0; i < numRequests; i++) {
expectedUrls.add(new URL(String.format(URL_FORMATTER, TEST_HOST, TEST_PORT, TEST_IDS.get(i), "time/start")));
futures.add(client.getStartTimeAsync(TEST_IDS.get(i)));
}
List<DateTime> responses = Futures.allAsList(futures).get();
verifyAll();
List<Request> requests = captured.getValues();
Assert.assertEquals(numRequests, requests.size());
Assert.assertEquals(numRequests, responses.size());
for (int i = 0; i < numRequests; i++) {
Assert.assertEquals(HttpMethod.GET, requests.get(i).getMethod());
Assert.assertTrue("unexpectedURL", expectedUrls.contains(requests.get(i).getUrl()));
Assert.assertEquals(now, responses.get(i));
}
}
use of com.google.common.util.concurrent.ListenableFuture in project druid by druid-io.
the class KafkaIndexTaskClientTest method testPauseAsync.
@Test
public void testPauseAsync() throws Exception {
final int numRequests = TEST_IDS.size();
Capture<Request> captured = Capture.newInstance(CaptureType.ALL);
expect(responseHolder.getStatus()).andReturn(HttpResponseStatus.OK).anyTimes();
expect(responseHolder.getContent()).andReturn("{\"0\":\"1\"}").anyTimes();
expect(httpClient.go(capture(captured), anyObject(FullResponseHandler.class), eq(TEST_HTTP_TIMEOUT))).andReturn(Futures.immediateFuture(responseHolder)).times(numRequests);
replayAll();
List<URL> expectedUrls = Lists.newArrayList();
List<ListenableFuture<Map<Integer, Long>>> futures = Lists.newArrayList();
for (int i = 0; i < numRequests; i++) {
expectedUrls.add(new URL(String.format(URL_FORMATTER, TEST_HOST, TEST_PORT, TEST_IDS.get(i), "pause")));
futures.add(client.pauseAsync(TEST_IDS.get(i)));
}
List<Map<Integer, Long>> responses = Futures.allAsList(futures).get();
verifyAll();
List<Request> requests = captured.getValues();
Assert.assertEquals(numRequests, requests.size());
Assert.assertEquals(numRequests, responses.size());
for (int i = 0; i < numRequests; i++) {
Assert.assertEquals(HttpMethod.POST, requests.get(i).getMethod());
Assert.assertTrue("unexpectedURL", expectedUrls.contains(requests.get(i).getUrl()));
Assert.assertEquals(Maps.newLinkedHashMap(ImmutableMap.of(0, 1L)), responses.get(i));
}
}
Aggregations