Examples with TestResultSummary com.facebook.buck.test.TestResultSummary used on opensource projects

Example 21 with TestResultSummary

use of com.facebook.buck.test.TestResultSummary in project buck by facebook.

the class TestRunning method transformTestResults.

private static ListenableFuture<TestResults> transformTestResults(final CommandRunnerParams params, ListenableFuture<TestResults> originalTestResults, final TestRule testRule, final TestRule.TestReportingCallback testReportingCallback, final ImmutableSet<String> testTargets, final AtomicInteger lastReportedTestSequenceNumber, final int totalNumberOfTests) {
    final SettableFuture<TestResults> transformedTestResults = SettableFuture.create();
    FutureCallback<TestResults> callback = new FutureCallback<TestResults>() {

        private TestResults postTestResults(TestResults testResults) {
            if (!testRule.supportsStreamingTests()) {
                // For test rules which don't support streaming tests, we'll
                // stream test summary events after interpreting the
                // results.
                LOG.debug("Simulating streaming test events for rule %s", testRule);
                testReportingCallback.testsDidBegin();
                for (TestCaseSummary testCaseSummary : testResults.getTestCases()) {
                    for (TestResultSummary testResultSummary : testCaseSummary.getTestResults()) {
                        testReportingCallback.testDidBegin(testResultSummary.getTestCaseName(), testResultSummary.getTestName());
                        testReportingCallback.testDidEnd(testResultSummary);
                    }
                }
                testReportingCallback.testsDidEnd(testResults.getTestCases());
                LOG.debug("Done simulating streaming test events for rule %s", testRule);
            }
            TestResults transformedTestResults = TestResults.builder().from(testResults).setSequenceNumber(lastReportedTestSequenceNumber.incrementAndGet()).setTotalNumberOfTests(totalNumberOfTests).build();
            params.getBuckEventBus().post(IndividualTestEvent.finished(testTargets, transformedTestResults));
            return transformedTestResults;
        }

        private String getStackTrace(Throwable throwable) {
            StringWriter sw = new StringWriter();
            PrintWriter pw = new PrintWriter(sw);
            throwable.printStackTrace(pw);
            return sw.toString();
        }

        @Override
        public void onSuccess(TestResults testResults) {
            LOG.debug("Transforming successful test results %s", testResults);
            postTestResults(testResults);
            transformedTestResults.set(testResults);
        }

        @Override
        public void onFailure(Throwable throwable) {
            LOG.warn(throwable, "Test command step failed, marking %s as failed", testRule);
            // If the test command steps themselves fail, report this as special test result.
            TestResults testResults = TestResults.of(testRule.getBuildTarget(), ImmutableList.of(new TestCaseSummary(testRule.getBuildTarget().toString(), ImmutableList.of(new TestResultSummary(testRule.getBuildTarget().toString(), "main", ResultType.FAILURE, 0L, throwable.getMessage(), getStackTrace(throwable), "", "")))), testRule.getContacts(), testRule.getLabels().stream().map(Object::toString).collect(MoreCollectors.toImmutableSet()));
            TestResults newTestResults = postTestResults(testResults);
            transformedTestResults.set(newTestResults);
        }
    };
    Futures.addCallback(originalTestResults, callback);
    return transformedTestResults;
}

Example 22 with TestResultSummary

use of com.facebook.buck.test.TestResultSummary in project buck by facebook.

the class CxxGtestTest method parseResults.

@Override
protected ImmutableList<TestResultSummary> parseResults(Path exitCode, Path output, Path results) throws IOException, SAXException {
    // Try to parse the results file first, which should be written if the test suite exited
    // normally (even in the event of a failing test).  If this fails, just construct a test
    // summary with the output we have.
    Document doc;
    try {
        doc = XmlDomParser.parse(getProjectFilesystem().resolve(results));
    } catch (SAXException e) {
        return ImmutableList.of(getProgramFailureSummary("test program aborted before finishing", getProjectFilesystem().readFileIfItExists(output).orElse("")));
    }
    ImmutableList.Builder<TestResultSummary> summariesBuilder = ImmutableList.builder();
    // It's possible the test output had invalid characters in it's output, so make sure to
    // ignore these as we parse the output lines.
    Optional<String> currentTest = Optional.empty();
    Map<String, ChunkAccumulator> stdout = Maps.newHashMap();
    CharsetDecoder decoder = Charsets.UTF_8.newDecoder();
    decoder.onMalformedInput(CodingErrorAction.IGNORE);
    try (InputStream input = getProjectFilesystem().newFileInputStream(output);
        BufferedReader reader = new BufferedReader(new InputStreamReader(input, decoder))) {
        String line;
        while ((line = reader.readLine()) != null) {
            Matcher matcher;
            if ((matcher = START.matcher(line.trim())).matches()) {
                String test = matcher.group(1);
                currentTest = Optional.of(test);
                stdout.put(test, new ChunkAccumulator(Charsets.UTF_8, maxTestOutputSize));
            } else if (END.matcher(line.trim()).matches()) {
                currentTest = Optional.empty();
            } else if (currentTest.isPresent()) {
                Preconditions.checkNotNull(stdout.get(currentTest.get())).append(line);
            }
        }
    }
    NodeList testcases = doc.getElementsByTagName("testcase");
    for (int index = 0; index < testcases.getLength(); index++) {
        Node testcase = testcases.item(index);
        NamedNodeMap attributes = testcase.getAttributes();
        String testCase = attributes.getNamedItem("classname").getNodeValue();
        String testName = attributes.getNamedItem("name").getNodeValue();
        String testFull = String.format("%s.%s", testCase, testName);
        Double time = Double.parseDouble(attributes.getNamedItem("time").getNodeValue()) * 1000;
        // Prepare the result message and type
        ResultType type = ResultType.SUCCESS;
        String message = "";
        if (testcase.getChildNodes().getLength() > 0) {
            Node failure = testcase.getChildNodes().item(1);
            type = ResultType.FAILURE;
            message = failure.getAttributes().getNamedItem("message").getNodeValue();
        } else if (attributes.getNamedItem("status").getNodeValue().equals(NOTRUN)) {
            type = ResultType.ASSUMPTION_VIOLATION;
            message = "DISABLED";
        }
        // Prepare the tests stdout.
        String testStdout = "";
        if (stdout.containsKey(testFull)) {
            testStdout = Joiner.on(System.lineSeparator()).join(stdout.get(testFull).getChunks());
        }
        summariesBuilder.add(new TestResultSummary(testCase, testName, type, time.longValue(), message, "", testStdout, ""));
    }
    return summariesBuilder.build();
}

Example 23 with TestResultSummary

use of com.facebook.buck.test.TestResultSummary in project buck by facebook.

the class CxxBoostTest method visitTestSuite.

private void visitTestSuite(ImmutableList.Builder<TestResultSummary> builder, Map<String, String> messages, Map<String, List<String>> stdout, Map<String, Long> times, String prefix, Node testSuite) {
    NamedNodeMap attributes = testSuite.getAttributes();
    String suiteName = attributes.getNamedItem("name").getNodeValue();
    if (!prefix.isEmpty()) {
        suiteName = prefix + "." + suiteName;
    }
    NodeList testCases = testSuite.getChildNodes();
    for (int index = 0; index < testCases.getLength(); index++) {
        Node testCase = testCases.item(index);
        if (!testCase.getNodeName().equals("TestCase")) {
            visitTestSuite(builder, messages, stdout, times, suiteName, testCase);
            continue;
        }
        NamedNodeMap attrs = testCase.getAttributes();
        String caseName = attrs.getNamedItem("name").getNodeValue();
        String test = String.format("%s.%s", suiteName, caseName);
        Long time = Optional.ofNullable(times.get(test)).orElse(0L);
        String resultString = attrs.getNamedItem("result").getNodeValue();
        ResultType result = ResultType.SUCCESS;
        String output = "";
        String message = "";
        if (!"passed".equals(resultString)) {
            result = ResultType.FAILURE;
            message = messages.get(test);
            output = Joiner.on("\n").join(stdout.get(test));
        }
        builder.add(new TestResultSummary(suiteName, caseName, result, time, message, /* stacktrace */
        "", /* stdOut */
        output, /* stdErr */
        ""));
    }
}

Example 24 with TestResultSummary

use of com.facebook.buck.test.TestResultSummary in project buck by facebook.

the class CxxGtestTestTest method testParseResults.

@Test
public void testParseResults() throws Exception {
    ProjectWorkspace workspace = TestDataHelper.createProjectWorkspaceForScenario(this, "gtest", tmp);
    workspace.setUp();
    ImmutableList<String> samples = ImmutableList.of("big_output", "malformed_output", "malformed_results", "multisuite_success", "no_tests", "simple_success", "simple_failure", "simple_failure_with_output", "simple_disabled");
    BuildTarget target = BuildTargetFactory.newInstance("//:test");
    ProjectFilesystem filesystem = new ProjectFilesystem(tmp.getRoot());
    BuildRuleResolver ruleResolver = new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer());
    SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(ruleResolver);
    CxxGtestTest test = new CxxGtestTest(new FakeBuildRuleParamsBuilder(target).setProjectFilesystem(filesystem).build(), ruleFinder, new CxxLink(new FakeBuildRuleParamsBuilder(BuildTargetFactory.newInstance("//:link")).build(), CxxPlatformUtils.DEFAULT_PLATFORM.getLd().resolve(ruleResolver), Paths.get("output"), ImmutableList.of(), Optional.empty(), /* cacheable */
    true), new CommandTool.Builder().addArg(StringArg.of("")).build(), ImmutableMap.of(), Suppliers.ofInstance(ImmutableList.of()), ImmutableSortedSet.of(), Suppliers.ofInstance(ImmutableSortedSet.of()), ImmutableSet.of(), ImmutableSet.of(), /* runTestSeparately */
    false, /* testRuleTimeoutMs */
    Optional.empty(), /* maxTestOutputSize */
    100L);
    for (String sample : samples) {
        Path exitCode = Paths.get("unused");
        Path output = workspace.resolve(Paths.get(sample)).resolve("output");
        Path results = workspace.resolve(Paths.get(sample)).resolve("results");
        Path summaries = workspace.resolve(Paths.get(sample)).resolve("summaries");
        List<TestResultSummary> expectedSummaries = mapper.readValue(summaries.toFile(), SUMMARIES_REFERENCE);
        ImmutableList<TestResultSummary> actualSummaries = test.parseResults(exitCode, output, results);
        assertEquals(sample, expectedSummaries, actualSummaries);
    }
}

Example 25 with TestResultSummary

use of com.facebook.buck.test.TestResultSummary in project buck by facebook.

the class SuperConsoleEventBusListenerTest method testSkippedTest.

@Test
public void testSkippedTest() {
    Clock fakeClock = new IncrementingFakeClock(TimeUnit.SECONDS.toNanos(1));
    BuckEventBus eventBus = BuckEventBusFactory.newInstance(fakeClock);
    SuperConsoleEventBusListener listener = createSuperConsole(fakeClock, eventBus);
    SourcePathResolver pathResolver = new SourcePathResolver(new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer())));
    BuildTarget testTarget = BuildTargetFactory.newInstance("//:test");
    ImmutableSet<BuildTarget> testTargets = ImmutableSet.of(testTarget);
    Iterable<String> testArgs = Iterables.transform(testTargets, Object::toString);
    FakeBuildRule testBuildRule = new FakeBuildRule(testTarget, pathResolver, ImmutableSortedSet.of());
    ProjectBuildFileParseEvents.Started parseEventStarted = new ProjectBuildFileParseEvents.Started();
    eventBus.postWithoutConfiguring(configureTestEventAtTime(parseEventStarted, 0L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    validateConsole(listener, 0L, ImmutableList.of("[+] PARSING BUCK FILES...0.0s"));
    validateConsole(listener, 100L, ImmutableList.of("[+] PARSING BUCK FILES...0.1s"));
    eventBus.postWithoutConfiguring(configureTestEventAtTime(new ProjectBuildFileParseEvents.Finished(parseEventStarted), 200L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    validateConsole(listener, 200L, ImmutableList.of("[-] PARSING BUCK FILES...FINISHED 0.2s"));
    BuildEvent.Started buildEventStarted = BuildEvent.started(testArgs);
    eventBus.postWithoutConfiguring(configureTestEventAtTime(buildEventStarted, 200L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    ParseEvent.Started parseStarted = ParseEvent.started(testTargets);
    eventBus.postWithoutConfiguring(configureTestEventAtTime(parseStarted, 200L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    validateConsole(listener, 300L, ImmutableList.of("[+] PROCESSING BUCK FILES...0.1s"));
    eventBus.postWithoutConfiguring(configureTestEventAtTime(ParseEvent.finished(parseStarted, Optional.empty()), 300L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    ActionGraphEvent.Started actionGraphStarted = ActionGraphEvent.started();
    eventBus.postWithoutConfiguring(configureTestEventAtTime(actionGraphStarted, 300L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    eventBus.postWithoutConfiguring(configureTestEventAtTime(ActionGraphEvent.finished(actionGraphStarted), 400L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    final String parsingLine = "[-] PROCESSING BUCK FILES...FINISHED 0.2s";
    validateConsole(listener, 540L, ImmutableList.of(parsingLine, DOWNLOAD_STRING, "[+] BUILDING...0.1s"));
    BuildRuleEvent.Started started = BuildRuleEvent.started(testBuildRule, durationTracker);
    eventBus.postWithoutConfiguring(configureTestEventAtTime(started, 600L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    validateConsole(listener, 800L, ImmutableList.of(parsingLine, DOWNLOAD_STRING, "[+] BUILDING...0.4s", " |=> //:test...  0.2s (checking_cache)"));
    eventBus.postWithoutConfiguring(configureTestEventAtTime(BuildRuleEvent.finished(started, BuildRuleKeys.of(new RuleKey("aaaa")), BuildRuleStatus.SUCCESS, CacheResult.miss(), Optional.of(BuildRuleSuccessType.BUILT_LOCALLY), Optional.empty(), Optional.empty()), 1000L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    eventBus.postWithoutConfiguring(configureTestEventAtTime(BuildEvent.finished(buildEventStarted, 0), 1234L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    final String buildingLine = "[-] BUILDING...FINISHED 0.8s";
    validateConsole(listener, 1300L, ImmutableList.of(parsingLine, FINISHED_DOWNLOAD_STRING, buildingLine));
    eventBus.postWithoutConfiguring(configureTestEventAtTime(TestRunEvent.started(// isRunAllTests
    true, TestSelectorList.empty(), // shouldExplainTestSelectorList
    false, ImmutableSet.copyOf(testArgs)), 2500L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    validateConsole(listener, 3000L, ImmutableList.of(parsingLine, FINISHED_DOWNLOAD_STRING, buildingLine, "[+] TESTING...0.5s"));
    eventBus.postWithoutConfiguring(configureTestEventAtTime(TestRuleEvent.started(testTarget), 3100L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    validateConsole(listener, 3200L, ImmutableList.of(parsingLine, FINISHED_DOWNLOAD_STRING, buildingLine, "[+] TESTING...0.7s", " |=> //:test...  0.1s"));
    UUID stepUuid = new UUID(0, 1);
    StepEvent.Started stepEventStarted = StepEvent.started("step_name", "step_desc", stepUuid);
    eventBus.postWithoutConfiguring(configureTestEventAtTime(stepEventStarted, 3300L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    validateConsole(listener, 3400L, ImmutableList.of(parsingLine, FINISHED_DOWNLOAD_STRING, buildingLine, "[+] TESTING...0.9s", " |=> //:test...  0.3s (running step_name[0.1s])"));
    eventBus.postWithoutConfiguring(configureTestEventAtTime(StepEvent.finished(stepEventStarted, 0), 3500L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    validateConsole(listener, 3600L, ImmutableList.of(parsingLine, FINISHED_DOWNLOAD_STRING, buildingLine, "[+] TESTING...1.1s", " |=> //:test...  0.5s"));
    UUID testUUID = new UUID(2, 3);
    eventBus.postWithoutConfiguring(configureTestEventAtTime(TestSummaryEvent.started(testUUID, "TestClass", "Foo"), 3700L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    validateConsole(listener, 3800L, ImmutableList.of(parsingLine, FINISHED_DOWNLOAD_STRING, buildingLine, "[+] TESTING...1.3s", " |=> //:test...  0.7s (running Foo[0.1s])"));
    TestResultSummary testResultSummary = new TestResultSummary("TestClass", "Foo", ResultType.ASSUMPTION_VIOLATION, // time
    0L, // message
    null, // stacktrace
    null, // stdOut
    null, // stdErr
    null);
    eventBus.postWithoutConfiguring(configureTestEventAtTime(TestSummaryEvent.finished(testUUID, testResultSummary), 3900L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    validateConsole(listener, 4000L, ImmutableList.of(parsingLine, FINISHED_DOWNLOAD_STRING, buildingLine, "[+] TESTING...1.5s (0 PASS/1 SKIP/0 FAIL)", " |=> //:test...  0.9s"));
    eventBus.postWithoutConfiguring(configureTestEventAtTime(TestRunEvent.finished(ImmutableSet.copyOf(testArgs), ImmutableList.of(TestResults.of(testTarget, ImmutableList.of(new TestCaseSummary("TestClass", ImmutableList.of(testResultSummary))), // contacts
    ImmutableSet.of(), // labels
    ImmutableSet.of()))), 4100L, TimeUnit.MILLISECONDS, /* threadId */
    0L));
    final String testingLine = "[-] TESTING...FINISHED 1.6s (0 PASS/1 SKIP/0 FAIL)";
    validateConsoleWithStdOutAndErr(listener, 4200L, ImmutableList.of(parsingLine, FINISHED_DOWNLOAD_STRING, buildingLine, testingLine), ImmutableList.of(), Optional.of(Joiner.on('\n').join("RESULTS FOR ALL TESTS", "ASSUME  <100ms  0 Passed   1 Skipped   0 Failed   TestClass", "NO TESTS RAN (assumption violations)", "")), // We don't care about stderr, since the last frame will be flushed there.
    Optional.empty());
}