Examples with TestResultSummary com.facebook.buck.test.TestResultSummary used on opensource projects

Example 11 with TestResultSummary

use of com.facebook.buck.test.TestResultSummary in project buck by facebook.

the class RunShTestAndRecordResultStep method execute.

@Override
public StepExecutionResult execute(ExecutionContext context) throws IOException, InterruptedException {
    TestResultSummary summary;
    if (context.getPlatform() == Platform.WINDOWS) {
        // Ignore sh_test on Windows.
        summary = new TestResultSummary(getShortName(), "sh_test", /* type */
        ResultType.SUCCESS, /* duration*/
        0, /* message */
        "sh_test ignored on Windows", /* stacktrace */
        null, /* stdout */
        null, /* stderr */
        null);
    } else {
        ShellStep test = new ShellStep(filesystem.getRootPath()) {

            boolean timedOut = false;

            @Override
            public String getShortName() {
                return pathToShellScript.toString();
            }

            @Override
            protected ImmutableList<String> getShellCommandInternal(ExecutionContext context) {
                return ImmutableList.<String>builder().add(pathToShellScript.toString()).addAll(args).build();
            }

            @Override
            public ImmutableMap<String, String> getEnvironmentVariables(ExecutionContext context) {
                return ImmutableMap.<String, String>builder().put("NO_BUCKD", "1").putAll(env).build();
            }

            @Override
            protected boolean shouldPrintStderr(Verbosity verbosity) {
                // Do not stream this output because we want to capture it.
                return false;
            }

            @Override
            public StepExecutionResult execute(ExecutionContext context) throws IOException, InterruptedException {
                StepExecutionResult executionResult = super.execute(context);
                if (timedOut) {
                    throw new HumanReadableException("Timed out running test: " + testCaseName + ", with exitCode: " + executionResult.getExitCode());
                }
                return executionResult;
            }

            @Override
            protected Optional<Consumer<Process>> getTimeoutHandler(final ExecutionContext context) {
                return Optional.of(process -> timedOut = true);
            }

            @Override
            protected Optional<Long> getTimeout() {
                return testRuleTimeoutMs;
            }

            @Override
            protected boolean shouldPrintStdout(Verbosity verbosity) {
                // Do not stream this output because we want to capture it.
                return false;
            }
        };
        StepExecutionResult executionResult = test.execute(context);
        // Write test result.
        boolean isSuccess = executionResult.isSuccess();
        summary = new TestResultSummary(getShortName(), "sh_test", /* type */
        isSuccess ? ResultType.SUCCESS : ResultType.FAILURE, test.getDuration(), /* message */
        null, /* stacktrace */
        null, test.getStdout(), test.getStderr());
    }
    ObjectMapper mapper = context.getObjectMapper();
    try (OutputStream outputStream = filesystem.newFileOutputStream(pathToTestResultFile)) {
        mapper.writeValue(outputStream, summary);
    }
    // should be zero.
    return StepExecutionResult.SUCCESS;
}

Example 12 with TestResultSummary

use of com.facebook.buck.test.TestResultSummary in project buck by facebook.

the class TestCaseSummariesBuildingXctoolEventHandlerTest method noTestCasesAndOcunitFailureReturnsFailedTestResultSummary.

@Test
public void noTestCasesAndOcunitFailureReturnsFailedTestResultSummary() throws Exception {
    Path jsonPath = TestDataHelper.getTestDataDirectory(this).resolve("xctool-output/ocunit-failure.json");
    try (Reader jsonReader = Files.newBufferedReader(jsonPath, StandardCharsets.UTF_8)) {
        TestCaseSummariesBuildingXctoolEventHandler testCaseSummariesBuilder = new TestCaseSummariesBuildingXctoolEventHandler(TestRule.NOOP_REPORTING_CALLBACK);
        XctoolOutputParsing.streamOutputFromReader(jsonReader, testCaseSummariesBuilder);
        List<TestCaseSummary> summaries = testCaseSummariesBuilder.getTestCaseSummaries();
        assertThat(summaries, hasSize(1));
        Matcher<TestResultSummary> summaryMatcher = allOf(hasProperty("type", equalTo(ResultType.FAILURE)), hasProperty("message", containsString("dyld: app was built for iOS 8.3 which is newer than this simulator 8.1")));
        assertThat(summaries.get(0).getTestResults(), contains(summaryMatcher));
    }
}

Example 13 with TestResultSummary

use of com.facebook.buck.test.TestResultSummary in project buck by facebook.

the class TestCaseSummariesBuildingXctoolEventHandlerTest method mixedPassAndFailReturnsMixedResultSummary.

@Test
@SuppressWarnings("unchecked")
public void mixedPassAndFailReturnsMixedResultSummary() throws Exception {
    Path jsonPath = TestDataHelper.getTestDataDirectory(this).resolve("xctool-output/mixed-pass-and-fail.json");
    try (Reader jsonReader = Files.newBufferedReader(jsonPath, StandardCharsets.UTF_8)) {
        TestCaseSummariesBuildingXctoolEventHandler testCaseSummariesBuilder = new TestCaseSummariesBuildingXctoolEventHandler(TestRule.NOOP_REPORTING_CALLBACK);
        XctoolOutputParsing.streamOutputFromReader(jsonReader, testCaseSummariesBuilder);
        List<TestCaseSummary> summaries = testCaseSummariesBuilder.getTestCaseSummaries();
        assertThat(summaries, hasSize(2));
        Matcher<TestResultSummary> isOtherTestsTestSomethingSuccess = allOf(hasProperty("testCaseName", equalTo("OtherTests")), hasProperty("testName", equalTo("-[OtherTests testSomething]")), hasProperty("type", equalTo(ResultType.SUCCESS)), hasProperty("time", equalTo(3L)), hasProperty("message", nullValue(String.class)), hasProperty("stacktrace", nullValue(String.class)), hasProperty("stdOut", nullValue(String.class)), hasProperty("stdErr", nullValue(String.class)));
        List<TestResultSummary> otherTestsResults = summaries.get(0).getTestResults();
        assertThat(otherTestsResults, contains(isOtherTestsTestSomethingSuccess));
        Matcher<TestResultSummary> isSomeTestsTestBacktraceOutputIsCaptured = allOf(hasProperty("testCaseName", equalTo("SomeTests")), hasProperty("testName", equalTo("-[SomeTests testBacktraceOutputIsCaptured]")), hasProperty("type", equalTo(ResultType.SUCCESS)), hasProperty("time", equalTo(0L)), hasProperty("message", nullValue(String.class)), hasProperty("stacktrace", nullValue(String.class)), hasProperty("stdOut", containsString("-[SenTestCase performTest:]")), hasProperty("stdErr", nullValue(String.class)));
        Matcher<TestResultSummary> isSomeTestsTestOutputMerging = allOf(hasProperty("testCaseName", equalTo("SomeTests")), hasProperty("testName", equalTo("-[SomeTests testOutputMerging]")), hasProperty("type", equalTo(ResultType.SUCCESS)), hasProperty("time", equalTo(0L)), hasProperty("message", nullValue(String.class)), hasProperty("stacktrace", nullValue(String.class)), hasProperty("stdOut", containsString("stdout-line1\nstderr-line1\n")), hasProperty("stdErr", nullValue(String.class)));
        Matcher<TestResultSummary> isSomeTestsTestPrintSDK = allOf(hasProperty("testCaseName", equalTo("SomeTests")), hasProperty("testName", equalTo("-[SomeTests testPrintSDK]")), hasProperty("type", equalTo(ResultType.SUCCESS)), hasProperty("time", equalTo(1L)), hasProperty("message", nullValue(String.class)), hasProperty("stacktrace", nullValue(String.class)), hasProperty("stdOut", containsString("SDK: 6.1")), hasProperty("stdErr", nullValue(String.class)));
        Matcher<TestResultSummary> isSomeTestsTestStream = allOf(hasProperty("testCaseName", equalTo("SomeTests")), hasProperty("testName", equalTo("-[SomeTests testStream]")), hasProperty("type", equalTo(ResultType.SUCCESS)), hasProperty("time", equalTo(754L)), hasProperty("message", nullValue(String.class)), hasProperty("stacktrace", nullValue(String.class)), hasProperty("stdOut", containsString(">>>> i = 0")), hasProperty("stdErr", nullValue(String.class)));
        Matcher<TestResultSummary> isSomeTestsTestWillFail = allOf(hasProperty("testCaseName", equalTo("SomeTests")), hasProperty("testName", equalTo("-[SomeTests testWillFail]")), hasProperty("type", equalTo(ResultType.FAILURE)), hasProperty("time", equalTo(0L)), hasProperty("message", containsString("SomeTests.m:40: 'a' should be equal to 'b'")), hasProperty("stacktrace", nullValue(String.class)), hasProperty("stdOut", nullValue(String.class)), hasProperty("stdErr", nullValue(String.class)));
        Matcher<TestResultSummary> isSomeTestsTestWillPass = allOf(hasProperty("testCaseName", equalTo("SomeTests")), hasProperty("testName", equalTo("-[SomeTests testWillPass]")), hasProperty("type", equalTo(ResultType.SUCCESS)), hasProperty("time", equalTo(0L)), hasProperty("message", nullValue(String.class)), hasProperty("stacktrace", nullValue(String.class)), hasProperty("stdOut", nullValue(String.class)), hasProperty("stdErr", nullValue(String.class)));
        List<TestResultSummary> someTestsResults = summaries.get(1).getTestResults();
        assertThat(someTestsResults, contains(isSomeTestsTestBacktraceOutputIsCaptured, isSomeTestsTestOutputMerging, isSomeTestsTestPrintSDK, isSomeTestsTestStream, isSomeTestsTestWillFail, isSomeTestsTestWillPass));
    }
}

Example 14 with TestResultSummary

use of com.facebook.buck.test.TestResultSummary in project buck by facebook.

the class XctestOutputParsingTest method mixedPassAndFailReturnsMixedResultSummary.

@Test
@SuppressWarnings("unchecked")
public void mixedPassAndFailReturnsMixedResultSummary() throws Exception {
    Path outputPath = TestDataHelper.getTestDataDirectory(this).resolve("xctest-output/mixed-pass-and-fail.txt");
    try (Reader outputReader = Files.newBufferedReader(outputPath, StandardCharsets.UTF_8)) {
        TestCaseSummariesBuildingXctestEventHandler xctestEventHandler = new TestCaseSummariesBuildingXctestEventHandler(TestRule.NOOP_REPORTING_CALLBACK);
        XctestOutputParsing.streamOutput(outputReader, xctestEventHandler);
        List<TestCaseSummary> summaries = xctestEventHandler.getTestCaseSummaries();
        assertThat(summaries, hasSize(2));
        Matcher<TestResultSummary> isOtherTestsTestSomethingSuccess = allOf(hasProperty("testCaseName", equalTo("OtherTests")), hasProperty("testName", equalTo("testSomething")), hasProperty("type", equalTo(ResultType.SUCCESS)), hasProperty("time", equalTo(3L)), hasProperty("message", nullValue(String.class)), hasProperty("stacktrace", nullValue(String.class)), hasProperty("stdOut", nullValue(String.class)), hasProperty("stdErr", nullValue(String.class)));
        List<TestResultSummary> otherTestsResults = summaries.get(0).getTestResults();
        assertThat(otherTestsResults, contains(isOtherTestsTestSomethingSuccess));
        Matcher<TestResultSummary> isSomeTestsTestWillFail = allOf(hasProperty("testCaseName", equalTo("SomeTests")), hasProperty("testName", equalTo("testWillFail")), hasProperty("type", equalTo(ResultType.FAILURE)), hasProperty("time", equalTo(0L)), hasProperty("message", nullValue(String.class)), hasProperty("stacktrace", nullValue(String.class)), hasProperty("stdOut", nullValue(String.class)), hasProperty("stdErr", nullValue(String.class)));
        Matcher<TestResultSummary> isSomeTestsTestWillPass = allOf(hasProperty("testCaseName", equalTo("SomeTests")), hasProperty("testName", equalTo("testWillPass")), hasProperty("type", equalTo(ResultType.SUCCESS)), hasProperty("time", equalTo(0L)), hasProperty("message", nullValue(String.class)), hasProperty("stacktrace", nullValue(String.class)), hasProperty("stdOut", nullValue(String.class)), hasProperty("stdErr", nullValue(String.class)));
        List<TestResultSummary> someTestsResults = summaries.get(1).getTestResults();
        assertThat(someTestsResults, contains(isSomeTestsTestWillFail, isSomeTestsTestWillPass));
    }
}

Example 15 with TestResultSummary

use of com.facebook.buck.test.TestResultSummary in project buck by facebook.

the class CxxBoostTestTest method testParseResults.

@Test
public void testParseResults() throws Exception {
    ProjectWorkspace workspace = TestDataHelper.createProjectWorkspaceForScenario(this, "boost_test", tmp);
    workspace.setUp();
    ImmutableList<String> samples = ImmutableList.of("simple_success", "simple_failure", "simple_failure_with_output");
    BuildTarget target = BuildTargetFactory.newInstance("//:test");
    BuildRuleResolver ruleResolver = new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer());
    SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(ruleResolver);
    CxxBoostTest test = new CxxBoostTest(new FakeBuildRuleParamsBuilder(target).setProjectFilesystem(new ProjectFilesystem(tmp.getRoot())).build(), ruleFinder, new CxxLink(new FakeBuildRuleParamsBuilder(BuildTargetFactory.newInstance("//:link")).build(), CxxPlatformUtils.DEFAULT_PLATFORM.getLd().resolve(ruleResolver), Paths.get("output"), ImmutableList.of(), Optional.empty(), /* cacheable */
    true), new CommandTool.Builder().addArg(StringArg.of("")).build(), ImmutableMap.of(), Suppliers.ofInstance(ImmutableList.of()), ImmutableSortedSet.of(), Suppliers.ofInstance(ImmutableSortedSet.of()), ImmutableSet.of(), ImmutableSet.of(), /* runTestSeparately */
    false, /* testRuleTimeoutMs */
    Optional.empty());
    for (String sample : samples) {
        Path exitCode = Paths.get("unused");
        Path output = workspace.resolve(Paths.get(sample)).resolve("output");
        Path results = workspace.resolve(Paths.get(sample)).resolve("results");
        Path summaries = workspace.resolve(Paths.get(sample)).resolve("summaries");
        List<TestResultSummary> expectedSummaries = mapper.readValue(summaries.toFile(), SUMMARIES_REFERENCE);
        ImmutableList<TestResultSummary> actualSummaries = test.parseResults(exitCode, output, results);
        assertEquals(sample, expectedSummaries, actualSummaries);
    }
}