use of com.facebook.buck.test.TestCaseSummary in project buck by facebook.
the class TestRunningTest method whenSeparateTestFailsThenBuildFails.
@Test
public void whenSeparateTestFailsThenBuildFails() throws Exception {
CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
final TestResults failingTestResults = FakeTestResults.of(ImmutableList.of(new TestCaseSummary("TestCase", ImmutableList.of(new TestResultSummary("TestCaseResult", "failTest", ResultType.FAILURE, 5000, null, null, null, null)))));
BuildTarget failingTestTarget = BuildTargetFactory.newInstance("//:failingtest");
SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()));
SourcePathResolver resolver = new SourcePathResolver(ruleFinder);
FakeTestRule failingTest = new FakeTestRule(new FakeBuildRuleParamsBuilder(failingTestTarget).build(), resolver, ImmutableSet.of(), Optional.of(Paths.get("failingTestStep1OutputDir")), // runTestSeparately
true, ImmutableList.of(), () -> failingTestResults);
ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
FakeBuildEngine fakeBuildEngine = new FakeBuildEngine(ImmutableMap.of(failingTestTarget, BuildResult.success(failingTest, BUILT_LOCALLY, CacheResult.miss())), ImmutableMap.of(failingTestTarget, new RuleKey("00")));
ExecutionContext fakeExecutionContext = TestExecutionContext.newInstance();
DefaultStepRunner stepRunner = new DefaultStepRunner();
int ret = TestRunning.runTests(commandRunnerParams, ImmutableList.of(failingTest), fakeExecutionContext, DEFAULT_OPTIONS, service, fakeBuildEngine, stepRunner, resolver, ruleFinder);
assertThat(ret, equalTo(TestRunning.TEST_FAILURES_EXIT_CODE));
}
use of com.facebook.buck.test.TestCaseSummary in project buck by facebook.
the class TestRunningTest method whenSomeTestsAreSeparateThenSeparateTestsRunAtEnd.
@Test
public void whenSomeTestsAreSeparateThenSeparateTestsRunAtEnd() throws Exception {
CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
AtomicInteger atomicExecutionOrder = new AtomicInteger(0);
ExecutionOrderAwareFakeStep separateTestStep1 = new ExecutionOrderAwareFakeStep("teststep1", "teststep1", 0, atomicExecutionOrder);
final TestResults fakeTestResults = FakeTestResults.of(ImmutableList.of(new TestCaseSummary("TestCase", ImmutableList.of(new TestResultSummary("TestCaseResult", "passTest", ResultType.SUCCESS, 5000, null, null, null, null)))));
BuildTarget separateTest1Target = BuildTargetFactory.newInstance("//:test1");
FakeTestRule separateTest1 = new FakeTestRule(new FakeBuildRuleParamsBuilder(separateTest1Target).build(), new SourcePathResolver(new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()))), ImmutableSet.of(), Optional.of(Paths.get("separateTestStep1OutputDir")), // runTestSeparately
true, ImmutableList.of(separateTestStep1), () -> fakeTestResults);
ExecutionOrderAwareFakeStep separateTestStep2 = new ExecutionOrderAwareFakeStep("teststep2", "teststep2", 0, atomicExecutionOrder);
BuildTarget separateTest2Target = BuildTargetFactory.newInstance("//:test2");
FakeTestRule separateTest2 = new FakeTestRule(new FakeBuildRuleParamsBuilder(separateTest2Target).build(), new SourcePathResolver(new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()))), ImmutableSet.of(), Optional.of(Paths.get("separateTestStep2OutputDir")), // runTestSeparately
true, ImmutableList.of(separateTestStep2), () -> fakeTestResults);
ExecutionOrderAwareFakeStep separateTestStep3 = new ExecutionOrderAwareFakeStep("teststep3", "teststep3", 0, atomicExecutionOrder);
BuildTarget separateTest3Target = BuildTargetFactory.newInstance("//:test3");
FakeTestRule separateTest3 = new FakeTestRule(new FakeBuildRuleParamsBuilder(separateTest3Target).build(), new SourcePathResolver(new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()))), ImmutableSet.of(), Optional.of(Paths.get("separateTestStep3OutputDir")), // runTestSeparately
true, ImmutableList.of(separateTestStep3), () -> fakeTestResults);
ExecutionOrderAwareFakeStep parallelTestStep1 = new ExecutionOrderAwareFakeStep("parallelteststep1", "parallelteststep1", 0, atomicExecutionOrder);
BuildTarget parallelTest1Target = BuildTargetFactory.newInstance("//:paralleltest1");
FakeTestRule parallelTest1 = new FakeTestRule(new FakeBuildRuleParamsBuilder(parallelTest1Target).build(), new SourcePathResolver(new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()))), ImmutableSet.of(), Optional.of(Paths.get("parallelTestStep1OutputDir")), // runTestSeparately
false, ImmutableList.of(parallelTestStep1), () -> fakeTestResults);
ExecutionOrderAwareFakeStep parallelTestStep2 = new ExecutionOrderAwareFakeStep("parallelteststep2", "parallelteststep2", 0, atomicExecutionOrder);
BuildTarget parallelTest2Target = BuildTargetFactory.newInstance("//:paralleltest2");
FakeTestRule parallelTest2 = new FakeTestRule(new FakeBuildRuleParamsBuilder(parallelTest2Target).build(), new SourcePathResolver(new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()))), ImmutableSet.of(), Optional.of(Paths.get("parallelTestStep2OutputDir")), // runTestSeparately
false, ImmutableList.of(parallelTestStep2), () -> fakeTestResults);
ExecutionOrderAwareFakeStep parallelTestStep3 = new ExecutionOrderAwareFakeStep("parallelteststep3", "parallelteststep3", 0, atomicExecutionOrder);
BuildTarget parallelTest3Target = BuildTargetFactory.newInstance("//:paralleltest3");
FakeTestRule parallelTest3 = new FakeTestRule(new FakeBuildRuleParamsBuilder(parallelTest3Target).build(), new SourcePathResolver(new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()))), ImmutableSet.of(), Optional.of(Paths.get("parallelTestStep3OutputDir")), // runTestSeparately
false, ImmutableList.of(parallelTestStep3), () -> fakeTestResults);
// We explicitly use an actual thread pool here; the logic should ensure the
// separate tests are run in the correct order.
ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
FakeBuildEngine fakeBuildEngine = new FakeBuildEngine(ImmutableMap.<BuildTarget, BuildResult>builder().put(separateTest1Target, BuildResult.success(separateTest1, BUILT_LOCALLY, CacheResult.miss())).put(separateTest2Target, BuildResult.success(separateTest2, BUILT_LOCALLY, CacheResult.miss())).put(separateTest3Target, BuildResult.success(separateTest3, BUILT_LOCALLY, CacheResult.miss())).put(parallelTest1Target, BuildResult.success(parallelTest1, BUILT_LOCALLY, CacheResult.miss())).put(parallelTest2Target, BuildResult.success(parallelTest2, BUILT_LOCALLY, CacheResult.miss())).put(parallelTest3Target, BuildResult.success(parallelTest3, BUILT_LOCALLY, CacheResult.miss())).build(), ImmutableMap.<BuildTarget, RuleKey>builder().put(separateTest1Target, new RuleKey("00")).put(separateTest2Target, new RuleKey("00")).put(separateTest3Target, new RuleKey("00")).put(parallelTest1Target, new RuleKey("00")).put(parallelTest2Target, new RuleKey("00")).put(parallelTest3Target, new RuleKey("00")).build());
ExecutionContext fakeExecutionContext = TestExecutionContext.newInstance();
DefaultStepRunner stepRunner = new DefaultStepRunner();
SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()));
int ret = TestRunning.runTests(commandRunnerParams, ImmutableList.of(separateTest1, parallelTest1, separateTest2, parallelTest2, separateTest3, parallelTest3), fakeExecutionContext, DEFAULT_OPTIONS, service, fakeBuildEngine, stepRunner, new SourcePathResolver(ruleFinder), ruleFinder);
assertThat(ret, equalTo(0));
// The tests not marked as separate could run in any order -- but they must run
// before the separate test steps.
ImmutableSet<Optional<Integer>> expectedParallelStepExecutionOrderSet = ImmutableSet.<Optional<Integer>>builder().add(Optional.of(0)).add(Optional.of(1)).add(Optional.of(2)).add(Optional.of(3)).add(Optional.of(4)).add(Optional.of(5)).build();
ImmutableSet<Optional<Integer>> actualParallelStepExecutionOrderSet = ImmutableSet.<Optional<Integer>>builder().add(parallelTestStep1.getExecutionBeginOrder()).add(parallelTestStep1.getExecutionEndOrder()).add(parallelTestStep2.getExecutionBeginOrder()).add(parallelTestStep2.getExecutionEndOrder()).add(parallelTestStep3.getExecutionBeginOrder()).add(parallelTestStep3.getExecutionEndOrder()).build();
LOG.debug("Expected parallel execution order: %s Actual parallel execution order: %s", expectedParallelStepExecutionOrderSet, actualParallelStepExecutionOrderSet);
// We allow the parallel steps to begin and end in any order (note the thread
// pool of size 3 above), so we use a set.
assertThat(actualParallelStepExecutionOrderSet, equalTo(expectedParallelStepExecutionOrderSet));
// The separate test steps must begin and end in a specific order, so we use a list.
ImmutableList<Optional<Integer>> expectedSeparateStepExecutionOrderList = ImmutableList.<Optional<Integer>>builder().add(Optional.of(6)).add(Optional.of(7)).add(Optional.of(8)).add(Optional.of(9)).add(Optional.of(10)).add(Optional.of(11)).build();
ImmutableList<Optional<Integer>> actualSeparateStepExecutionOrderList = ImmutableList.<Optional<Integer>>builder().add(separateTestStep1.getExecutionBeginOrder()).add(separateTestStep1.getExecutionEndOrder()).add(separateTestStep2.getExecutionBeginOrder()).add(separateTestStep2.getExecutionEndOrder()).add(separateTestStep3.getExecutionBeginOrder()).add(separateTestStep3.getExecutionEndOrder()).build();
LOG.debug("Expected separate execution order: %s Actual separate execution order: %s", expectedSeparateStepExecutionOrderList, actualSeparateStepExecutionOrderList);
assertThat(actualSeparateStepExecutionOrderList, equalTo(expectedSeparateStepExecutionOrderList));
}
use of com.facebook.buck.test.TestCaseSummary in project buck by facebook.
the class TestRunningTest method testXmlGeneration.
/**
* Tests the --xml flag, ensuring that test result data is correctly
* formatted.
*/
@Test
public void testXmlGeneration() throws Exception {
// Set up sample test data.
TestResultSummary result1 = new TestResultSummary(/* testCaseName */
"TestCase", /* testName */
"passTest", /* type */
ResultType.SUCCESS, /* time */
5000, /* message */
null, /* stacktrace */
null, /* stdOut */
null, /* stdErr */
null);
TestResultSummary result2 = new TestResultSummary(/* testCaseName */
"TestCase", /* testName */
"failWithMsg", /* type */
ResultType.FAILURE, /* time */
7000, /* message */
"Index out of bounds!", /* stacktrace */
"Stacktrace", /* stdOut */
null, /* stdErr */
null);
TestResultSummary result3 = new TestResultSummary(/* testCaseName */
"TestCase", /* testName */
"failNoMsg", /* type */
ResultType.SUCCESS, /* time */
4000, /* message */
null, /* stacktrace */
null, /* stdOut */
null, /* stdErr */
null);
List<TestResultSummary> resultList = ImmutableList.of(result1, result2, result3);
TestCaseSummary testCase = new TestCaseSummary("TestCase", resultList);
List<TestCaseSummary> testCases = ImmutableList.of(testCase);
TestResults testResults = FakeTestResults.of(testCases);
List<TestResults> testResultsList = ImmutableList.of(testResults);
// Call the XML generation method with our test data.
StringWriter writer = new StringWriter();
TestRunning.writeXmlOutput(testResultsList, writer);
ByteArrayInputStream resultStream = new ByteArrayInputStream(writer.toString().getBytes());
// Convert the raw XML data into a DOM object, which we will check.
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder docBuilder = dbf.newDocumentBuilder();
Document doc = docBuilder.parse(resultStream);
// Check for exactly one <tests> tag.
NodeList testsList = doc.getElementsByTagName("tests");
assertEquals(testsList.getLength(), 1);
// Check for exactly one <test> tag.
Element testsEl = (Element) testsList.item(0);
NodeList testList = testsEl.getElementsByTagName("test");
assertEquals(testList.getLength(), 1);
// Check for exactly three <testresult> tags.
// There should be two failures and one success.
Element testEl = (Element) testList.item(0);
NodeList resultsList = testEl.getElementsByTagName("testresult");
assertEquals(resultsList.getLength(), 3);
// Verify the text elements of the first <testresult> tag.
Element passResultEl = (Element) resultsList.item(0);
assertEquals(passResultEl.getAttribute("name"), "passTest");
assertEquals(passResultEl.getAttribute("time"), "5000");
checkXmlTextContents(passResultEl, "message", "");
checkXmlTextContents(passResultEl, "stacktrace", "");
// Verify the text elements of the second <testresult> tag.
assertEquals(testEl.getAttribute("name"), "TestCase");
Element failResultEl1 = (Element) resultsList.item(1);
assertEquals(failResultEl1.getAttribute("name"), "failWithMsg");
assertEquals(failResultEl1.getAttribute("time"), "7000");
checkXmlTextContents(failResultEl1, "message", "Index out of bounds!");
checkXmlTextContents(failResultEl1, "stacktrace", "Stacktrace");
// Verify the text elements of the third <testresult> tag.
Element failResultEl2 = (Element) resultsList.item(2);
assertEquals(failResultEl2.getAttribute("name"), "failNoMsg");
assertEquals(failResultEl2.getAttribute("time"), "4000");
checkXmlTextContents(failResultEl2, "message", "");
checkXmlTextContents(failResultEl2, "stacktrace", "");
}
use of com.facebook.buck.test.TestCaseSummary in project buck by facebook.
the class TestRunningTest method whenAllTestsAreSeparateTestsRunInOrder.
@Test
public void whenAllTestsAreSeparateTestsRunInOrder() throws Exception {
CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
AtomicInteger atomicExecutionOrder = new AtomicInteger(0);
ExecutionOrderAwareFakeStep separateTestStep1 = new ExecutionOrderAwareFakeStep("teststep1", "teststep1", 0, atomicExecutionOrder);
final TestResults fakeTestResults = FakeTestResults.of(ImmutableList.of(new TestCaseSummary("TestCase", ImmutableList.of(new TestResultSummary("TestCaseResult", "passTest", ResultType.SUCCESS, 5000, null, null, null, null)))));
BuildTarget separateTest1Target = BuildTargetFactory.newInstance("//:test1");
FakeTestRule separateTest1 = new FakeTestRule(new FakeBuildRuleParamsBuilder(separateTest1Target).build(), new SourcePathResolver(new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()))), ImmutableSet.of(), Optional.of(Paths.get("separateTestStep1OutputDir")), // runTestSeparately
true, ImmutableList.of(separateTestStep1), () -> fakeTestResults);
ExecutionOrderAwareFakeStep separateTestStep2 = new ExecutionOrderAwareFakeStep("teststep2", "teststep2", 0, atomicExecutionOrder);
BuildTarget separateTest2Target = BuildTargetFactory.newInstance("//:test2");
FakeTestRule separateTest2 = new FakeTestRule(new FakeBuildRuleParamsBuilder(separateTest2Target).build(), new SourcePathResolver(new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()))), ImmutableSet.of(), Optional.of(Paths.get("separateTestStep2OutputDir")), // runTestSeparately
true, ImmutableList.of(separateTestStep2), () -> fakeTestResults);
ExecutionOrderAwareFakeStep separateTestStep3 = new ExecutionOrderAwareFakeStep("teststep3", "teststep3", 0, atomicExecutionOrder);
BuildTarget separateTest3Target = BuildTargetFactory.newInstance("//:test3");
FakeTestRule separateTest3 = new FakeTestRule(new FakeBuildRuleParamsBuilder(separateTest3Target).build(), new SourcePathResolver(new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()))), ImmutableSet.of(), Optional.of(Paths.get("separateTestStep3OutputDir")), // runTestSeparately
true, ImmutableList.of(separateTestStep3), () -> fakeTestResults);
// We explicitly use an actual thread pool here; the logic should ensure the
// separate tests are run in the correct order.
ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
FakeBuildEngine fakeBuildEngine = new FakeBuildEngine(ImmutableMap.of(separateTest1Target, BuildResult.success(separateTest1, BUILT_LOCALLY, CacheResult.miss()), separateTest2Target, BuildResult.success(separateTest2, BUILT_LOCALLY, CacheResult.miss()), separateTest3Target, BuildResult.success(separateTest3, BUILT_LOCALLY, CacheResult.miss())), ImmutableMap.of(separateTest1Target, new RuleKey("00"), separateTest2Target, new RuleKey("00"), separateTest3Target, new RuleKey("00")));
ExecutionContext fakeExecutionContext = TestExecutionContext.newInstance();
DefaultStepRunner stepRunner = new DefaultStepRunner();
SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()));
int ret = TestRunning.runTests(commandRunnerParams, ImmutableList.of(separateTest1, separateTest2, separateTest3), fakeExecutionContext, DEFAULT_OPTIONS, service, fakeBuildEngine, stepRunner, new SourcePathResolver(ruleFinder), ruleFinder);
assertThat(ret, equalTo(0));
assertThat(separateTestStep1.getExecutionBeginOrder(), equalTo(Optional.of(0)));
assertThat(separateTestStep1.getExecutionEndOrder(), equalTo(Optional.of(1)));
assertThat(separateTestStep2.getExecutionBeginOrder(), equalTo(Optional.of(2)));
assertThat(separateTestStep2.getExecutionEndOrder(), equalTo(Optional.of(3)));
assertThat(separateTestStep3.getExecutionBeginOrder(), equalTo(Optional.of(4)));
assertThat(separateTestStep3.getExecutionEndOrder(), equalTo(Optional.of(5)));
}
use of com.facebook.buck.test.TestCaseSummary in project buck by facebook.
the class TestResultFormatterTest method shouldOutputLogLinesOfFailingTest.
@Test
public void shouldOutputLogLinesOfFailingTest() throws IOException {
TestResultFormatter formatter = createFormatterWithMaxLogLines(10);
TestCaseSummary summary = new TestCaseSummary("com.example.FooTest", ImmutableList.of(failingTest));
Files.write(logPath, ImmutableList.of("This is a debug log", "Here's another one"), StandardCharsets.UTF_8);
TestResults results = TestResults.builder().setBuildTarget(BuildTargetFactory.newInstance("//foo:bar")).setTestCases(ImmutableList.of(summary)).addTestLogPaths(logPath).build();
ImmutableList.Builder<String> builder = ImmutableList.builder();
formatter.reportResult(builder, results);
String expected = String.format(Joiner.on('\n').join("FAIL 200ms 0 Passed 0 Skipped 1 Failed com.example.FooTest", "FAILURE %s %s: %s", "%s", "====TEST LOGS====", "Logs from log.txt:", "This is a debug log", "Here's another one"), failingTest.getTestCaseName(), failingTest.getTestName(), failingTest.getMessage(), stackTrace);
assertEquals(expected, toString(builder));
}
Aggregations