Examples with ExecutionResult org.evosuite.testcase.execution.ExecutionResult used on opensource projects

Example 76 with ExecutionResult

use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.

the class RegressionSuiteMinimizer method removeDuplicateExceptions.

private void removeDuplicateExceptions(RegressionTestSuiteChromosome suite) {
    Set<String> uniqueExceptions = new HashSet<>();
    Map<String, String> exceptionStatementMapping = new HashMap<>();
    List<TestChromosome> chromosomes = suite.getTestChromosomes();
    for (int i = 0; i < chromosomes.size(); i++) {
        RegressionTestChromosome test = (RegressionTestChromosome) chromosomes.get(i);
        boolean changed = false;
        boolean hadAssertion = test.getTheTest().getTestCase().getAssertions().size() > 0;
        ExecutionResult resultA = test.getLastExecutionResult();
        ExecutionResult resultB = test.getLastRegressionExecutionResult();
        // re-execute the tests and check
        if (resultA == null || resultB == null) {
            executeTest(test);
            resultA = test.getLastExecutionResult();
            resultB = test.getLastRegressionExecutionResult();
            if (resultA == null || resultB == null) {
                continue;
            }
        }
        Map<Integer, Throwable> exceptionMapA = resultA.getCopyOfExceptionMapping();
        Map<Integer, Throwable> exceptionMapB = resultB.getCopyOfExceptionMapping();
        // exceptionMapA.size(), exceptionMapB.size());
        if (!resultA.noThrownExceptions() || !resultB.noThrownExceptions()) {
            double exDiff = RegressionExceptionHelper.compareExceptionDiffs(exceptionMapA, exceptionMapB);
            logger.warn("Test{} - Difference in number of exceptions: {}", i, exDiff);
            if (exDiff > 0) {
                /*
           * Three scenarios:
           * 1. Same exception, different messages
           * 2. Different exception in A
           * 3. Different exception in B
           */
                for (Entry<Integer, Throwable> ex : exceptionMapA.entrySet()) {
                    Throwable exA = ex.getValue();
                    // unique statement key over all tests (to avoid removing the same exception twice)
                    String exKey = i + ":" + ex.getKey();
                    // exceptionA signatures
                    String exception = RegressionExceptionHelper.simpleExceptionName(test, ex.getKey(), exA);
                    String exSignatureA = RegressionExceptionHelper.getExceptionSignature(exA, Properties.TARGET_CLASS);
                    String signaturePair = exSignatureA + ",";
                    Throwable exB = exceptionMapB.get(ex.getKey());
                    if (exB != null) {
                        // if the same statement in B also throws an exception
                        // exceptionB signatures
                        String exceptionB = RegressionExceptionHelper.simpleExceptionName(test, ex.getKey(), exB);
                        String exSignatureB = RegressionExceptionHelper.getExceptionSignature(exA, Properties.TARGET_CLASS);
                        signaturePair += exSignatureB;
                        if (exception.equals(exceptionB) || exSignatureA.equals(exSignatureB)) {
                            // We will be taking care of removing this exception when checking from A to B
                            // so there isn't a need to check again from B to A
                            exceptionMapB.remove(ex.getKey());
                        }
                    }
                    logger.warn("Test{}, uniqueExceptions: {}", i, uniqueExceptions);
                    logger.warn("checking exception: {} at {}", exception, ex.getKey());
                    List<String> signatures = Arrays.asList(exception, signaturePair);
                    for (String sig : signatures) {
                        // Compare exceptions on the merit of their message or signature
                        if (uniqueExceptions.contains(sig) && exceptionStatementMapping.get(sig) != exKey && !hadAssertion) {
                            TestChromosome originalTestChromosome = (TestChromosome) test.getTheTest().clone();
                            try {
                                TestFactory testFactory = TestFactory.getInstance();
                                testFactory.deleteStatementGracefully(test.getTheTest().getTestCase(), ex.getKey());
                                test.getTheTest().setChanged(true);
                                logger.warn("Removed exceptionA throwing line {}", ex.getKey());
                            } catch (ConstructionFailedException e) {
                                test.getTheTest().setChanged(false);
                                test.getTheTest().setTestCase(originalTestChromosome.getTestCase());
                                logger.error("ExceptionA deletion failed");
                                continue;
                            }
                            changed = true;
                            break;
                        } else {
                            uniqueExceptions.add(sig);
                            exceptionStatementMapping.put(sig, exKey);
                        }
                    }
                }
                // Check from the other way around (B to A)
                for (Entry<Integer, Throwable> ex : exceptionMapB.entrySet()) {
                    String exception = RegressionExceptionHelper.simpleExceptionName(test, ex.getKey(), ex.getValue());
                    String exKey = i + ":" + ex.getKey();
                    logger.warn("Test{}, uniqueExceptions: {}", i, uniqueExceptions);
                    logger.warn("checking exceptionB: {} at {}", exception, ex.getKey());
                    if (uniqueExceptions.contains(exception) && exceptionStatementMapping.get(exception) != exKey && !hadAssertion && test.getTheTest().getTestCase().hasStatement(ex.getKey())) {
                        TestChromosome originalTestChromosome = (TestChromosome) test.getTheTest().clone();
                        try {
                            TestFactory testFactory = TestFactory.getInstance();
                            logger.warn("removing statementB: {}", test.getTheTest().getTestCase().getStatement(ex.getKey()));
                            testFactory.deleteStatementGracefully(test.getTheTest().getTestCase(), ex.getKey());
                            test.getTheTest().setChanged(true);
                            logger.warn("removed exceptionB throwing line {}", ex.getKey());
                        } catch (ConstructionFailedException e) {
                            test.getTheTest().setChanged(false);
                            test.getTheTest().setTestCase(originalTestChromosome.getTestCase());
                            logger.error("ExceptionB deletion failed");
                            continue;
                        }
                        changed = true;
                    } else {
                        uniqueExceptions.add(exception);
                        exceptionStatementMapping.put(exception, exKey);
                    }
                }
            }
        }
        if (changed) {
            test.updateClassloader();
            executeTest(test);
            i--;
        }
    }
    if (uniqueExceptions.size() > 0) {
        logger.warn("unique exceptions: {}", uniqueExceptions);
    }
}

Example 77 with ExecutionResult

use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.

the class RegressionSuiteMinimizer method numFailingAssertions.

private int numFailingAssertions(RegressionTestChromosome test) {
    int count = 0;
    Set<Assertion> invalidAssertions = new HashSet<>();
    ExecutionResult resultA = test.getLastExecutionResult();
    ExecutionResult resultB = test.getLastRegressionExecutionResult();
    for (Assertion assertion : test.getTheSameTestForTheOtherClassLoader().getTestCase().getAssertions()) {
        for (OutputTrace<?> outputTrace : resultA.getTraces()) {
            if (outputTrace.isDetectedBy(assertion)) {
                logger.error("shouldn't be happening: assertion was failing on original version");
                invalidAssertions.add(assertion);
                break;
            }
        }
        for (OutputTrace<?> outputTrace : resultB.getTraces()) {
            if (outputTrace.isDetectedBy(assertion) && !invalidAssertions.contains(assertion)) {
                count++;
                break;
            }
        }
    }
    if (invalidAssertions.size() != 0) {
        logger.warn("{} invalid assertion(s) to be removed", invalidAssertions);
        for (Assertion assertion : invalidAssertions) {
            test.getTheTest().getTestCase().removeAssertion(assertion);
            test.getTheTest().setChanged(true);
            test.updateClassloader();
        }
    }
    int exDiffCount = 0;
    if (resultA != null && resultB != null) {
        exDiffCount = (int) RegressionExceptionHelper.compareExceptionDiffs(resultA.getCopyOfExceptionMapping(), resultB.getCopyOfExceptionMapping());
        // logger.warn("exDiffCount: {}: \nmap1:{}\nmap2:{}\n",exDiffCount,resultA.getCopyOfExceptionMapping(),
        // resultB.getCopyOfExceptionMapping());
        count += exDiffCount;
        if (exDiffCount > 0 && !test.exCommentsAdded) {
            // logger.warn("Adding Exception Comments for test: \nmap1:{}\nmap2:{}\n",resultA.getCopyOfExceptionMapping(),
            // resultB.getCopyOfExceptionMapping());
            RegressionExceptionHelper.addExceptionAssertionComments(test, resultA.getCopyOfExceptionMapping(), resultB.getCopyOfExceptionMapping());
            test.exCommentsAdded = true;
        }
    } else {
        logger.error("resultA: {} | resultB: {}", resultA, resultB);
    }
    return count;
}

Example 78 with ExecutionResult

use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.

the class TestSuiteWriter method writeTestSuite.

/**
 * Create JUnit test suite for class
 *
 * @param name      Name of the class
 * @param directory Output directory
 */
public List<File> writeTestSuite(String name, String directory, List<ExecutionResult> cachedResults) throws IllegalArgumentException {
    if (name == null || name.isEmpty()) {
        throw new IllegalArgumentException("Empty test class name");
    }
    if (!name.endsWith("Test")) {
        /*
             * This is VERY important, as otherwise tests can get ignored by "mvn test"
             */
        throw new IllegalArgumentException("Test classes should have name ending with 'Test'. Invalid input name: " + name);
    }
    List<File> generated = new ArrayList<File>();
    String dir = TestSuiteWriterUtils.makeDirectory(directory);
    String content = "";
    // Execute all tests
    executor.newObservers();
    // be sure it is active here, as JUnit checks might have left it to false
    LoopCounter.getInstance().setActive(true);
    List<ExecutionResult> results = new ArrayList<>();
    for (int i = 0; i < testCases.size(); i++) {
        TestCase test = testCases.get(i);
        boolean added = false;
        if (!TimeController.getInstance().hasTimeToExecuteATestCase()) {
            logger.info("Using cached result");
            for (ExecutionResult result : cachedResults) {
                if (result != null && result.test == test) {
                    results.add(result);
                    added = true;
                    break;
                }
            }
        }
        if (!added) {
            ExecutionResult result = runTest(test);
            results.add(result);
        }
    }
    if (Properties.TEST_NAMING_STRATEGY == Properties.TestNamingStrategy.NUMBERED) {
        nameGenerator = new NumberedTestNameGenerationStrategy(testCases, results);
    } else if (Properties.TEST_NAMING_STRATEGY == Properties.TestNamingStrategy.COVERAGE) {
        nameGenerator = new CoverageGoalTestNameGenerationStrategy(testCases, results);
    } else {
        throw new RuntimeException("Unsupported naming strategy: " + Properties.TEST_NAMING_STRATEGY);
    }
    // Avoid downcasts that could break
    removeUnnecessaryDownCasts(results);
    // Sometimes some timeouts lead to assertions being attached to statements
    // related to exceptions. This is not currently handled, so as a workaround
    // let's try to remove any remaining assertions. TODO: Better solution
    removeAssertionsAfterException(results);
    if (Properties.OUTPUT_GRANULARITY == OutputGranularity.MERGED || testCases.size() == 0) {
        File file = new File(dir + "/" + name + ".java");
        // executor.newObservers();
        content = getUnitTestsAllInSameFile(name, results);
        FileIOUtils.writeFile(content, file);
        generated.add(file);
    } else {
        for (int i = 0; i < testCases.size(); i++) {
            // e.g., dir/Foo_ESTest_0.java
            File file = new File(dir + "/" + name + "_" + i + ".java");
            // executor.newObservers();
            String testCode = getOneUnitTestInAFile(name, i, results);
            FileIOUtils.writeFile(testCode, file);
            content += testCode;
            generated.add(file);
        }
    }
    if (Properties.TEST_SCAFFOLDING && !Properties.NO_RUNTIME_DEPENDENCY) {
        String scaffoldingName = Scaffolding.getFileName(name);
        File file = new File(dir + "/" + scaffoldingName + ".java");
        String scaffoldingContent = Scaffolding.getScaffoldingFileContent(name, results, TestSuiteWriterUtils.hasAnySecurityException(results));
        FileIOUtils.writeFile(scaffoldingContent, file);
        generated.add(file);
        content += scaffoldingContent;
    }
    writeCoveredGoalsFile();
    TestGenerationResultBuilder.getInstance().setTestSuiteCode(content);
    return generated;
}

Example 79 with ExecutionResult

use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.

the class TestSuiteWriter method getImports.

// -----------------------------------------------------------
// --------------   code generation methods ------------------
// -----------------------------------------------------------
/**
 * Determine packages that need to be imported in the JUnit file
 *
 * @param results a {@link java.util.List} object.
 * @return a {@link java.lang.String} object.
 */
protected String getImports(List<ExecutionResult> results) {
    StringBuilder builder = new StringBuilder();
    Set<Class<?>> imports = new HashSet<Class<?>>();
    Set<Class<?>> accessedClasses = new HashSet<Class<?>>();
    boolean wasSecurityException = TestSuiteWriterUtils.hasAnySecurityException(results);
    boolean hasException = false;
    for (ExecutionResult result : results) {
        visitor.clearExceptions();
        visitor.setExceptions(result.exposeExceptionMapping());
        result.test.accept(visitor);
        imports.addAll(visitor.getImports());
        accessedClasses.addAll(result.test.getAccessedClasses());
        if (!hasException)
            hasException = !result.noThrownExceptions();
    }
    visitor.clearExceptions();
    if (doesUseMocks(results)) {
        String mockito = Mockito.class.getCanonicalName();
        builder.append("import static " + mockito + ".*;" + NEWLINE);
        // MockitoExtension is now deprecated
        // String extension = MockitoExtension.class.getCanonicalName();
        // builder.append("import static "+extension+".*;"+NEWLINE);
        imports.add(ViolatedAssumptionAnswer.class);
    }
    if (hasException && !Properties.NO_RUNTIME_DEPENDENCY) {
        builder.append("import static " + EvoAssertions.class.getCanonicalName() + ".*;" + NEWLINE);
    }
    if (Properties.RESET_STANDARD_STREAMS) {
        imports.add(PrintStream.class);
        imports.add(DebugGraphics.class);
    }
    if (TestSuiteWriterUtils.needToUseAgent() && !Properties.NO_RUNTIME_DEPENDENCY) {
        imports.add(EvoRunner.class);
        imports.add(EvoRunnerParameters.class);
        imports.add(RunWith.class);
    }
    Set<String> importNames = new HashSet<>();
    for (Class<?> imp : imports) {
        while (imp.isArray()) imp = imp.getComponentType();
        if (imp.isPrimitive())
            continue;
        if (imp.getName().startsWith("java.lang")) {
            String name = imp.getName().replace("java.lang.", "");
            if (!name.contains("."))
                continue;
        }
        if (!imp.getName().contains("."))
            continue;
        // TODO: Check for anonymous type?
        if (imp.getName().contains("$"))
            importNames.add(imp.getName().replace("$", "."));
        else
            importNames.add(imp.getName());
    }
    for (Class<?> klass : EnvironmentDataList.getListOfClasses()) {
        // TODO: not paramount, but best if could check if actually used in the test suite
        if (accessedClasses.contains(klass))
            importNames.add(klass.getCanonicalName());
    }
    if (wasSecurityException) {
        // Add import info for EvoSuite classes used in the generated test suite
        importNames.add(java.util.concurrent.ExecutorService.class.getCanonicalName());
        importNames.add(java.util.concurrent.Executors.class.getCanonicalName());
        importNames.add(java.util.concurrent.Future.class.getCanonicalName());
        importNames.add(java.util.concurrent.TimeUnit.class.getCanonicalName());
    }
    if (!Properties.TEST_SCAFFOLDING && !Properties.NO_RUNTIME_DEPENDENCY) {
        importNames.addAll(Scaffolding.getScaffoldingImports(wasSecurityException, results));
    }
    // If a CodeUnderTestException happens, the test will be chopped before that exception
    // but it would still be in the imports
    importNames.remove(CodeUnderTestException.class.getCanonicalName());
    List<String> importsSorted = new ArrayList<>(importNames);
    Collections.sort(importsSorted);
    for (String imp : importsSorted) {
        builder.append("import ");
        builder.append(imp);
        builder.append(";");
        builder.append(NEWLINE);
    }
    builder.append(NEWLINE);
    return builder.toString();
}

Example 80 with ExecutionResult

use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.

the class BranchCoverageSuiteFitness method analyzeTraces.

/**
 * Iterate over all execution results and summarize statistics
 *
 * @param results
 * @param predicateCount
 * @param callCount
 * @param trueDistance
 * @param falseDistance
 * @return
 */
private boolean analyzeTraces(AbstractTestSuiteChromosome<? extends ExecutableChromosome> suite, List<ExecutionResult> results, Map<Integer, Integer> predicateCount, Map<String, Integer> callCount, Map<Integer, Double> trueDistance, Map<Integer, Double> falseDistance) {
    boolean hasTimeoutOrTestException = false;
    for (ExecutionResult result : results) {
        if (result.hasTimeout() || result.hasTestException()) {
            hasTimeoutOrTestException = true;
            continue;
        }
        TestChromosome test = new TestChromosome();
        test.setTestCase(result.test);
        test.setLastExecutionResult(result);
        test.setChanged(false);
        handleBranchlessMethods(test, result, callCount);
        handlePredicateCount(result, predicateCount);
        handleTrueDistances(test, result, trueDistance);
        handleFalseDistances(test, result, falseDistance);
        // In case there were exceptions in a constructor
        handleConstructorExceptions(test, result, callCount);
    }
    return hasTimeoutOrTestException;
}