Examples with ExecutionResult org.evosuite.testcase.execution.ExecutionResult used on opensource projects

Example 1 with ExecutionResult

use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.

the class TestSuiteGenerator method postProcessTests.

/**
 * Apply any readability optimizations and other techniques that should use
 * or modify the generated tests
 *
 * @param testSuite
 */
protected void postProcessTests(TestSuiteChromosome testSuite) {
    // If overall time is short, the search might not have had enough time
    // to come up with a suite without timeouts. However, they will slow
    // down
    // the rest of the process, and may lead to invalid tests
    testSuite.getTestChromosomes().removeIf(t -> t.getLastExecutionResult() != null && (t.getLastExecutionResult().hasTimeout() || t.getLastExecutionResult().hasTestException()));
    if (Properties.CTG_SEEDS_FILE_OUT != null) {
        TestSuiteSerialization.saveTests(testSuite, new File(Properties.CTG_SEEDS_FILE_OUT));
    } else if (Properties.TEST_FACTORY == TestFactory.SERIALIZATION) {
        TestSuiteSerialization.saveTests(testSuite, new File(Properties.SEED_DIR + File.separator + Properties.TARGET_CLASS));
    }
    /*
		 * Remove covered goals that are not part of the minimization targets,
		 * as they might screw up coverage analysis when a minimization timeout
		 * occurs. This may happen e.g. when MutationSuiteFitness calls
		 * BranchCoverageSuiteFitness which adds branch goals.
		 */
    // TODO: This creates an inconsistency between
    // suite.getCoveredGoals().size() and suite.getNumCoveredGoals()
    // but it is not clear how to update numcoveredgoals
    List<TestFitnessFunction> goals = new ArrayList<>();
    for (TestFitnessFactory<?> ff : getFitnessFactories()) {
        goals.addAll(ff.getCoverageGoals());
    }
    for (TestFitnessFunction f : testSuite.getCoveredGoals()) {
        if (!goals.contains(f)) {
            testSuite.removeCoveredGoal(f);
        }
    }
    if (Properties.INLINE) {
        ClientServices.getInstance().getClientNode().changeState(ClientState.INLINING);
        ConstantInliner inliner = new ConstantInliner();
        // progressMonitor.setCurrentPhase("Inlining constants");
        // Map<FitnessFunction<? extends TestSuite<?>>, Double> fitnesses =
        // testSuite.getFitnesses();
        inliner.inline(testSuite);
    }
    if (Properties.MINIMIZE) {
        ClientServices.getInstance().getClientNode().changeState(ClientState.MINIMIZATION);
        // progressMonitor.setCurrentPhase("Minimizing test cases");
        if (!TimeController.getInstance().hasTimeToExecuteATestCase()) {
            LoggingUtils.getEvoLogger().info("* Skipping minimization because not enough time is left");
            ClientServices.track(RuntimeVariable.Result_Size, testSuite.size());
            ClientServices.track(RuntimeVariable.Minimized_Size, testSuite.size());
            ClientServices.track(RuntimeVariable.Result_Length, testSuite.totalLengthOfTestCases());
            ClientServices.track(RuntimeVariable.Minimized_Length, testSuite.totalLengthOfTestCases());
        } else if (Properties.isRegression()) {
            RegressionSuiteMinimizer minimizer = new RegressionSuiteMinimizer();
            minimizer.minimize(testSuite);
        } else {
            double before = testSuite.getFitness();
            TestSuiteMinimizer minimizer = new TestSuiteMinimizer(getFitnessFactories());
            LoggingUtils.getEvoLogger().info("* Minimizing test suite");
            minimizer.minimize(testSuite, true);
            double after = testSuite.getFitness();
            if (after > before + 0.01d) {
                // assume minimization
                throw new Error("EvoSuite bug: minimization lead fitness from " + before + " to " + after);
            }
        }
    } else {
        if (!TimeController.getInstance().hasTimeToExecuteATestCase()) {
            LoggingUtils.getEvoLogger().info("* Skipping minimization because not enough time is left");
        }
        ClientServices.track(RuntimeVariable.Result_Size, testSuite.size());
        ClientServices.track(RuntimeVariable.Minimized_Size, testSuite.size());
        ClientServices.track(RuntimeVariable.Result_Length, testSuite.totalLengthOfTestCases());
        ClientServices.track(RuntimeVariable.Minimized_Length, testSuite.totalLengthOfTestCases());
    }
    if (Properties.COVERAGE) {
        ClientServices.getInstance().getClientNode().changeState(ClientState.COVERAGE_ANALYSIS);
        CoverageCriteriaAnalyzer.analyzeCoverage(testSuite);
    }
    double coverage = testSuite.getCoverage();
    if (ArrayUtil.contains(Properties.CRITERION, Criterion.MUTATION) || ArrayUtil.contains(Properties.CRITERION, Criterion.STRONGMUTATION)) {
    // SearchStatistics.getInstance().mutationScore(coverage);
    }
    StatisticsSender.executedAndThenSendIndividualToMaster(testSuite);
    LoggingUtils.getEvoLogger().info("* Generated " + testSuite.size() + " tests with total length " + testSuite.totalLengthOfTestCases());
    // TODO: In the end we will only need one analysis technique
    if (!Properties.ANALYSIS_CRITERIA.isEmpty()) {
        // SearchStatistics.getInstance().addCoverage(Properties.CRITERION.toString(),
        // coverage);
        CoverageCriteriaAnalyzer.analyzeCriteria(testSuite, Properties.ANALYSIS_CRITERIA);
    // FIXME: can we send all bestSuites?
    }
    if (Properties.CRITERION.length > 1)
        LoggingUtils.getEvoLogger().info("* Resulting test suite's coverage: " + NumberFormat.getPercentInstance().format(coverage) + " (average coverage for all fitness functions)");
    else
        LoggingUtils.getEvoLogger().info("* Resulting test suite's coverage: " + NumberFormat.getPercentInstance().format(coverage));
    // printBudget(ga); // TODO - need to move this somewhere else
    if (ArrayUtil.contains(Properties.CRITERION, Criterion.DEFUSE) && Properties.ANALYSIS_CRITERIA.isEmpty())
        DefUseCoverageSuiteFitness.printCoverage();
    DSEStats.getInstance().trackConstraintTypes();
    DSEStats.getInstance().trackSolverStatistics();
    if (Properties.DSE_PROBABILITY > 0.0 && Properties.LOCAL_SEARCH_RATE > 0 && Properties.LOCAL_SEARCH_PROBABILITY > 0.0) {
        DSEStats.getInstance().logStatistics();
    }
    if (Properties.FILTER_SANDBOX_TESTS) {
        for (TestChromosome test : testSuite.getTestChromosomes()) {
            // delete all statements leading to security exceptions
            ExecutionResult result = test.getLastExecutionResult();
            if (result == null) {
                result = TestCaseExecutor.runTest(test.getTestCase());
            }
            if (result.hasSecurityException()) {
                int position = result.getFirstPositionOfThrownException();
                if (position > 0) {
                    test.getTestCase().chop(position);
                    result = TestCaseExecutor.runTest(test.getTestCase());
                    test.setLastExecutionResult(result);
                }
            }
        }
    }
    if (Properties.ASSERTIONS && !Properties.isRegression()) {
        LoggingUtils.getEvoLogger().info("* Generating assertions");
        // progressMonitor.setCurrentPhase("Generating assertions");
        ClientServices.getInstance().getClientNode().changeState(ClientState.ASSERTION_GENERATION);
        if (!TimeController.getInstance().hasTimeToExecuteATestCase()) {
            LoggingUtils.getEvoLogger().info("* Skipping assertion generation because not enough time is left");
        } else {
            TestSuiteGeneratorHelper.addAssertions(testSuite);
        }
        // FIXME: can we
        StatisticsSender.sendIndividualToMaster(testSuite);
    // pass the list
    // of
    // testsuitechromosomes?
    }
    if (Properties.NO_RUNTIME_DEPENDENCY) {
        LoggingUtils.getEvoLogger().info("* Property NO_RUNTIME_DEPENDENCY is set to true - skipping JUnit compile check");
        LoggingUtils.getEvoLogger().info("* WARNING: Not including the runtime dependencies is likely to lead to flaky tests!");
    } else if (Properties.JUNIT_TESTS && Properties.JUNIT_CHECK) {
        compileAndCheckTests(testSuite);
    }
    if (Properties.SERIALIZE_REGRESSION_TEST_SUITE) {
        RegressionSuiteSerializer.appendToRegressionTestSuite(testSuite);
    }
    if (Properties.isRegression() && Properties.KEEP_REGRESSION_ARCHIVE) {
        RegressionSuiteSerializer.storeRegressionArchive();
    }
}

Example 2 with ExecutionResult

use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.

the class TestSuiteGenerator method initializeTargetClass.

private void initializeTargetClass() throws Throwable {
    String cp = ClassPathHandler.getInstance().getTargetProjectClasspath();
    // Here is where the <clinit> code should be invoked for the first time
    DefaultTestCase test = buildLoadTargetClassTestCase(Properties.TARGET_CLASS);
    ExecutionResult execResult = TestCaseExecutor.getInstance().execute(test, Integer.MAX_VALUE);
    if (hasThrownInitializerError(execResult)) {
        // create single test suite with Class.forName()
        writeJUnitTestSuiteForFailedInitialization();
        ExceptionInInitializerError ex = getInitializerError(execResult);
        throw ex;
    } else if (!execResult.getAllThrownExceptions().isEmpty()) {
        // some other exception has been thrown during initialization
        Throwable t = execResult.getAllThrownExceptions().iterator().next();
        throw t;
    }
    DependencyAnalysis.analyzeClass(Properties.TARGET_CLASS, Arrays.asList(cp.split(File.pathSeparator)));
    LoggingUtils.getEvoLogger().info("* Finished analyzing classpath");
}

Example 3 with ExecutionResult

use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.

the class OnlyBranchCoverageSuiteFitness method analyzeTraces.

/**
 * Iterate over all execution results and summarize statistics
 *
 * @param results
 * @param predicateCount
 * @param callCount
 * @param trueDistance
 * @param falseDistance
 * @return
 */
private boolean analyzeTraces(AbstractTestSuiteChromosome<? extends ExecutableChromosome> suite, List<ExecutionResult> results, Map<Integer, Integer> predicateCount, Map<Integer, Double> trueDistance, Map<Integer, Double> falseDistance) {
    boolean hasTimeoutOrTestException = false;
    for (ExecutionResult result : results) {
        if (result.hasTimeout() || result.hasTestException()) {
            hasTimeoutOrTestException = true;
            continue;
        }
        TestChromosome test = new TestChromosome();
        test.setTestCase(result.test);
        test.setLastExecutionResult(result);
        test.setChanged(false);
        for (Entry<Integer, Integer> entry : result.getTrace().getPredicateExecutionCount().entrySet()) {
            if (!branchesId.contains(entry.getKey()) || (removedBranchesT.contains(entry.getKey()) && removedBranchesF.contains(entry.getKey())))
                continue;
            if (!predicateCount.containsKey(entry.getKey()))
                predicateCount.put(entry.getKey(), entry.getValue());
            else {
                predicateCount.put(entry.getKey(), predicateCount.get(entry.getKey()) + entry.getValue());
            }
        }
        for (Entry<Integer, Double> entry : result.getTrace().getTrueDistances().entrySet()) {
            if (!branchesId.contains(entry.getKey()) || removedBranchesT.contains(entry.getKey()))
                continue;
            if (!trueDistance.containsKey(entry.getKey()))
                trueDistance.put(entry.getKey(), entry.getValue());
            else {
                trueDistance.put(entry.getKey(), Math.min(trueDistance.get(entry.getKey()), entry.getValue()));
            }
            OnlyBranchCoverageTestFitness goal = (OnlyBranchCoverageTestFitness) branchCoverageTrueMap.get(entry.getKey());
            if ((Double.compare(entry.getValue(), 0.0) == 0)) {
                test.getTestCase().addCoveredGoal(goal);
                toRemoveBranchesT.add(entry.getKey());
            }
            if (Properties.TEST_ARCHIVE) {
                Archive.getArchiveInstance().updateArchive(goal, test, entry.getValue());
            }
        }
        for (Entry<Integer, Double> entry : result.getTrace().getFalseDistances().entrySet()) {
            if (!branchesId.contains(entry.getKey()) || removedBranchesF.contains(entry.getKey()))
                continue;
            if (!falseDistance.containsKey(entry.getKey()))
                falseDistance.put(entry.getKey(), entry.getValue());
            else {
                falseDistance.put(entry.getKey(), Math.min(falseDistance.get(entry.getKey()), entry.getValue()));
            }
            OnlyBranchCoverageTestFitness goal = (OnlyBranchCoverageTestFitness) branchCoverageFalseMap.get(entry.getKey());
            if ((Double.compare(entry.getValue(), 0.0) == 0)) {
                test.getTestCase().addCoveredGoal(goal);
                toRemoveBranchesF.add(entry.getKey());
            }
            if (Properties.TEST_ARCHIVE) {
                Archive.getArchiveInstance().updateArchive(goal, test, entry.getValue());
            }
        }
    }
    return hasTimeoutOrTestException;
}

Example 4 with ExecutionResult

use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.

the class AmbiguityCoverageSuiteFitness method getFitness.

@Override
public double getFitness(AbstractTestSuiteChromosome<? extends ExecutableChromosome> suite) {
    List<StringBuilder> transposedMatrix = new ArrayList<StringBuilder>(AmbiguityCoverageFactory.getTransposedMatrix());
    List<Set<Integer>> coveredLines = new ArrayList<Set<Integer>>();
    // Execute test cases and collect the covered lines
    List<ExecutionResult> results = runTestSuite(suite);
    for (ExecutionResult result : results) {
        coveredLines.add(result.getTrace().getCoveredLines());
    }
    Map<String, Integer> groups = new HashMap<String, Integer>();
    int g_i = 0;
    for (Integer goal : this.goals) {
        StringBuffer str = null;
        if (transposedMatrix.size() > g_i) {
            str = new StringBuffer(transposedMatrix.get(g_i).length() + coveredLines.size());
            str.append(transposedMatrix.get(g_i));
        } else {
            str = new StringBuffer(coveredLines.size());
        }
        for (Set<Integer> covered : coveredLines) {
            str.append(covered.contains(goal) ? "1" : "0");
        }
        if (!groups.containsKey(str.toString())) {
            // in the beginning they are ambiguity, so they belong to the same group '1'
            groups.put(str.toString(), 1);
        } else {
            groups.put(str.toString(), groups.get(str.toString()) + 1);
        }
        g_i++;
    }
    // double fitness = AmbiguityCoverageFactory.getAmbiguity(this.goals.size(), groups) * 1.0 / AmbiguityCoverageFactory.getMaxAmbiguityScore();
    double fitness = TestFitnessFunction.normalize(AmbiguityCoverageFactory.getAmbiguity(this.goals.size(), groups));
    updateIndividual(this, suite, fitness);
    return fitness;
}

Example 5 with ExecutionResult

use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.

the class CompleteAssertionGenerator method addAssertions.

/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * org.evosuite.assertion.AssertionGenerator#addAssertions(org.evosuite.
	 * testcase.TestCase)
	 */
/**
 * {@inheritDoc}
 */
@Override
public void addAssertions(TestCase test) {
    ExecutionResult result = runTest(test);
    for (OutputTrace<?> trace : result.getTraces()) {
        trace.getAllAssertions(test);
        trace.clear();
    }
    logger.debug("Test after adding assertions: " + test.toCode());
}