use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.
the class DefUseCoverageSuiteFitness method getFitnessAlternative.
// Not working yet
// @Override
public double getFitnessAlternative(AbstractTestSuiteChromosome<? extends ExecutableChromosome> individual) {
TestSuiteChromosome suite = (TestSuiteChromosome) individual;
List<ExecutionResult> results = runTestSuite(suite);
if (DefUseCoverageFactory.detectAliasingGoals(results)) {
logger.debug("New total number of goals: " + goals.size());
totalGoals = initTotalGoals();
for (DefUsePairType type : totalGoals.keySet()) {
logger.info(type + ":" + totalGoals.get(type));
}
}
Map<Definition, Set<TestChromosome>> passedDefinitions = new HashMap<Definition, Set<TestChromosome>>();
Map<Definition, Integer> passedDefinitionCount = new HashMap<Definition, Integer>();
Map<String, Set<TestChromosome>> executedMethods = new HashMap<String, Set<TestChromosome>>();
Map<String, Integer> executedMethodsCount = new HashMap<String, Integer>();
for (Definition def : maxDefinitionCount.keySet()) {
passedDefinitionCount.put(def, 0);
}
for (String methodName : maxMethodCount.keySet()) {
executedMethodsCount.put(methodName, 0);
}
for (TestChromosome test : suite.getTestChromosomes()) {
ExecutionResult result = test.getLastExecutionResult();
if (result.hasTimeout()) {
logger.debug("Skipping test with timeout");
double fitness = goals.size() * 100;
updateIndividual(this, individual, fitness);
suite.setCoverage(this, 0.0);
logger.debug("Test case has timed out, setting fitness to max value " + fitness);
return fitness;
}
for (Entry<Integer, Integer> entry : result.getTrace().getDefinitionExecutionCount().entrySet()) {
Definition def = DefUsePool.getDefinitionByDefId(entry.getKey());
if (def == null) {
logger.warn("Could not find def " + entry.getKey());
continue;
}
if (!passedDefinitions.containsKey(def))
passedDefinitions.put(def, new HashSet<TestChromosome>());
if (!passedDefinitionCount.containsKey(def)) {
// logger.warn("Weird, definition is not known: " + def);
passedDefinitionCount.put(def, 0);
}
passedDefinitions.get(def).add(test);
passedDefinitionCount.put(def, passedDefinitionCount.get(def) + entry.getValue());
}
for (Entry<String, Integer> entry : result.getTrace().getMethodExecutionCount().entrySet()) {
if (executedMethodsCount.containsKey(entry.getKey()))
executedMethodsCount.put(entry.getKey(), executedMethodsCount.get(entry.getKey()) + entry.getValue());
if (!executedMethods.containsKey(entry.getKey())) {
executedMethods.put(entry.getKey(), new HashSet<TestChromosome>());
}
executedMethods.get(entry.getKey()).add(test);
}
/*
for (Integer id : result.getTrace().getPassedDefIDs()) {
Definition def = DefUsePool.getDefinitionByDefId(id);
if (!passedDefinitions.containsKey(def))
passedDefinitions.put(def, new HashSet<TestChromosome>());
passedDefinitions.get(def).add(test);
passedDefinitionCount.put(def, passedDefinitionCount.get(def) + 1);
}
*/
}
// 1. Need to reach each definition
double fitness = branchFitness.getFitness(individual);
// logger.info("Branch fitness: " + fitness);
// 3. For all covered defs, calculate minimal use distance
// Set<DefUseCoverageTestFitness> coveredGoalsSet = DefUseExecutionTraceAnalyzer.getCoveredGoals(results);
// DefUseExecutionTraceAnalyzer.getCoveredGoals(results);
Set<DefUseCoverageTestFitness> coveredGoalsSet = new HashSet<DefUseCoverageTestFitness>();
initCoverageMaps();
Set<Definition> notFullyCoveredDefs = new HashSet<Definition>();
boolean methodIsNotFullyCovered = false;
for (DefUseCoverageTestFitness goal : goals) {
if (coveredGoalsSet.contains(goal)) {
continue;
}
double goalFitness = 2.0;
Set<TestChromosome> coveringTests = new HashSet<TestChromosome>();
if (goal.isParameterGoal()) {
String methodKey = goal.getGoalUse().getClassName() + "." + goal.getGoalUse().getMethodName();
if (executedMethods.containsKey(methodKey)) {
coveringTests.addAll(executedMethods.get(methodKey));
}
} else {
if (passedDefinitions.containsKey(goal.getGoalDefinition())) {
coveringTests.addAll(passedDefinitions.get(goal.getGoalDefinition()));
}
}
if (coveringTests.isEmpty()) {
logger.debug("No tests cover " + goal);
} else {
logger.debug("Checking " + coveringTests.size() + " tests covering " + goal);
}
// for (TestChromosome test : passedDefinitions.get(goal.getGoalDefinition())) {
for (TestChromosome test : coveringTests) {
// for (TestChromosome test : suite.getTestChromosomes()) {
ExecutionResult result = test.getLastExecutionResult();
DefUseFitnessCalculator calculator = new DefUseFitnessCalculator(goal, test, result);
// double resultFitness = goal.getFitness(test, result);
double resultFitness = calculator.calculateDUFitness();
if (resultFitness < goalFitness)
goalFitness = resultFitness;
if (goalFitness == 0.0) {
result.test.addCoveredGoal(goal);
coveredGoalsSet.add(goal);
break;
}
}
if (goalFitness > 0.0) {
if (goal.isParameterGoal())
notFullyCoveredDefs.add(goal.getGoalDefinition());
else
methodIsNotFullyCovered = true;
}
fitness += goalFitness;
}
// TODO ...unless all defuse pairs are covered?
for (Entry<Definition, Integer> defCount : maxDefinitionCount.entrySet()) {
if (notFullyCoveredDefs.contains(defCount.getKey())) {
int executionCount = passedDefinitionCount.get(defCount.getKey());
int max = defCount.getValue();
if (executionCount < max) {
fitness += normalize(max - executionCount);
}
}
}
if (methodIsNotFullyCovered) {
for (Entry<String, Integer> methodCount : maxMethodCount.entrySet()) {
int executionCount = executedMethodsCount.get(methodCount.getKey());
int max = methodCount.getValue();
if (executionCount < max) {
fitness += normalize(max - executionCount);
}
}
}
countCoveredGoals(coveredGoalsSet);
trackCoverageStatistics(suite);
updateIndividual(this, individual, fitness);
int coveredGoalCount = countCoveredGoals();
int totalGoalCount = countTotalGoals();
if (fitness == 0.0 && coveredGoalCount < totalGoalCount)
throw new IllegalStateException("Fitness 0 implies 100% coverage " + coveredGoalCount + " / " + totalGoals + " (covered / total)");
return fitness;
}
use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.
the class DefUseCoverageSuiteFitness method getFitnessOld.
/*
* (non-Javadoc)
*
* @see
* org.evosuite.ga.FitnessFunction#getFitness(org.
* evosuite.ga.Chromosome)
*/
/**
* {@inheritDoc}
*/
// @Override
public double getFitnessOld(Chromosome individual) {
logger.trace("Calculating defuse fitness");
TestSuiteChromosome suite = (TestSuiteChromosome) individual;
List<ExecutionResult> results = runTestSuite(suite);
double fitness = 0.0;
if (DefUseCoverageFactory.detectAliasingGoals(results)) {
goals = DefUseCoverageFactory.getDUGoals();
logger.debug("New total number of goals: " + goals.size());
totalGoals = initTotalGoals();
for (DefUsePairType type : totalGoals.keySet()) {
logger.info(type + ":" + totalGoals.get(type));
}
}
Set<DefUseCoverageTestFitness> coveredGoalsSet = DefUseExecutionTraceAnalyzer.getCoveredGoals(results);
initCoverageMaps();
for (DefUseCoverageTestFitness goal : goals) {
if (coveredGoalsSet.contains(goal)) {
goal.setCovered(true);
continue;
}
double goalFitness = 2.0;
for (ExecutionResult result : results) {
TestChromosome tc = new TestChromosome();
tc.setTestCase(result.test);
double resultFitness = goal.getFitness(tc, result);
if (resultFitness < goalFitness)
goalFitness = resultFitness;
if (goalFitness == 0.0) {
result.test.addCoveredGoal(goal);
coveredGoalsSet.add(goal);
goal.setCovered(true);
break;
}
}
fitness += goalFitness;
}
countCoveredGoals(coveredGoalsSet);
trackCoverageStatistics(suite);
updateIndividual(this, suite, fitness);
int coveredGoalCount = countCoveredGoals();
int totalGoalCount = countTotalGoals();
if (fitness == 0.0 && coveredGoalCount < totalGoalCount)
throw new IllegalStateException("Fitness 0 implies 100% coverage " + coveredGoalCount + " / " + totalGoals + " (covered / total)");
return fitness;
}
use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.
the class SimpleMutationAssertionGenerator method addAssertions.
/**
* Add assertions to current test set for given set of mutants
*
* @param test
* a {@link org.evosuite.testcase.TestCase} object.
* @param killed
* a {@link java.util.Set} object.
* @param mutants
* a {@link java.util.Map} object.
*/
private void addAssertions(TestCase test, Set<Integer> killed, Map<Integer, Mutation> mutants) {
if (test.isEmpty())
return;
logger.debug("Generating assertions");
int s1 = killed.size();
logger.debug("Running on original");
ExecutionResult origResult = runTest(test);
if (origResult.hasTimeout() || origResult.hasTestException()) {
logger.debug("Skipping test, as it has timeouts or exceptions");
return;
}
Map<Mutation, List<OutputTrace<?>>> mutationTraces = new HashMap<Mutation, List<OutputTrace<?>>>();
List<Mutation> executedMutants = new ArrayList<Mutation>();
for (Integer mutationId : origResult.getTrace().getTouchedMutants()) {
if (!mutants.containsKey(mutationId)) {
// logger.warn("Mutation ID unknown: " + mutationId);
// logger.warn(mutants.keySet().toString());
} else
executedMutants.add(mutants.get(mutationId));
}
Randomness.shuffle(executedMutants);
logger.debug("Executed mutants: " + origResult.getTrace().getTouchedMutants());
int numExecutedMutants = 0;
for (Mutation m : executedMutants) {
numExecutedMutants++;
if (!TimeController.getInstance().isThereStillTimeInThisPhase()) {
logger.info("Reached maximum time to generate assertions!");
break;
}
assert (m != null);
if (MutationTimeoutStoppingCondition.isDisabled(m)) {
killed.add(m.getId());
continue;
}
if (timedOutMutations.containsKey(m)) {
if (timedOutMutations.get(m) >= Properties.MUTATION_TIMEOUTS) {
logger.debug("Skipping timed out mutant");
killed.add(m.getId());
continue;
}
}
if (exceptionMutations.containsKey(m)) {
if (exceptionMutations.get(m) >= Properties.MUTATION_TIMEOUTS) {
logger.debug("Skipping mutant with exceptions");
killed.add(m.getId());
continue;
}
}
if (Properties.MAX_MUTANTS_PER_TEST > 0 && numExecutedMutants > Properties.MAX_MUTANTS_PER_TEST)
break;
/*
if (killed.contains(m.getId())) {
logger.info("Skipping dead mutant");
continue;
}
*/
logger.debug("Running test on mutation {}", m.getMutationName());
ExecutionResult mutantResult = runTest(test, m);
int numKilled = 0;
for (Class<?> observerClass : observerClasses) {
if (mutantResult.getTrace(observerClass) == null || origResult.getTrace(observerClass) == null)
continue;
numKilled += origResult.getTrace(observerClass).getAssertions(test, mutantResult.getTrace(observerClass));
}
List<OutputTrace<?>> traces = new ArrayList<OutputTrace<?>>(mutantResult.getTraces());
mutationTraces.put(m, traces);
if (mutantResult.hasTimeout()) {
logger.debug("Increasing timeout count!");
if (!timedOutMutations.containsKey(m)) {
timedOutMutations.put(m, 1);
} else {
timedOutMutations.put(m, timedOutMutations.get(m) + 1);
}
MutationTimeoutStoppingCondition.timeOut(m);
} else if (!mutantResult.noThrownExceptions() && origResult.noThrownExceptions()) {
logger.debug("Increasing exception count.");
if (!exceptionMutations.containsKey(m)) {
exceptionMutations.put(m, 1);
} else {
exceptionMutations.put(m, exceptionMutations.get(m) + 1);
}
MutationTimeoutStoppingCondition.raisedException(m);
}
if (numKilled > 0 || mutantResult.hasTimeout() || (!mutantResult.noThrownExceptions() && origResult.noThrownExceptions())) {
killed.add(m.getId());
}
}
List<Assertion> assertions = test.getAssertions();
logger.info("Got " + assertions.size() + " assertions");
Map<Integer, Set<Integer>> killMap = new HashMap<Integer, Set<Integer>>();
int num = 0;
for (Assertion assertion : assertions) {
Set<Integer> killedMutations = new HashSet<Integer>();
for (Mutation m : executedMutants) {
boolean isKilled = false;
if (mutationTraces.containsKey(m)) {
for (OutputTrace<?> trace : mutationTraces.get(m)) {
if (trace.isDetectedBy(assertion)) {
isKilled = true;
break;
}
}
}
if (isKilled) {
killedMutations.add(m.getId());
assertion.addKilledMutation(m);
}
}
killMap.put(num, killedMutations);
// logger.info("Assertion " + num + " kills mutants " + killedMutations);
num++;
}
int killedBefore = getNumKilledMutants(test, mutationTraces, executedMutants);
logger.debug("Need to kill mutants: " + killedBefore);
logger.debug(killMap.toString());
minimize(test, executedMutants, assertions, killMap);
int killedAfter = getNumKilledMutants(test, mutationTraces, executedMutants);
int s2 = killed.size() - s1;
assert (killedBefore == killedAfter) : "Mutants killed before / after / should be: " + killedBefore + "/" + killedAfter + "/" + s2 + ": " + test.toCode();
logger.info("Mutants killed before / after / should be: " + killedBefore + "/" + killedAfter + "/" + s2);
logger.info("Assertions in this test: " + test.getAssertions().size());
if (primitiveWithoutAssertion(test.getStatement(test.size() - 1))) {
logger.info("Last statement has primitive return value but no assertions: " + test.toCode());
for (Assertion assertion : assertions) {
if (assertion instanceof PrimitiveAssertion) {
if (assertion.getStatement().equals(test.getStatement(test.size() - 1))) {
logger.debug("Adding a primitive assertion " + assertion);
test.getStatement(test.size() - 1).addAssertion(assertion);
break;
}
}
}
filterInspectorPrimitiveDuplication(test.getStatement(test.size() - 1));
}
// IF there are no mutant killing assertions on the last statement, still assert something
if (test.getStatement(test.size() - 1).getAssertions().isEmpty() || justNullAssertion(test.getStatement(test.size() - 1))) {
logger.info("Last statement has no assertions: " + test.toCode());
logger.info("Assertions to choose from: " + assertions.size());
if (test.getStatement(test.size() - 1).getAssertions().isEmpty()) {
logger.debug("Last statement: " + test.getStatement(test.size() - 1).getCode());
}
if (origResult.isThereAnExceptionAtPosition(test.size() - 1))
logger.debug("Exception on last statement!");
if (justNullAssertion(test.getStatement(test.size() - 1)))
logger.debug("Just null assertions on last statement: " + test.toCode());
boolean haveAssertion = false;
for (Assertion assertion : assertions) {
if (assertion instanceof PrimitiveAssertion) {
if (assertion.getStatement().equals(test.getStatement(test.size() - 1))) {
logger.debug("Adding a primitive assertion " + assertion);
test.getStatement(test.size() - 1).addAssertion(assertion);
haveAssertion = true;
break;
}
}
}
if (!haveAssertion) {
logger.info("Could not find a primitive assertion, continuing search");
for (Assertion assertion : assertions) {
if (assertion instanceof NullAssertion)
continue;
if (assertion.getStatement().equals(test.getStatement(test.size() - 1))) {
logger.info("Adding an assertion: " + assertion);
test.getStatement(test.size() - 1).addAssertion(assertion);
haveAssertion = true;
break;
}
}
}
// if (!test.hasAssertions()) {
if (!haveAssertion) {
logger.info("After second round we still have no assertion");
Method inspectorMethod = null;
if (test.getStatement(test.size() - 1) instanceof MethodStatement) {
MethodStatement methodStatement = (MethodStatement) test.getStatement(test.size() - 1);
Method method = methodStatement.getMethod().getMethod();
if (method.getParameterTypes().length == 0) {
if (method.getReturnType().isPrimitive() && !method.getReturnType().equals(void.class)) {
inspectorMethod = method;
}
}
}
for (OutputTrace<?> trace : origResult.getTraces()) {
trace.getAllAssertions(test);
}
Set<Assertion> target = new HashSet<Assertion>(test.getStatement(test.size() - 1).getAssertions());
logger.debug("Found assertions: " + target.size());
test.removeAssertions();
// test.addAssertions(clone);
VariableReference targetVar = test.getStatement(test.size() - 1).getReturnValue();
if (!targetVar.isVoid()) {
logger.debug("Return value is non void: " + targetVar.getClassName());
int maxAssertions = 1;
int numAssertions = 0;
for (Assertion ass : target) {
if (ass.getReferencedVariables().contains(targetVar) && !(ass instanceof NullAssertion)) {
if (ass instanceof InspectorAssertion) {
if (((InspectorAssertion) ass).inspector.getMethod().equals(inspectorMethod)) {
continue;
}
}
test.getStatement(test.size() - 1).addAssertion(ass);
logger.debug("Adding assertion " + ass.getCode());
if (++numAssertions >= maxAssertions)
break;
} else {
logger.debug("Assertion does not contain target: " + ass.getCode());
}
}
if (numAssertions == 0) {
for (Assertion ass : target) {
if (ass.getReferencedVariables().contains(targetVar)) {
test.getStatement(test.size() - 1).addAssertion(ass);
logger.debug("Adding assertion " + ass.getCode());
if (++numAssertions >= maxAssertions)
break;
} else {
logger.debug("Assertion does not contain target: " + ass.getCode());
}
}
}
} else {
logger.debug("Return value is void");
Set<VariableReference> targetVars = test.getStatement(test.size() - 1).getVariableReferences();
int maxAssertions = 1;
int numAssertions = 0;
for (Assertion ass : target) {
Set<VariableReference> vars = ass.getReferencedVariables();
vars.retainAll(targetVars);
if (!vars.isEmpty()) {
test.getStatement(test.size() - 1).addAssertion(ass);
if (++numAssertions >= maxAssertions)
break;
}
}
}
logger.info("1. Done with assertions");
}
logger.info("2. Done with assertions");
filterInspectorPrimitiveDuplication(test.getStatement(test.size() - 1));
}
if (!origResult.noThrownExceptions()) {
if (!test.getStatement(test.size() - 1).getAssertions().isEmpty()) {
logger.debug("Removing assertions after exception");
test.getStatement(test.size() - 1).removeAssertions();
}
}
}
use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.
the class ExceptionCoverageSuiteFitness method calculateExceptionInfo.
/**
* Given the list of results, fill the 3 given (empty) maps with exception information.
* Also, add exception coverage goals to mapping in {@link ExceptionCoverageFactory}
*
* @param results
* @param implicitTypesOfExceptions
* @param explicitTypesOfExceptions
* @param declaredTypesOfExceptions
* @throws IllegalArgumentException
*/
public static void calculateExceptionInfo(List<ExecutionResult> results, Map<String, Set<Class<?>>> implicitTypesOfExceptions, Map<String, Set<Class<?>>> explicitTypesOfExceptions, Map<String, Set<Class<?>>> declaredTypesOfExceptions, ExceptionCoverageSuiteFitness contextFitness) throws IllegalArgumentException {
if (results == null || implicitTypesOfExceptions == null || explicitTypesOfExceptions == null || !implicitTypesOfExceptions.isEmpty() || !explicitTypesOfExceptions.isEmpty() || declaredTypesOfExceptions == null || !declaredTypesOfExceptions.isEmpty()) {
throw new IllegalArgumentException();
}
// for each test case
for (ExecutionResult result : results) {
// use reflection for basic criteria, not for exception
if (result.hasTimeout() || result.hasTestException() || result.noThrownExceptions() || result.calledReflection()) {
continue;
}
TestChromosome test = new TestChromosome();
test.setTestCase(result.test);
test.setLastExecutionResult(result);
test.setChanged(false);
// iterate on the indexes of the statements that resulted in an exception
for (Integer i : result.getPositionsWhereExceptionsWereThrown()) {
if (ExceptionCoverageHelper.shouldSkip(result, i)) {
continue;
}
Class<?> exceptionClass = ExceptionCoverageHelper.getExceptionClass(result, i);
// eg name+descriptor
String methodIdentifier = ExceptionCoverageHelper.getMethodIdentifier(result, i);
// was the exception originated by a direct call on the SUT?
boolean sutException = ExceptionCoverageHelper.isSutException(result, i);
if (sutException) {
boolean notDeclared = !ExceptionCoverageHelper.isDeclared(result, i);
if (notDeclared) {
/*
* we need to distinguish whether it is explicit (ie "throw" in the code, eg for validating
* input for pre-condition) or implicit ("likely" a real fault).
*/
boolean isExplicit = ExceptionCoverageHelper.isExplicit(result, i);
if (isExplicit) {
if (!explicitTypesOfExceptions.containsKey(methodIdentifier)) {
explicitTypesOfExceptions.put(methodIdentifier, new HashSet<Class<?>>());
}
explicitTypesOfExceptions.get(methodIdentifier).add(exceptionClass);
} else {
if (!implicitTypesOfExceptions.containsKey(methodIdentifier)) {
implicitTypesOfExceptions.put(methodIdentifier, new HashSet<Class<?>>());
}
implicitTypesOfExceptions.get(methodIdentifier).add(exceptionClass);
}
} else {
if (!declaredTypesOfExceptions.containsKey(methodIdentifier)) {
declaredTypesOfExceptions.put(methodIdentifier, new HashSet<Class<?>>());
}
declaredTypesOfExceptions.get(methodIdentifier).add(exceptionClass);
}
ExceptionCoverageTestFitness.ExceptionType type = ExceptionCoverageHelper.getType(result, i);
/*
* Add goal to ExceptionCoverageFactory
*/
ExceptionCoverageTestFitness goal = new ExceptionCoverageTestFitness(Properties.TARGET_CLASS, methodIdentifier, exceptionClass, type);
String key = goal.getKey();
if (!ExceptionCoverageFactory.getGoals().containsKey(key)) {
ExceptionCoverageFactory.getGoals().put(key, goal);
test.getTestCase().addCoveredGoal(goal);
if (Properties.TEST_ARCHIVE && contextFitness != null) {
Archive.getArchiveInstance().addTarget(goal);
Archive.getArchiveInstance().updateArchive(goal, test, 0.0);
}
}
}
}
}
}
use of org.evosuite.testcase.execution.ExecutionResult in project evosuite by EvoSuite.
the class DefUseCoverageFactory method detectAliasingGoals.
public static boolean detectAliasingGoals(List<ExecutionResult> results) {
if (!Properties.DEFUSE_ALIASES)
return false;
Set<DefUseCoverageTestFitness> aliasingGoals = new HashSet<DefUseCoverageTestFitness>();
for (ExecutionResult result : results) {
aliasingGoals.addAll(detectAliasingGoals(result));
}
// Need to add here to avoid concurrent access
if (!aliasingGoals.isEmpty()) {
goals.addAll(aliasingGoals);
duGoals.addAll(aliasingGoals);
}
return !aliasingGoals.isEmpty();
}
Aggregations