Examples with SynthesizeException edu.rice.cs.caper.floodgage.application.floodgage.synthesizer.SynthesizeException used on opensource projects

Example 1 with SynthesizeException

use of edu.rice.cs.caper.floodgage.application.floodgage.synthesizer.SynthesizeException in project bayou by capergroup.

the class TrialsRunner method runTrails.

/**
 * Execute the given trials synthesizing result programs from the given Synthesizer and report progress
 * to the given view.
 *
 * @param trials the trials to run
 * @param synthesizer the synthesizer to use for mapping draft programs onto results
 * @param view the view used to track execution progress
 */
static void runTrails(List<Trial> trials, Synthesizer synthesizer, View view) {
    if (trials == null)
        throw new NullPointerException("trials");
    if (synthesizer == null)
        throw new NullPointerException("synthesizer");
    if (view == null)
        throw new NullPointerException("view");
    // unique number for the current trial.
    int currentTrialId = 0;
    int possibleCompilePointsAccum = 0;
    int obtainedCompilePointsAccum = 0;
    int possibleTestCasePointsAccum = 0;
    int obtainedTestCasePointsAccum = 0;
    int possibleSketchMatchPointsAccum = 0;
    int obtainedSketchMatchPointsAccum = 0;
    // track any thrown SynthesizeException from synthesizer.synthesize(...)
    boolean anySynthesizeFailed = false;
    for (Trial trial : trials) {
        currentTrialId++;
        String draftProgram = trial.getDraftProgramSource();
        view.declareStartOfTrial(trial.getDescription(), draftProgram);
        /*
             * Provide the draft program to the synthesizer and collect the synthesis results.
             */
        List<String> synthResults;
        try {
            synthResults = synthesizer.synthesize(draftProgram);
        } catch (SynthesizeException e) {
            view.declareSynthesisFailed();
            view.declareTrialResultSynthesisFailed();
            anySynthesizeFailed = true;
            continue;
        }
        view.declareNumberOfResults(synthResults.size());
        /*
             * Rewrite each result class source to have a unique class name.
             *
             * We will compile and class-load each result so each class needs to have a unique class name.
             */
        List<SourceClass> synthResultsWithUniqueClassNames = new LinkedList<>();
        {
            // floodgage assumes that the name of the synthesized class remains unchanged from the input class name.
            String assumedResultClassName = trial.getDraftProgramClassName();
            int resultId = 0;
            for (String result : synthResults) {
                resultId++;
                String uniqueResultClassName = assumedResultClassName + "_" + currentTrialId + "_" + resultId;
                String classSource = result.replaceFirst("public class " + assumedResultClassName, "public class " + uniqueResultClassName);
                synthResultsWithUniqueClassNames.add(new SourceClass(uniqueResultClassName, classSource));
            }
        }
        /*
             * If an expected sketch source was added to this trial, construct the sketch from the source.
             * Also increment possiblePointsAccum if there is an expected sketch.
             *
             * If no expected sketch is defined, set expectedSketch to null.
             */
        // true: sketch expected and a match found in some result.
        Boolean anySketchMatch;
        // false: sketch expected but no match found so far among any result.
        // null: no expected sketch.
        DSubTree expectedSketch;
        {
            if (trial.containsSketchProgramSource()) {
                String expectedSketchSource = trial.tryGetSketchProgramSource(null);
                expectedSketch = makeSketchFromSource(expectedSketchSource, trial.getDraftProgramClassName() + ".java");
                // we will test to sketch matches, so init to none seen yet
                anySketchMatch = false;
            } else {
                expectedSketch = null;
                // we wont do sketch testing to singal no expected sketch
                anySketchMatch = null;
            }
        }
        if (trial.containsSketchProgramSource())
            // possible point for some result matching the sketch.
            possibleSketchMatchPointsAccum++;
        /*
             * For each result:
             *
             *     1.) Check that the result compiles
             *     2.) If compiles, check that the result passes the test suite.
             *     3.) If compiles and the trial contains an expected sketch, check for sketch equivalency.
             */
        int resultId = 0;
        for (SourceClass result : synthResultsWithUniqueClassNames) {
            resultId++;
            // possible point for result compiling.
            possibleCompilePointsAccum++;
            view.declareSynthesizeResult(resultId, result.classSource);
            /*
                 * If a sketch expectation is declared for this trial, perform sketch comparison.
                 */
            boolean sketchMatch = false;
            if (// if there is an expected sketch
            anySketchMatch != null) {
                DSubTree resultSketch = makeSketchFromSource(result.classSource, result.className + ".java");
                if (// both null
                expectedSketch == null && resultSketch == null) {
                    view.warnSketchesNull();
                    sketchMatch = true;
                } else if (// both non-null
                expectedSketch != null && resultSketch != null) {
                    EqualityASTMetric m = new EqualityASTMetric();
                    float equalityResult = m.compute(expectedSketch, Collections.singletonList(resultSketch), "");
                    view.declareSketchMetricResult(equalityResult);
                    if (equalityResult == 1)
                        sketchMatch = true;
                } else // one null
                {
                    view.warnSketchesNullMismatch();
                }
                anySketchMatch = anySketchMatch || sketchMatch;
            }
            /*
                 * Check that the result compiles.
                 */
            boolean resultCompiled;
            try {
                // ensure any compiler errors start on new line
                System.out.println("");
                CompilerUtils.CACHED_COMPILER.loadFromJava(result.className, result.classSource);
                resultCompiled = true;
                // for compiling
                obtainedCompilePointsAccum++;
            } catch (ClassNotFoundException e) {
                resultCompiled = false;
            }
            /*
                 * If result compiles, run test cases.
                 *
                 * (Construct a test suite for result and run the result against the suite.)
                 */
            boolean testCasesPass;
            {
                if (resultCompiled) {
                    view.declareStartOfTestCases();
                    Class resultSpecificTestSuite = makeResultSpecificTestSuite(result.className);
                    TestSuiteRunner.RunResult runResult = TestSuiteRunner.runTestSuiteAgainst(resultSpecificTestSuite, view);
                    possibleTestCasePointsAccum += runResult.TestCaseCount;
                    obtainedTestCasePointsAccum += runResult.TestCaseCount - runResult.FailCount;
                    testCasesPass = runResult.FailCount == 0;
                } else {
                    // even if no test cases we say uncompilable code fails the null t.c.
                    testCasesPass = false;
                    /*
                         * Count the number of test cases in TestSuite (since we didnt run JUnit) and add each
                         * to the possiblePointsAccum count.
                         */
                    Class testSuiteClass;
                    try {
                        testSuiteClass = Class.forName("TestSuite");
                    } catch (ClassNotFoundException e) {
                        // TestSuite should be in the trialpack
                        throw new RuntimeException(e);
                    }
                    for (Method m : testSuiteClass.getMethods()) {
                        if (m.getAnnotation(org.junit.Test.class) != null)
                            possibleTestCasePointsAccum++;
                    }
                }
            }
            /*
                 * Report results for the result.
                 */
            Boolean reportSketchMatch = trial.containsSketchProgramSource() ? sketchMatch : null;
            view.declareSynthResultResult(resultCompiled, testCasesPass, reportSketchMatch);
        }
        /*
             * If any sketch matched among the results, award a point. 
             */
        if (anySketchMatch != null && anySketchMatch)
            obtainedSketchMatchPointsAccum++;
    }
    /*
         * Report total tallies.
         */
    view.declarePointScore(possibleCompilePointsAccum, obtainedCompilePointsAccum, possibleTestCasePointsAccum, obtainedTestCasePointsAccum, possibleSketchMatchPointsAccum, obtainedSketchMatchPointsAccum);
    if (anySynthesizeFailed)
        // this can really influence the score by skipping points, so remind.
        view.declareSynthesisFailed();
}