Examples with Trial edu.rice.cs.caper.floodgage.application.floodgage.model.plan.Trial used on opensource projects

Example 1 with Trial

use of edu.rice.cs.caper.floodgage.application.floodgage.model.plan.Trial in project bayou by capergroup.

the class FloodGage method run.

/**
 * Parses "trials.xml" as found as a resource on the current class path and executes the trials against
 * the given Synthesizer reporting progress to the given View.
 *
 * Expects a trialpack to be on the current classpath.
 *
 * @param synthesizer the Synthesizer to use for creating programs from draft programs identified in trials.
 * @param view the UI component for reporting execution progress.
 * @throws IOException if there is a program reading resources (like "trails.xml") on the classpath.
 * @throws ParseException if "trials.xml" is not of the expected format
 * @throws ClassCastException if "PassProgram" or "PassProgramTest" cannot be loaded from teh current classpath.
 */
void run(Synthesizer synthesizer, View view) throws IOException, ParseException, ClassNotFoundException {
    if (synthesizer == null)
        throw new NullPointerException("synthesizer");
    if (view == null)
        throw new NullPointerException("view");
    /*
         * Parse trials.xml from inside the trialpack (assumed to be on classpath) into a Directive structure.
         */
    Directive directive;
    {
        // We expect the trialpack jar file to be on the classpath.  That jar file should have a
        // trails.xml file its root directory.  So look for it at just "trails.xml" path.
        String trailsXml = new String(getResource("trials.xml"), StandardCharsets.UTF_8);
        directive = DirectiveXmlV1Parser.makeDefault().parse(trailsXml);
    }
    /*
         * Test the test suite by asserting that all test cases pass when given the pass program.
         *
         * Stop the program if this is not the case.
         */
    // expect PassProgram class inside the trailpack which is assumed to be on the class path
    Class passProgram = getClass().getClassLoader().loadClass("PassProgram");
    // expect PassProgramTest class inside the trailpack which is assumed to be on the class path
    Class passProgramTestSuite = getClass().getClassLoader().loadClass("PassProgramTest");
    view.declareTestingTestSuite(passProgramTestSuite, passProgram);
    // run the test cases against PassProgram
    Result result = new JUnitCore().run(passProgramTestSuite);
    view.declareNumberOfTestCasesInSuite(result.getRunCount());
    if (// if the pass program did not pass the test suite
    result.getFailureCount() > 0) {
        for (Failure f : result.getFailures()) view.declarePassProgramTestFailure(f.getTrace());
    } else // pass program did pass the test suite, proceed to running the user specified trials
    {
        List<Trial> trials = makeTrials(directive, DraftBuilderBayou1_1_0::new);
        TrialsRunner.runTrails(trials, synthesizer, view);
    }
}

Example 2 with Trial

use of edu.rice.cs.caper.floodgage.application.floodgage.model.plan.Trial in project bayou by capergroup.

the class TrialsRunner method runTrails.

/**
 * Execute the given trials synthesizing result programs from the given Synthesizer and report progress
 * to the given view.
 *
 * @param trials the trials to run
 * @param synthesizer the synthesizer to use for mapping draft programs onto results
 * @param view the view used to track execution progress
 */
static void runTrails(List<Trial> trials, Synthesizer synthesizer, View view) {
    if (trials == null)
        throw new NullPointerException("trials");
    if (synthesizer == null)
        throw new NullPointerException("synthesizer");
    if (view == null)
        throw new NullPointerException("view");
    // unique number for the current trial.
    int currentTrialId = 0;
    int possibleCompilePointsAccum = 0;
    int obtainedCompilePointsAccum = 0;
    int possibleTestCasePointsAccum = 0;
    int obtainedTestCasePointsAccum = 0;
    int possibleSketchMatchPointsAccum = 0;
    int obtainedSketchMatchPointsAccum = 0;
    // track any thrown SynthesizeException from synthesizer.synthesize(...)
    boolean anySynthesizeFailed = false;
    for (Trial trial : trials) {
        currentTrialId++;
        String draftProgram = trial.getDraftProgramSource();
        view.declareStartOfTrial(trial.getDescription(), draftProgram);
        /*
             * Provide the draft program to the synthesizer and collect the synthesis results.
             */
        List<String> synthResults;
        try {
            synthResults = synthesizer.synthesize(draftProgram);
        } catch (SynthesizeException e) {
            view.declareSynthesisFailed();
            view.declareTrialResultSynthesisFailed();
            anySynthesizeFailed = true;
            continue;
        }
        view.declareNumberOfResults(synthResults.size());
        /*
             * Rewrite each result class source to have a unique class name.
             *
             * We will compile and class-load each result so each class needs to have a unique class name.
             */
        List<SourceClass> synthResultsWithUniqueClassNames = new LinkedList<>();
        {
            // floodgage assumes that the name of the synthesized class remains unchanged from the input class name.
            String assumedResultClassName = trial.getDraftProgramClassName();
            int resultId = 0;
            for (String result : synthResults) {
                resultId++;
                String uniqueResultClassName = assumedResultClassName + "_" + currentTrialId + "_" + resultId;
                String classSource = result.replaceFirst("public class " + assumedResultClassName, "public class " + uniqueResultClassName);
                synthResultsWithUniqueClassNames.add(new SourceClass(uniqueResultClassName, classSource));
            }
        }
        /*
             * If an expected sketch source was added to this trial, construct the sketch from the source.
             * Also increment possiblePointsAccum if there is an expected sketch.
             *
             * If no expected sketch is defined, set expectedSketch to null.
             */
        // true: sketch expected and a match found in some result.
        Boolean anySketchMatch;
        // false: sketch expected but no match found so far among any result.
        // null: no expected sketch.
        DSubTree expectedSketch;
        {
            if (trial.containsSketchProgramSource()) {
                String expectedSketchSource = trial.tryGetSketchProgramSource(null);
                expectedSketch = makeSketchFromSource(expectedSketchSource, trial.getDraftProgramClassName() + ".java");
                // we will test to sketch matches, so init to none seen yet
                anySketchMatch = false;
            } else {
                expectedSketch = null;
                // we wont do sketch testing to singal no expected sketch
                anySketchMatch = null;
            }
        }
        if (trial.containsSketchProgramSource())
            // possible point for some result matching the sketch.
            possibleSketchMatchPointsAccum++;
        /*
             * For each result:
             *
             *     1.) Check that the result compiles
             *     2.) If compiles, check that the result passes the test suite.
             *     3.) If compiles and the trial contains an expected sketch, check for sketch equivalency.
             */
        int resultId = 0;
        for (SourceClass result : synthResultsWithUniqueClassNames) {
            resultId++;
            // possible point for result compiling.
            possibleCompilePointsAccum++;
            view.declareSynthesizeResult(resultId, result.classSource);
            /*
                 * If a sketch expectation is declared for this trial, perform sketch comparison.
                 */
            boolean sketchMatch = false;
            if (// if there is an expected sketch
            anySketchMatch != null) {
                DSubTree resultSketch = makeSketchFromSource(result.classSource, result.className + ".java");
                if (// both null
                expectedSketch == null && resultSketch == null) {
                    view.warnSketchesNull();
                    sketchMatch = true;
                } else if (// both non-null
                expectedSketch != null && resultSketch != null) {
                    EqualityASTMetric m = new EqualityASTMetric();
                    float equalityResult = m.compute(expectedSketch, Collections.singletonList(resultSketch), "");
                    view.declareSketchMetricResult(equalityResult);
                    if (equalityResult == 1)
                        sketchMatch = true;
                } else // one null
                {
                    view.warnSketchesNullMismatch();
                }
                anySketchMatch = anySketchMatch || sketchMatch;
            }
            /*
                 * Check that the result compiles.
                 */
            boolean resultCompiled;
            try {
                // ensure any compiler errors start on new line
                System.out.println("");
                CompilerUtils.CACHED_COMPILER.loadFromJava(result.className, result.classSource);
                resultCompiled = true;
                // for compiling
                obtainedCompilePointsAccum++;
            } catch (ClassNotFoundException e) {
                resultCompiled = false;
            }
            /*
                 * If result compiles, run test cases.
                 *
                 * (Construct a test suite for result and run the result against the suite.)
                 */
            boolean testCasesPass;
            {
                if (resultCompiled) {
                    view.declareStartOfTestCases();
                    Class resultSpecificTestSuite = makeResultSpecificTestSuite(result.className);
                    TestSuiteRunner.RunResult runResult = TestSuiteRunner.runTestSuiteAgainst(resultSpecificTestSuite, view);
                    possibleTestCasePointsAccum += runResult.TestCaseCount;
                    obtainedTestCasePointsAccum += runResult.TestCaseCount - runResult.FailCount;
                    testCasesPass = runResult.FailCount == 0;
                } else {
                    // even if no test cases we say uncompilable code fails the null t.c.
                    testCasesPass = false;
                    /*
                         * Count the number of test cases in TestSuite (since we didnt run JUnit) and add each
                         * to the possiblePointsAccum count.
                         */
                    Class testSuiteClass;
                    try {
                        testSuiteClass = Class.forName("TestSuite");
                    } catch (ClassNotFoundException e) {
                        // TestSuite should be in the trialpack
                        throw new RuntimeException(e);
                    }
                    for (Method m : testSuiteClass.getMethods()) {
                        if (m.getAnnotation(org.junit.Test.class) != null)
                            possibleTestCasePointsAccum++;
                    }
                }
            }
            /*
                 * Report results for the result.
                 */
            Boolean reportSketchMatch = trial.containsSketchProgramSource() ? sketchMatch : null;
            view.declareSynthResultResult(resultCompiled, testCasesPass, reportSketchMatch);
        }
        /*
             * If any sketch matched among the results, award a point. 
             */
        if (anySketchMatch != null && anySketchMatch)
            obtainedSketchMatchPointsAccum++;
    }
    /*
         * Report total tallies.
         */
    view.declarePointScore(possibleCompilePointsAccum, obtainedCompilePointsAccum, possibleTestCasePointsAccum, obtainedTestCasePointsAccum, possibleSketchMatchPointsAccum, obtainedSketchMatchPointsAccum);
    if (anySynthesizeFailed)
        // this can really influence the score by skipping points, so remind.
        view.declareSynthesisFailed();
}

Example 3 with Trial

use of edu.rice.cs.caper.floodgage.application.floodgage.model.plan.Trial in project bayou by capergroup.

the class FloodGage method makeTrials.

/*
     * Convert the trials specified in the given Directive to a list of Trials.  Use a DraftBuilder returned by
     * makeDraftBuilder (one per trial) to rewrite the draft program from the format used inside a trialpack
     * to the format the synthesis system expects as input.
     */
private List<Trial> makeTrials(Directive directive, Function<String, DraftBuilder> makeDraftBuilder) throws IOException {
    // to be returned
    List<Trial> trialsAccum = new LinkedList<>();
    // build one Trial for every trial specified by the user as found in the directive
    for (edu.rice.cs.caper.floodgage.application.floodgage.model.directive.Trial trial : directive.getTrials()) {
        /*
             * Retrieve the description of the trial, if any.
             */
        // from <description> in trails.xml (if present)
        String description = trial.getDescription();
        // if description not specified, use empty string
        description = description == null ? "" : description;
        /*
             * Build the draft program source as it should be sent to the synthesizer.  This is a two step process:
             *
             * 1.) Retrieve the "proto" draft program source from the trialpack that uses the "/// #" hole notation.
             *     An example might looks like:
             *
             *     import java.util.function.*;
             *
             *     public class ReadFileLinesDraft implements Consumer<String>
             *     {
             *         public void accept(String filePath)
             *         {
             *             /// 1
             *         }
             *     }
             *
             * 2.) Replace each hole in the "/// #" notation with the hole format that the synthesis system expects
             *     as guided by the hole definition in trials.xml. The returned DraftBuilder from the given
             *     makeDraftBuilder parameter should know how to do this conversion for us.
             *
             *     For example, with Bayou 1.1.0 that would be (with the trials.xml above)
             *
             *     import java.util.function.*;
             *
             *     public class ReadFileLinesDraft implements Consumer<String>
             *     {
             *         public void accept(String filePath)
             *         {
             *             { call:readLine type:FileReader }
             *         }
             *     }
             *
             */
        String draftProgramSource;
        {
            // We expect the trialpack jar file to be on the classpath.  That jar file should have all draft
            // programs in its root directory.  So look for it at a path that is just the contents of the xml entry.
            // / using the "// #" notation
            String protoDraftProgramSrc = new String(getResource(trial.getDraftProgramPath()), StandardCharsets.UTF_8);
            DraftBuilder draftBuilder = makeDraftBuilder.apply(protoDraftProgramSrc);
            for (Hole hole : trial.getHoles()) {
                // convert evidences from the directive structure to the trial structure
                List<Evidence> evidence = hole.getEvidence().stream().map(this::makeEvidence).collect(Collectors.toList());
                draftBuilder.setHole(hole.getId(), evidence);
            }
            draftProgramSource = draftBuilder.buildDraft();
        }
        /*
             * Retrieve other elements from user defined trial.
             */
        // FileReaderDraft in example above
        String assumeDraftProgramClassName = getFilenameWithoutExtension(trial.getDraftProgramPath());
        // look for optional sketch entry. null if not present.
        String sketchProgramPath = trial.getSketchProgramPath();
        /*
             * Create Trial instance and append to trialsAccum.
             */
        Trial trialToAppend;
        {
            if (// has a sketch specified
            sketchProgramPath != null) {
                String sketchProgramSource = new String(getResource(trial.getSketchProgramPath()), StandardCharsets.UTF_8);
                trialToAppend = TrialWithSketch.make(description, draftProgramSource, assumeDraftProgramClassName, sketchProgramSource);
            } else // does not have a sketch specified
            {
                trialToAppend = TrialWithoutSketch.make(description, draftProgramSource, assumeDraftProgramClassName);
            }
        }
        trialsAccum.add(trialToAppend);
    }
    return trialsAccum;
}