Examples with ParserWrapper org.apache.sysml.parser.ParserWrapper used on opensource projects

Example 1 with ParserWrapper

use of org.apache.sysml.parser.ParserWrapper in project incubator-systemml by apache.

the class Connection method prepareScript.

/**
 * Prepares (precompiles) a script, sets input parameter values, and registers input and output variables.
 *
 * @param script string representing the DML or PyDML script
 * @param args map of input parameters ($) and their values
 * @param inputs string array of input variables to register
 * @param outputs string array of output variables to register
 * @param parsePyDML {@code true} if PyDML, {@code false} if DML
 * @return PreparedScript object representing the precompiled script
 */
public PreparedScript prepareScript(String script, Map<String, String> args, String[] inputs, String[] outputs, boolean parsePyDML) {
    DMLScript.SCRIPT_TYPE = parsePyDML ? ScriptType.PYDML : ScriptType.DML;
    // check for valid names of passed arguments
    String[] invalidArgs = args.keySet().stream().filter(k -> k == null || !k.startsWith("$")).toArray(String[]::new);
    if (invalidArgs.length > 0)
        throw new LanguageException("Invalid argument names: " + Arrays.toString(invalidArgs));
    // check for valid names of input and output variables
    String[] invalidVars = UtilFunctions.asSet(inputs, outputs).stream().filter(k -> k == null || k.startsWith("$")).toArray(String[]::new);
    if (invalidVars.length > 0)
        throw new LanguageException("Invalid variable names: " + Arrays.toString(invalidVars));
    setLocalConfigs();
    // simplified compilation chain
    Program rtprog = null;
    try {
        // parsing
        ParserWrapper parser = ParserFactory.createParser(parsePyDML ? ScriptType.PYDML : ScriptType.DML);
        DMLProgram prog = parser.parse(null, script, args);
        // language validate
        DMLTranslator dmlt = new DMLTranslator(prog);
        dmlt.liveVariableAnalysis(prog);
        dmlt.validateParseTree(prog);
        // hop construct/rewrite
        dmlt.constructHops(prog);
        dmlt.rewriteHopsDAG(prog);
        // rewrite persistent reads/writes
        RewriteRemovePersistentReadWrite rewrite = new RewriteRemovePersistentReadWrite(inputs, outputs);
        ProgramRewriter rewriter2 = new ProgramRewriter(rewrite);
        rewriter2.rewriteProgramHopDAGs(prog);
        // lop construct and runtime prog generation
        dmlt.constructLops(prog);
        rtprog = dmlt.getRuntimeProgram(prog, _dmlconf);
        // final cleanup runtime prog
        JMLCUtils.cleanupRuntimeProgram(rtprog, outputs);
    } catch (ParseException pe) {
        // don't chain ParseException (for cleaner error output)
        throw pe;
    } catch (Exception ex) {
        throw new DMLException(ex);
    }
    // return newly create precompiled script
    return new PreparedScript(rtprog, inputs, outputs, _dmlconf, _cconf);
}

Example 2 with ParserWrapper

use of org.apache.sysml.parser.ParserWrapper in project incubator-systemml by apache.

the class DMLScript method launchDebugger.

/**
 * Launcher for DML debugger. This method should be called after
 * execution and debug properties have been correctly set, and customized parameters
 *
 * @param dmlScriptStr DML script contents (including new lines)
 * @param fnameOptConfig Full path of configuration file for SystemML
 * @param argVals Key-value pairs defining arguments of DML script
 * @param scriptType type of script (DML or PyDML)
 * @throws IOException if IOException occurs
 */
private static void launchDebugger(String dmlScriptStr, String fnameOptConfig, Map<String, String> argVals, ScriptType scriptType) throws IOException {
    DMLDebuggerProgramInfo dbprog = new DMLDebuggerProgramInfo();
    // Step 1: parse configuration files
    DMLConfig conf = DMLConfig.readConfigurationFile(fnameOptConfig);
    ConfigurationManager.setGlobalConfig(conf);
    // Step 2: parse dml script
    ParserWrapper parser = ParserFactory.createParser(scriptType);
    DMLProgram prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
    // Step 3: construct HOP DAGs (incl LVA and validate)
    DMLTranslator dmlt = new DMLTranslator(prog);
    dmlt.liveVariableAnalysis(prog);
    dmlt.validateParseTree(prog);
    dmlt.constructHops(prog);
    // Step 4: rewrite HOP DAGs (incl IPA and memory estimates)
    dmlt.rewriteHopsDAG(prog);
    // Step 5: construct LOP DAGs
    dmlt.constructLops(prog);
    // Step 6: generate runtime program
    dbprog.rtprog = dmlt.getRuntimeProgram(prog, conf);
    try {
        // set execution environment
        initHadoopExecution(conf);
        // initialize an instance of SystemML debugger
        DMLDebugger SystemMLdb = new DMLDebugger(dbprog, dmlScriptStr);
        // run SystemML debugger
        SystemMLdb.runSystemMLDebugger();
    } finally {
        // cleanup scratch_space and all working dirs
        cleanupHadoopExecution(conf);
    }
}

Example 3 with ParserWrapper

use of org.apache.sysml.parser.ParserWrapper in project incubator-systemml by apache.

the class ParForDependencyAnalysisTest method runTest.

private void runTest(String scriptFilename, boolean expectedException) {
    boolean raisedException = false;
    try {
        // Tell the superclass about the name of this test, so that the superclass can
        // create temporary directories.
        int index = scriptFilename.lastIndexOf(".dml");
        String testName = scriptFilename.substring(0, index > 0 ? index : scriptFilename.length());
        TestConfiguration testConfig = new TestConfiguration(TEST_CLASS_DIR, testName, new String[] {});
        addTestConfiguration(testName, testConfig);
        loadTestConfiguration(testConfig);
        DMLConfig conf = new DMLConfig(getCurConfigFile().getPath());
        ConfigurationManager.setLocalConfig(conf);
        String dmlScriptString = "";
        HashMap<String, String> argVals = new HashMap<String, String>();
        // read script
        try (BufferedReader in = new BufferedReader(new FileReader(HOME + scriptFilename))) {
            String s1 = null;
            while ((s1 = in.readLine()) != null) dmlScriptString += s1 + "\n";
        }
        // parsing and dependency analysis
        ParserWrapper parser = ParserFactory.createParser(org.apache.sysml.api.mlcontext.ScriptType.DML);
        DMLProgram prog = parser.parse(DMLScript.DML_FILE_PATH_ANTLR_PARSER, dmlScriptString, argVals);
        DMLTranslator dmlt = new DMLTranslator(prog);
        dmlt.validateParseTree(prog);
    } catch (LanguageException ex) {
        raisedException = true;
        if (raisedException != expectedException)
            ex.printStackTrace();
    } catch (Exception ex2) {
        ex2.printStackTrace();
        throw new RuntimeException(ex2);
    // Assert.fail( "Unexpected exception occured during test run." );
    }
    // check correctness
    Assert.assertEquals(expectedException, raisedException);
}

Example 4 with ParserWrapper

use of org.apache.sysml.parser.ParserWrapper in project systemml by apache.

the class DMLScript method execute.

// /////////////////////////////
// private internal interface
// (core compilation and execute)
// //////
/**
 * The running body of DMLScript execution. This method should be called after execution properties have been correctly set,
 * and customized parameters have been put into _argVals
 *
 * @param dmlScriptStr DML script string
 * @param fnameOptConfig configuration file
 * @param argVals map of argument values
 * @param allArgs arguments
 * @param scriptType type of script (DML or PyDML)
 * @throws IOException if IOException occurs
 */
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String, String> argVals, String[] allArgs, ScriptType scriptType) throws IOException {
    SCRIPT_TYPE = scriptType;
    // print basic time and environment info
    printStartExecInfo(dmlScriptStr);
    // Step 1: parse configuration files & write any configuration specific global variables
    DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig);
    ConfigurationManager.setGlobalConfig(dmlconf);
    CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
    ConfigurationManager.setGlobalConfig(cconf);
    LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo());
    // Sets the GPUs to use for this process (a range, all GPUs, comma separated list or a specific GPU)
    GPUContextPool.AVAILABLE_GPUS = dmlconf.getTextValue(DMLConfig.AVAILABLE_GPUS);
    String evictionPolicy = dmlconf.getTextValue(DMLConfig.GPU_EVICTION_POLICY).toUpperCase();
    try {
        DMLScript.GPU_EVICTION_POLICY = EvictionPolicy.valueOf(evictionPolicy);
    } catch (IllegalArgumentException e) {
        throw new RuntimeException("Unsupported eviction policy:" + evictionPolicy);
    }
    // Step 2: set local/remote memory if requested (for compile in AM context)
    if (dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER)) {
        DMLAppMasterUtils.setupConfigRemoteMaxMemory(dmlconf);
    }
    // Step 3: parse dml script
    Statistics.startCompileTimer();
    ParserWrapper parser = ParserFactory.createParser(scriptType);
    DMLProgram prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
    // Step 4: construct HOP DAGs (incl LVA, validate, and setup)
    DMLTranslator dmlt = new DMLTranslator(prog);
    dmlt.liveVariableAnalysis(prog);
    dmlt.validateParseTree(prog);
    dmlt.constructHops(prog);
    // init working directories (before usage by following compilation steps)
    initHadoopExecution(dmlconf);
    // Step 5: rewrite HOP DAGs (incl IPA and memory estimates)
    dmlt.rewriteHopsDAG(prog);
    // Step 6: construct lops (incl exec type and op selection)
    dmlt.constructLops(prog);
    if (LOG.isDebugEnabled()) {
        LOG.debug("\n********************** LOPS DAG *******************");
        dmlt.printLops(prog);
        dmlt.resetLopsDAGVisitStatus(prog);
    }
    // Step 7: generate runtime program, incl codegen
    Program rtprog = dmlt.getRuntimeProgram(prog, dmlconf);
    // launch SystemML appmaster (if requested and not already in launched AM)
    if (dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER)) {
        if (!isActiveAM() && DMLYarnClientProxy.launchDMLYarnAppmaster(dmlScriptStr, dmlconf, allArgs, rtprog))
            // if AM launch unsuccessful, fall back to normal execute
            return;
        if (// in AM context (not failed AM launch)
        isActiveAM())
            DMLAppMasterUtils.setupProgramMappingRemoteMaxMemory(rtprog);
    }
    // Step 9: prepare statistics [and optional explain output]
    // count number compiled MR jobs / SP instructions
    ExplainCounts counts = Explain.countDistributedOperations(rtprog);
    Statistics.resetNoOfCompiledJobs(counts.numJobs);
    // explain plan of program (hops or runtime)
    if (EXPLAIN != ExplainType.NONE)
        LOG.info(Explain.display(prog, rtprog, EXPLAIN, counts));
    Statistics.stopCompileTimer();
    // double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext());
    // System.out.println("Estimated costs: "+costs);
    // Step 10: execute runtime program
    ExecutionContext ec = null;
    try {
        ec = ExecutionContextFactory.createContext(rtprog);
        ScriptExecutorUtils.executeRuntimeProgram(rtprog, ec, dmlconf, STATISTICS ? STATISTICS_COUNT : 0);
    } finally {
        if (ec != null && ec instanceof SparkExecutionContext)
            ((SparkExecutionContext) ec).close();
        LOG.info("END DML run " + getDateTime());
        // cleanup scratch_space and all working dirs
        cleanupHadoopExecution(dmlconf);
    }
}

Example 5 with ParserWrapper

use of org.apache.sysml.parser.ParserWrapper in project systemml by apache.

the class ScriptExecutor method parseScript.

/**
 * Parse the script into an ANTLR parse tree, and convert this parse tree
 * into a SystemML program. Parsing includes lexical/syntactic analysis.
 */
protected void parseScript() {
    try {
        ParserWrapper parser = ParserFactory.createParser(script.getScriptType());
        Map<String, Object> inputParameters = script.getInputParameters();
        Map<String, String> inputParametersStringMaps = MLContextUtil.convertInputParametersForParser(inputParameters, script.getScriptType());
        String scriptExecutionString = script.getScriptExecutionString();
        dmlProgram = parser.parse(null, scriptExecutionString, inputParametersStringMaps);
    } catch (ParseException e) {
        throw new MLContextException("Exception occurred while parsing script", e);
    }
}