Examples with DMLTranslator org.apache.sysml.parser.DMLTranslator used on opensource projects

Example 1 with DMLTranslator

use of org.apache.sysml.parser.DMLTranslator in project incubator-systemml by apache.

the class Connection method prepareScript.

/**
 * Prepares (precompiles) a script, sets input parameter values, and registers input and output variables.
 *
 * @param script string representing the DML or PyDML script
 * @param args map of input parameters ($) and their values
 * @param inputs string array of input variables to register
 * @param outputs string array of output variables to register
 * @param parsePyDML {@code true} if PyDML, {@code false} if DML
 * @return PreparedScript object representing the precompiled script
 */
public PreparedScript prepareScript(String script, Map<String, String> args, String[] inputs, String[] outputs, boolean parsePyDML) {
    DMLScript.SCRIPT_TYPE = parsePyDML ? ScriptType.PYDML : ScriptType.DML;
    // check for valid names of passed arguments
    String[] invalidArgs = args.keySet().stream().filter(k -> k == null || !k.startsWith("$")).toArray(String[]::new);
    if (invalidArgs.length > 0)
        throw new LanguageException("Invalid argument names: " + Arrays.toString(invalidArgs));
    // check for valid names of input and output variables
    String[] invalidVars = UtilFunctions.asSet(inputs, outputs).stream().filter(k -> k == null || k.startsWith("$")).toArray(String[]::new);
    if (invalidVars.length > 0)
        throw new LanguageException("Invalid variable names: " + Arrays.toString(invalidVars));
    setLocalConfigs();
    // simplified compilation chain
    Program rtprog = null;
    try {
        // parsing
        ParserWrapper parser = ParserFactory.createParser(parsePyDML ? ScriptType.PYDML : ScriptType.DML);
        DMLProgram prog = parser.parse(null, script, args);
        // language validate
        DMLTranslator dmlt = new DMLTranslator(prog);
        dmlt.liveVariableAnalysis(prog);
        dmlt.validateParseTree(prog);
        // hop construct/rewrite
        dmlt.constructHops(prog);
        dmlt.rewriteHopsDAG(prog);
        // rewrite persistent reads/writes
        RewriteRemovePersistentReadWrite rewrite = new RewriteRemovePersistentReadWrite(inputs, outputs);
        ProgramRewriter rewriter2 = new ProgramRewriter(rewrite);
        rewriter2.rewriteProgramHopDAGs(prog);
        // lop construct and runtime prog generation
        dmlt.constructLops(prog);
        rtprog = dmlt.getRuntimeProgram(prog, _dmlconf);
        // final cleanup runtime prog
        JMLCUtils.cleanupRuntimeProgram(rtprog, outputs);
    } catch (ParseException pe) {
        // don't chain ParseException (for cleaner error output)
        throw pe;
    } catch (Exception ex) {
        throw new DMLException(ex);
    }
    // return newly create precompiled script
    return new PreparedScript(rtprog, inputs, outputs, _dmlconf, _cconf);
}

Example 2 with DMLTranslator

use of org.apache.sysml.parser.DMLTranslator in project incubator-systemml by apache.

the class ScriptExecutor method liveVariableAnalysis.

/**
 * Liveness analysis is performed on the program, obtaining sets of live-in
 * and live-out variables by forward and backward passes over the program.
 */
protected void liveVariableAnalysis() {
    try {
        dmlTranslator = new DMLTranslator(dmlProgram);
        dmlTranslator.liveVariableAnalysis(dmlProgram);
    } catch (DMLRuntimeException e) {
        throw new MLContextException("Exception occurred during live variable analysis", e);
    } catch (LanguageException e) {
        throw new MLContextException("Exception occurred during live variable analysis", e);
    }
}

Example 3 with DMLTranslator

use of org.apache.sysml.parser.DMLTranslator in project incubator-systemml by apache.

the class DMLScript method launchDebugger.

/**
 * Launcher for DML debugger. This method should be called after
 * execution and debug properties have been correctly set, and customized parameters
 *
 * @param dmlScriptStr DML script contents (including new lines)
 * @param fnameOptConfig Full path of configuration file for SystemML
 * @param argVals Key-value pairs defining arguments of DML script
 * @param scriptType type of script (DML or PyDML)
 * @throws IOException if IOException occurs
 */
private static void launchDebugger(String dmlScriptStr, String fnameOptConfig, Map<String, String> argVals, ScriptType scriptType) throws IOException {
    DMLDebuggerProgramInfo dbprog = new DMLDebuggerProgramInfo();
    // Step 1: parse configuration files
    DMLConfig conf = DMLConfig.readConfigurationFile(fnameOptConfig);
    ConfigurationManager.setGlobalConfig(conf);
    // Step 2: parse dml script
    ParserWrapper parser = ParserFactory.createParser(scriptType);
    DMLProgram prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
    // Step 3: construct HOP DAGs (incl LVA and validate)
    DMLTranslator dmlt = new DMLTranslator(prog);
    dmlt.liveVariableAnalysis(prog);
    dmlt.validateParseTree(prog);
    dmlt.constructHops(prog);
    // Step 4: rewrite HOP DAGs (incl IPA and memory estimates)
    dmlt.rewriteHopsDAG(prog);
    // Step 5: construct LOP DAGs
    dmlt.constructLops(prog);
    // Step 6: generate runtime program
    dbprog.rtprog = dmlt.getRuntimeProgram(prog, conf);
    try {
        // set execution environment
        initHadoopExecution(conf);
        // initialize an instance of SystemML debugger
        DMLDebugger SystemMLdb = new DMLDebugger(dbprog, dmlScriptStr);
        // run SystemML debugger
        SystemMLdb.runSystemMLDebugger();
    } finally {
        // cleanup scratch_space and all working dirs
        cleanupHadoopExecution(conf);
    }
}

Example 4 with DMLTranslator

use of org.apache.sysml.parser.DMLTranslator in project incubator-systemml by apache.

the class ParForDependencyAnalysisTest method runTest.

private void runTest(String scriptFilename, boolean expectedException) {
    boolean raisedException = false;
    try {
        // Tell the superclass about the name of this test, so that the superclass can
        // create temporary directories.
        int index = scriptFilename.lastIndexOf(".dml");
        String testName = scriptFilename.substring(0, index > 0 ? index : scriptFilename.length());
        TestConfiguration testConfig = new TestConfiguration(TEST_CLASS_DIR, testName, new String[] {});
        addTestConfiguration(testName, testConfig);
        loadTestConfiguration(testConfig);
        DMLConfig conf = new DMLConfig(getCurConfigFile().getPath());
        ConfigurationManager.setLocalConfig(conf);
        String dmlScriptString = "";
        HashMap<String, String> argVals = new HashMap<String, String>();
        // read script
        try (BufferedReader in = new BufferedReader(new FileReader(HOME + scriptFilename))) {
            String s1 = null;
            while ((s1 = in.readLine()) != null) dmlScriptString += s1 + "\n";
        }
        // parsing and dependency analysis
        ParserWrapper parser = ParserFactory.createParser(org.apache.sysml.api.mlcontext.ScriptType.DML);
        DMLProgram prog = parser.parse(DMLScript.DML_FILE_PATH_ANTLR_PARSER, dmlScriptString, argVals);
        DMLTranslator dmlt = new DMLTranslator(prog);
        dmlt.validateParseTree(prog);
    } catch (LanguageException ex) {
        raisedException = true;
        if (raisedException != expectedException)
            ex.printStackTrace();
    } catch (Exception ex2) {
        ex2.printStackTrace();
        throw new RuntimeException(ex2);
    // Assert.fail( "Unexpected exception occured during test run." );
    }
    // check correctness
    Assert.assertEquals(expectedException, raisedException);
}

Example 5 with DMLTranslator

use of org.apache.sysml.parser.DMLTranslator in project systemml by apache.

the class ProgramRecompiler method generatePartitialRuntimeProgram.

public static ArrayList<ProgramBlock> generatePartitialRuntimeProgram(Program rtprog, ArrayList<StatementBlock> sbs) {
    ArrayList<ProgramBlock> ret = new ArrayList<>();
    DMLConfig config = ConfigurationManager.getDMLConfig();
    // construct lops from hops if not existing
    DMLTranslator dmlt = new DMLTranslator(sbs.get(0).getDMLProg());
    for (StatementBlock sb : sbs) {
        dmlt.constructLops(sb);
    }
    // construct runtime program from lops
    for (StatementBlock sb : sbs) {
        ret.add(dmlt.createRuntimeProgramBlock(rtprog, sb, config));
    }
    // enhance runtime program by automatic operator fusion
    if (ConfigurationManager.isCodegenEnabled() && SpoofCompiler.INTEGRATION == IntegrationType.RUNTIME) {
        for (ProgramBlock pb : ret) dmlt.codgenHopsDAG(pb);
    }
    return ret;
}