Examples with DMLProgram org.apache.sysml.parser.DMLProgram used on opensource projects

Example 1 with DMLProgram

use of org.apache.sysml.parser.DMLProgram in project incubator-systemml by apache.

the class PyDMLParserWrapper method doParse.

/**
	 * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import'
	 * @param fileName script file name
	 * @param dmlScript script file contents
	 * @param sourceNamespace namespace from source statement
	 * @param argVals script arguments
	 * @return dml program, or null if at least one error
	 * @throws ParseException if ParseException occurs
	 */
public DMLProgram doParse(String fileName, String dmlScript, String sourceNamespace, Map<String, String> argVals) throws ParseException {
    DMLProgram dmlPgm = null;
    ANTLRInputStream in;
    try {
        if (dmlScript == null) {
            dmlScript = readDMLScript(fileName, LOG);
        }
        InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
        in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
    } catch (FileNotFoundException e) {
        throw new ParseException("Cannot find file/resource: " + fileName, e);
    } catch (IOException e) {
        throw new ParseException("Cannot open file: " + fileName, e);
    } catch (LanguageException e) {
        throw new ParseException(e.getMessage(), e);
    }
    ProgramrootContext ast = null;
    CustomErrorListener errorListener = new CustomErrorListener();
    try {
        PydmlLexer lexer = new PydmlLexer(in);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        PydmlParser antlr4Parser = new PydmlParser(tokens);
        // For now no optimization, since it is not able to parse integer value. 
        boolean tryOptimizedParsing = false;
        if (tryOptimizedParsing) {
            // Try faster and simpler SLL
            antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            antlr4Parser.removeErrorListeners();
            antlr4Parser.setErrorHandler(new BailErrorStrategy());
            try {
                ast = antlr4Parser.programroot();
            // If successful, no need to try out full LL(*) ... SLL was enough
            } catch (ParseCancellationException ex) {
                // Error occurred, so now try full LL(*) for better error messages
                tokens.reset();
                antlr4Parser.reset();
                if (fileName != null) {
                    errorListener.setCurrentFileName(fileName);
                } else {
                    errorListener.setCurrentFileName("MAIN_SCRIPT");
                }
                // Set our custom error listener
                antlr4Parser.addErrorListener(errorListener);
                antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
                antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
                ast = antlr4Parser.programroot();
            }
        } else {
            // Set our custom error listener
            antlr4Parser.removeErrorListeners();
            antlr4Parser.addErrorListener(errorListener);
            errorListener.setCurrentFileName(fileName);
            // Now do the parsing
            ast = antlr4Parser.programroot();
        }
    } catch (Exception e) {
        throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
    }
    // Now convert the parse tree into DMLProgram
    // Do syntactic validation while converting 
    ParseTree tree = ast;
    // And also do syntactic validation
    ParseTreeWalker walker = new ParseTreeWalker();
    // Get list of function definitions which take precedence over built-in functions if same name
    PydmlPreprocessor prep = new PydmlPreprocessor(errorListener);
    walker.walk(prep, tree);
    // Syntactic validation
    PydmlSyntacticValidator validator = new PydmlSyntacticValidator(errorListener, argVals, sourceNamespace, prep.getFunctionDefs());
    walker.walk(validator, tree);
    errorListener.unsetCurrentFileName();
    this.parseIssues = errorListener.getParseIssues();
    this.atLeastOneWarning = errorListener.isAtLeastOneWarning();
    this.atLeastOneError = errorListener.isAtLeastOneError();
    if (atLeastOneError) {
        throw new ParseException(parseIssues, dmlScript);
    }
    if (atLeastOneWarning) {
        LOG.warn(CustomErrorListener.generateParseIssuesMessage(dmlScript, parseIssues));
    }
    dmlPgm = createDMLProgram(ast, sourceNamespace);
    return dmlPgm;
}

Example 2 with DMLProgram

use of org.apache.sysml.parser.DMLProgram in project incubator-systemml by apache.

the class PyDMLParserWrapper method createDMLProgram.

private DMLProgram createDMLProgram(ProgramrootContext ast, String sourceNamespace) {
    DMLProgram dmlPgm = new DMLProgram();
    String namespace = (sourceNamespace != null && sourceNamespace.length() > 0) ? sourceNamespace : DMLProgram.DEFAULT_NAMESPACE;
    dmlPgm.getNamespaces().put(namespace, dmlPgm);
    // First add all the functions
    for (FunctionStatementContext fn : ast.functionBlocks) {
        FunctionStatementBlock functionStmtBlk = new FunctionStatementBlock();
        functionStmtBlk.addStatement(fn.info.stmt);
        try {
            dmlPgm.addFunctionStatementBlock(namespace, fn.info.functionName, functionStmtBlk);
        } catch (LanguageException e) {
            LOG.error("line: " + fn.start.getLine() + ":" + fn.start.getCharPositionInLine() + " cannot process the function " + fn.info.functionName);
            return null;
        }
    }
    // Then add all the statements
    for (StatementContext stmtCtx : ast.blocks) {
        Statement current = stmtCtx.info.stmt;
        if (current == null) {
            LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement");
            return null;
        }
        // Ignore Newline logic 
        if (current.isEmptyNewLineStatement()) {
            continue;
        }
        if (current instanceof ImportStatement) {
            // Handle import statements separately
            if (stmtCtx.info.namespaces != null) {
                // Add the DMLProgram entries into current program
                for (Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) {
                    // TODO handle namespace key already exists for different program value instead of overwriting
                    DMLProgram prog = entry.getValue();
                    if (prog != null && prog.getNamespaces().size() > 0) {
                        dmlPgm.getNamespaces().put(entry.getKey(), prog);
                    }
                    // Add dependent programs (handle imported script that also imports scripts)
                    for (Map.Entry<String, DMLProgram> dependency : entry.getValue().getNamespaces().entrySet()) {
                        String depNamespace = dependency.getKey();
                        DMLProgram depProgram = dependency.getValue();
                        if (dmlPgm.getNamespaces().get(depNamespace) == null) {
                            dmlPgm.getNamespaces().put(depNamespace, depProgram);
                        }
                    }
                }
            } else {
                LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the import statement");
                return null;
            }
        }
        // Now wrap statement into individual statement block
        // merge statement will take care of merging these blocks
        dmlPgm.addStatementBlock(getStatementBlock(current));
    }
    dmlPgm.mergeStatementBlocks();
    return dmlPgm;
}

Example 3 with DMLProgram

use of org.apache.sysml.parser.DMLProgram in project incubator-systemml by apache.

the class DMLParserWrapper method createDMLProgram.

private DMLProgram createDMLProgram(ProgramrootContext ast, String sourceNamespace) {
    DMLProgram dmlPgm = new DMLProgram();
    String namespace = (sourceNamespace != null && sourceNamespace.length() > 0) ? sourceNamespace : DMLProgram.DEFAULT_NAMESPACE;
    dmlPgm.getNamespaces().put(namespace, dmlPgm);
    // First add all the functions
    for (FunctionStatementContext fn : ast.functionBlocks) {
        FunctionStatementBlock functionStmtBlk = new FunctionStatementBlock();
        functionStmtBlk.addStatement(fn.info.stmt);
        try {
            dmlPgm.addFunctionStatementBlock(namespace, fn.info.functionName, functionStmtBlk);
        } catch (LanguageException e) {
            LOG.error("line: " + fn.start.getLine() + ":" + fn.start.getCharPositionInLine() + " cannot process the function " + fn.info.functionName);
            return null;
        }
    }
    // Then add all the statements
    for (StatementContext stmtCtx : ast.blocks) {
        org.apache.sysml.parser.Statement current = stmtCtx.info.stmt;
        if (current == null) {
            LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement");
            return null;
        }
        if (current instanceof ImportStatement) {
            // Handle import statements separately
            if (stmtCtx.info.namespaces != null) {
                // Add the DMLProgram entries into current program
                for (Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) {
                    // TODO handle namespace key already exists for different program value instead of overwriting
                    DMLProgram prog = entry.getValue();
                    if (prog != null && prog.getNamespaces().size() > 0) {
                        dmlPgm.getNamespaces().put(entry.getKey(), prog);
                    }
                    // Add dependent programs (handle imported script that also imports scripts)
                    for (Map.Entry<String, DMLProgram> dependency : entry.getValue().getNamespaces().entrySet()) {
                        String depNamespace = dependency.getKey();
                        DMLProgram depProgram = dependency.getValue();
                        if (dmlPgm.getNamespaces().get(depNamespace) == null) {
                            dmlPgm.getNamespaces().put(depNamespace, depProgram);
                        }
                    }
                }
            } else {
                LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the import statement");
                return null;
            }
        }
        // Now wrap statement into individual statement block
        // merge statement will take care of merging these blocks
        dmlPgm.addStatementBlock(getStatementBlock(current));
    }
    dmlPgm.mergeStatementBlocks();
    return dmlPgm;
}

Example 4 with DMLProgram

use of org.apache.sysml.parser.DMLProgram in project incubator-systemml by apache.

the class Explain method explain.

public static String explain(Program rtprog) throws HopsException {
    //counts number of instructions
    boolean sparkExec = OptimizerUtils.isSparkExecutionMode();
    ExplainCounts counts = new ExplainCounts();
    countCompiledInstructions(rtprog, counts, !sparkExec, true, sparkExec);
    StringBuilder sb = new StringBuilder();
    //create header
    sb.append("\nPROGRAM ( size CP/" + (sparkExec ? "SP" : "MR") + " = ");
    sb.append(counts.numCPInst);
    sb.append("/");
    sb.append(counts.numJobs);
    sb.append(" )\n");
    //explain functions (if exists)
    Map<String, FunctionProgramBlock> funcMap = rtprog.getFunctionProgramBlocks();
    if (funcMap != null && !funcMap.isEmpty()) {
        sb.append("--FUNCTIONS\n");
        //show function call graph
        if (!rtprog.getProgramBlocks().isEmpty() && rtprog.getProgramBlocks().get(0).getStatementBlock() != null) {
            sb.append("----FUNCTION CALL GRAPH\n");
            sb.append("------MAIN PROGRAM\n");
            DMLProgram prog = rtprog.getProgramBlocks().get(0).getStatementBlock().getDMLProg();
            FunctionCallGraph fgraph = new FunctionCallGraph(prog);
            sb.append(explainFunctionCallGraph(fgraph, new HashSet<String>(), null, 3));
        }
        //show individual functions
        for (Entry<String, FunctionProgramBlock> e : funcMap.entrySet()) {
            String fkey = e.getKey();
            FunctionProgramBlock fpb = e.getValue();
            if (fpb instanceof ExternalFunctionProgramBlock)
                sb.append("----EXTERNAL FUNCTION " + fkey + "\n");
            else {
                sb.append("----FUNCTION " + fkey + " [recompile=" + fpb.isRecompileOnce() + "]\n");
                for (ProgramBlock pb : fpb.getChildBlocks()) sb.append(explainProgramBlock(pb, 3));
            }
        }
    }
    //explain main program
    sb.append("--MAIN PROGRAM\n");
    for (ProgramBlock pb : rtprog.getProgramBlocks()) sb.append(explainProgramBlock(pb, 2));
    return sb.toString();
}

Example 5 with DMLProgram

use of org.apache.sysml.parser.DMLProgram in project incubator-systemml by apache.

the class DMLScript method launchDebugger.

/**
	 * Launcher for DML debugger. This method should be called after 
	 * execution and debug properties have been correctly set, and customized parameters
	 * 
	 * @param dmlScriptStr DML script contents (including new lines)
	 * @param fnameOptConfig Full path of configuration file for SystemML
	 * @param argVals Key-value pairs defining arguments of DML script
	 * @param scriptType type of script (DML or PyDML)
	 * @throws ParseException if ParseException occurs
	 * @throws IOException if IOException occurs
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 * @throws DMLDebuggerException if DMLDebuggerException occurs
	 * @throws LanguageException if LanguageException occurs
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 */
private static void launchDebugger(String dmlScriptStr, String fnameOptConfig, Map<String, String> argVals, ScriptType scriptType) throws ParseException, IOException, DMLRuntimeException, DMLDebuggerException, LanguageException, HopsException, LopsException {
    DMLDebuggerProgramInfo dbprog = new DMLDebuggerProgramInfo();
    //Step 1: parse configuration files
    DMLConfig conf = DMLConfig.readConfigurationFile(fnameOptConfig);
    ConfigurationManager.setGlobalConfig(conf);
    //Step 2: parse dml script
    ParserWrapper parser = ParserFactory.createParser(scriptType);
    DMLProgram prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
    //Step 3: construct HOP DAGs (incl LVA and validate)
    DMLTranslator dmlt = new DMLTranslator(prog);
    dmlt.liveVariableAnalysis(prog);
    dmlt.validateParseTree(prog);
    dmlt.constructHops(prog);
    //Step 4: rewrite HOP DAGs (incl IPA and memory estimates)
    dmlt.rewriteHopsDAG(prog);
    //Step 5: construct LOP DAGs
    dmlt.constructLops(prog);
    //Step 6: generate runtime program
    dbprog.rtprog = prog.getRuntimeProgram(conf);
    try {
        //set execution environment
        initHadoopExecution(conf);
        //initialize an instance of SystemML debugger
        DMLDebugger SystemMLdb = new DMLDebugger(dbprog, dmlScriptStr);
        //run SystemML debugger
        SystemMLdb.runSystemMLDebugger();
    } finally {
        //cleanup scratch_space and all working dirs
        cleanupHadoopExecution(conf);
    }
}