Examples with CompilerConfig org.apache.sysml.conf.CompilerConfig used on opensource projects

Example 6 with CompilerConfig

use of org.apache.sysml.conf.CompilerConfig in project incubator-systemml by apache.

the class FrameReadWriteTest method runFrameReadWriteTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runFrameReadWriteTest(OutputInfo oinfo, ValueType[] schema1, ValueType[] schema2, boolean parallel) {
    boolean oldParText = CompilerConfig.FLAG_PARREADWRITE_TEXT;
    boolean oldParBin = CompilerConfig.FLAG_PARREADWRITE_BINARY;
    try {
        CompilerConfig.FLAG_PARREADWRITE_TEXT = parallel;
        CompilerConfig.FLAG_PARREADWRITE_BINARY = parallel;
        ConfigurationManager.setGlobalConfig(new CompilerConfig());
        // data generation
        double[][] A = getRandomMatrix(rows, schema1.length, -10, 10, 0.9, 2373);
        double[][] B = getRandomMatrix(rows, schema2.length, -10, 10, 0.9, 129);
        // Initialize the frame data.
        // init data frame 1
        FrameBlock frame1 = new FrameBlock(schema1);
        initFrameData(frame1, A, schema1);
        // init data frame 2
        FrameBlock frame2 = new FrameBlock(schema2);
        initFrameData(frame2, B, schema2);
        // Write frame data to disk
        CSVFileFormatProperties fprop = new CSVFileFormatProperties();
        fprop.setDelim(DELIMITER);
        fprop.setHeader(HEADER);
        writeAndVerifyData(oinfo, frame1, frame2, fprop);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    } finally {
        CompilerConfig.FLAG_PARREADWRITE_TEXT = oldParText;
        CompilerConfig.FLAG_PARREADWRITE_BINARY = oldParBin;
        ConfigurationManager.setGlobalConfig(new CompilerConfig());
    }
}

Example 7 with CompilerConfig

use of org.apache.sysml.conf.CompilerConfig in project systemml by apache.

the class DMLAppMasterUtils method setupConfigRemoteMaxMemory.

public static void setupConfigRemoteMaxMemory(DMLConfig conf) {
    // set remote max memory (if in yarn appmaster context)
    if (DMLScript.isActiveAM()) {
        // set optimization level (for awareness of resource optimization)
        CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(conf);
        ConfigurationManager.setGlobalConfig(cconf);
        if (isResourceOptimizerEnabled()) {
            // handle optimized memory (mr memory budget per program block)
            // ensure cluster has been analyzed
            InfrastructureAnalyzer.getRemoteMaxMemoryMap();
            String memStr = conf.getTextValue(DMLConfig.YARN_MAPREDUCEMEM);
            ResourceConfig rc = ResourceConfig.deserialize(memStr);
            // keep resource config for later program mapping
            _rc = rc;
        } else {
            // handle user configuration
            if (conf.getIntValue(DMLConfig.YARN_MAPREDUCEMEM) > 0) {
                // ensure cluster has been analyzed
                InfrastructureAnalyzer.getRemoteMaxMemoryMap();
                // set max map and reduce memory (to be used by the compiler)
                // see GMR and parfor EMR and DPEMR for runtime configuration
                long mem = ((long) conf.getIntValue(DMLConfig.YARN_MAPREDUCEMEM)) * 1024 * 1024;
                InfrastructureAnalyzer.setRemoteMaxMemoryMap(mem);
                InfrastructureAnalyzer.setRemoteMaxMemoryReduce(mem);
            }
        }
    }
}

Example 8 with CompilerConfig

use of org.apache.sysml.conf.CompilerConfig in project systemml by apache.

the class ProgramConverter method parseParForBody.

// //////////////////////////////
// PARSING
// //////////////////////////////
public static ParForBody parseParForBody(String in, int id) {
    ParForBody body = new ParForBody();
    // header elimination
    // normalization
    String tmpin = in.replaceAll(NEWLINE, "");
    // remove start/end
    tmpin = tmpin.substring(PARFORBODY_BEGIN.length(), tmpin.length() - PARFORBODY_END.length());
    HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(tmpin, COMPONENTS_DELIM);
    // handle DMLScript UUID (NOTE: set directly in DMLScript)
    // (master UUID is used for all nodes (in order to simply cleanup))
    DMLScript.setUUID(st.nextToken());
    // handle DML config (NOTE: set directly in ConfigurationManager)
    String confStr = st.nextToken();
    JobConf job = ConfigurationManager.getCachedJobConf();
    if (!InfrastructureAnalyzer.isLocalMode(job)) {
        if (confStr != null && !confStr.trim().isEmpty()) {
            DMLConfig dmlconf = DMLConfig.parseDMLConfig(confStr);
            CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
            ConfigurationManager.setLocalConfig(dmlconf);
            ConfigurationManager.setLocalConfig(cconf);
        }
        // init internal configuration w/ parsed or default config
        ParForProgramBlock.initInternalConfigurations(ConfigurationManager.getDMLConfig());
    }
    // handle additional configs
    String aconfs = st.nextToken();
    parseAndSetAdditionalConfigurations(aconfs);
    // handle program
    String progStr = st.nextToken();
    Program prog = parseProgram(progStr, id);
    // handle result variable names
    String rvarStr = st.nextToken();
    ArrayList<ResultVar> rvars = parseResultVariables(rvarStr);
    body.setResultVariables(rvars);
    // handle execution context
    String ecStr = st.nextToken();
    ExecutionContext ec = parseExecutionContext(ecStr, prog);
    // handle program blocks
    String spbs = st.nextToken();
    ArrayList<ProgramBlock> pbs = rParseProgramBlocks(spbs, prog, id);
    body.setChildBlocks(pbs);
    body.setEc(ec);
    return body;
}

Example 9 with CompilerConfig

use of org.apache.sysml.conf.CompilerConfig in project systemml by apache.

the class DMLScript method execute.

// /////////////////////////////
// private internal interface
// (core compilation and execute)
// //////
/**
 * The running body of DMLScript execution. This method should be called after execution properties have been correctly set,
 * and customized parameters have been put into _argVals
 *
 * @param dmlScriptStr DML script string
 * @param fnameOptConfig configuration file
 * @param argVals map of argument values
 * @param allArgs arguments
 * @param scriptType type of script (DML or PyDML)
 * @throws IOException if IOException occurs
 */
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String, String> argVals, String[] allArgs, ScriptType scriptType) throws IOException {
    SCRIPT_TYPE = scriptType;
    // print basic time and environment info
    printStartExecInfo(dmlScriptStr);
    // Step 1: parse configuration files & write any configuration specific global variables
    DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig);
    ConfigurationManager.setGlobalConfig(dmlconf);
    CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
    ConfigurationManager.setGlobalConfig(cconf);
    LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo());
    // Sets the GPUs to use for this process (a range, all GPUs, comma separated list or a specific GPU)
    GPUContextPool.AVAILABLE_GPUS = dmlconf.getTextValue(DMLConfig.AVAILABLE_GPUS);
    String evictionPolicy = dmlconf.getTextValue(DMLConfig.GPU_EVICTION_POLICY).toUpperCase();
    try {
        DMLScript.GPU_EVICTION_POLICY = EvictionPolicy.valueOf(evictionPolicy);
    } catch (IllegalArgumentException e) {
        throw new RuntimeException("Unsupported eviction policy:" + evictionPolicy);
    }
    // Step 2: set local/remote memory if requested (for compile in AM context)
    if (dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER)) {
        DMLAppMasterUtils.setupConfigRemoteMaxMemory(dmlconf);
    }
    // Step 3: parse dml script
    Statistics.startCompileTimer();
    ParserWrapper parser = ParserFactory.createParser(scriptType);
    DMLProgram prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
    // Step 4: construct HOP DAGs (incl LVA, validate, and setup)
    DMLTranslator dmlt = new DMLTranslator(prog);
    dmlt.liveVariableAnalysis(prog);
    dmlt.validateParseTree(prog);
    dmlt.constructHops(prog);
    // init working directories (before usage by following compilation steps)
    initHadoopExecution(dmlconf);
    // Step 5: rewrite HOP DAGs (incl IPA and memory estimates)
    dmlt.rewriteHopsDAG(prog);
    // Step 6: construct lops (incl exec type and op selection)
    dmlt.constructLops(prog);
    if (LOG.isDebugEnabled()) {
        LOG.debug("\n********************** LOPS DAG *******************");
        dmlt.printLops(prog);
        dmlt.resetLopsDAGVisitStatus(prog);
    }
    // Step 7: generate runtime program, incl codegen
    Program rtprog = dmlt.getRuntimeProgram(prog, dmlconf);
    // launch SystemML appmaster (if requested and not already in launched AM)
    if (dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER)) {
        if (!isActiveAM() && DMLYarnClientProxy.launchDMLYarnAppmaster(dmlScriptStr, dmlconf, allArgs, rtprog))
            // if AM launch unsuccessful, fall back to normal execute
            return;
        if (// in AM context (not failed AM launch)
        isActiveAM())
            DMLAppMasterUtils.setupProgramMappingRemoteMaxMemory(rtprog);
    }
    // Step 9: prepare statistics [and optional explain output]
    // count number compiled MR jobs / SP instructions
    ExplainCounts counts = Explain.countDistributedOperations(rtprog);
    Statistics.resetNoOfCompiledJobs(counts.numJobs);
    // explain plan of program (hops or runtime)
    if (EXPLAIN != ExplainType.NONE)
        LOG.info(Explain.display(prog, rtprog, EXPLAIN, counts));
    Statistics.stopCompileTimer();
    // double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext());
    // System.out.println("Estimated costs: "+costs);
    // Step 10: execute runtime program
    ExecutionContext ec = null;
    try {
        ec = ExecutionContextFactory.createContext(rtprog);
        ScriptExecutorUtils.executeRuntimeProgram(rtprog, ec, dmlconf, STATISTICS ? STATISTICS_COUNT : 0);
    } finally {
        if (ec != null && ec instanceof SparkExecutionContext)
            ((SparkExecutionContext) ec).close();
        LOG.info("END DML run " + getDateTime());
        // cleanup scratch_space and all working dirs
        cleanupHadoopExecution(dmlconf);
    }
}

Example 10 with CompilerConfig

use of org.apache.sysml.conf.CompilerConfig in project systemml by apache.

the class MLContextUtil method setCompilerConfig.

/**
 * Set SystemML compiler configuration properties for MLContext
 */
public static void setCompilerConfig() {
    CompilerConfig compilerConfig = new CompilerConfig();
    compilerConfig.set(ConfigType.IGNORE_UNSPECIFIED_ARGS, true);
    compilerConfig.set(ConfigType.REJECT_READ_WRITE_UNKNOWNS, false);
    compilerConfig.set(ConfigType.ALLOW_CSE_PERSISTENT_READS, false);
    compilerConfig.set(ConfigType.MLCONTEXT, true);
    ConfigurationManager.setGlobalConfig(compilerConfig);
}