Examples with Timing org.apache.sysml.runtime.controlprogram.parfor.stat.Timing used on opensource projects

Example 16 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project incubator-systemml by apache.

the class CompressedMatrixBlock method transposeSelfMatrixMultOperations.

@Override
public MatrixBlock transposeSelfMatrixMultOperations(MatrixBlock out, MMTSJType tstype) {
    // call uncompressed matrix mult if necessary
    if (!isCompressed()) {
        return super.transposeSelfMatrixMultOperations(out, tstype);
    }
    // single-threaded tsmm of single uncompressed colgroup
    if (isSingleUncompressedGroup()) {
        return ((ColGroupUncompressed) _colGroups.get(0)).getData().transposeSelfMatrixMultOperations(out, tstype);
    }
    Timing time = LOG.isDebugEnabled() ? new Timing(true) : null;
    // check for transpose type
    if (// right not supported yet
    tstype != MMTSJType.LEFT)
        throw new DMLRuntimeException("Invalid MMTSJ type '" + tstype.toString() + "'.");
    // create output matrix block
    if (out == null)
        out = new MatrixBlock(clen, clen, false);
    else
        out.reset(clen, clen, false);
    out.allocateDenseBlock();
    if (!isEmptyBlock(false)) {
        // compute matrix mult
        leftMultByTransposeSelf(_colGroups, out, 0, _colGroups.size());
        // post-processing
        out.setNonZeros(LinearAlgebraUtils.copyUpperToLowerTriangle(out));
    }
    if (LOG.isDebugEnabled())
        LOG.debug("Compressed TSMM in " + time.stop());
    return out;
}

Example 17 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project incubator-systemml by apache.

the class DMLYarnClient method launchDMLYarnAppmaster.

/**
 * Method to launch the dml yarn app master and execute the given dml script
 * with the given configuration and jar file.
 *
 * NOTE: on launching the yarn app master, we do not explicitly probe if we
 *	  are running on a yarn or MR1 cluster. In case of MR1, already the class
 *	  YarnConfiguration will not be found and raise a classnotfound. In case of any
 *	  exception we fall back to run CP directly in the client process.
 *
 * @return true if dml program successfully executed as yarn app master
 * @throws IOException if IOException occurs
 * @throws DMLScriptException if DMLScriptException occurs
 */
protected boolean launchDMLYarnAppmaster() throws IOException, DMLScriptException {
    boolean ret = false;
    String hdfsWD = null;
    try {
        Timing time = new Timing(true);
        // load yarn configuration
        YarnConfiguration yconf = new YarnConfiguration();
        // create yarn client
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(yconf);
        yarnClient.start();
        // create application and get the ApplicationID
        YarnClientApplication app = yarnClient.createApplication();
        ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
        ApplicationId appId = appContext.getApplicationId();
        LOG.debug("Created application (applicationID: " + appId + ")");
        // prepare hdfs working directory via ApplicationID
        // copy script, config, jar file to hdfs
        hdfsWD = DMLAppMasterUtils.constructHDFSWorkingDir(_dmlConfig, appId);
        copyResourcesToHdfsWorkingDir(yconf, hdfsWD);
        // construct command line argument
        String command = constructAMCommand(_args, _dmlConfig);
        LOG.debug("Constructed application master command: \n" + command);
        // set up the container launch context for the application master
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
        amContainer.setCommands(Collections.singletonList(command));
        amContainer.setLocalResources(constructLocalResourceMap(yconf));
        amContainer.setEnvironment(constructEnvionmentMap(yconf));
        // Set up resource type requirements for ApplicationMaster
        int memHeap = _dmlConfig.getIntValue(DMLConfig.YARN_APPMASTERMEM);
        int memAlloc = (int) computeMemoryAllocation(memHeap);
        Resource capability = Records.newRecord(Resource.class);
        capability.setMemory(memAlloc);
        capability.setVirtualCores(NUM_CORES);
        LOG.debug("Requested application resources: memory=" + memAlloc + ", vcores=" + NUM_CORES);
        // Finally, set-up ApplicationSubmissionContext for the application
        String qname = _dmlConfig.getTextValue(DMLConfig.YARN_APPQUEUE);
        // application name
        appContext.setApplicationName(APPMASTER_NAME);
        appContext.setAMContainerSpec(amContainer);
        appContext.setResource(capability);
        // queue
        appContext.setQueue(qname);
        LOG.debug("Configured application meta data: name=" + APPMASTER_NAME + ", queue=" + qname);
        // submit application (non-blocking)
        yarnClient.submitApplication(appContext);
        // Check application status periodically (and output web ui address)
        ApplicationReport appReport = yarnClient.getApplicationReport(appId);
        LOG.info("Application tracking-URL: " + appReport.getTrackingUrl());
        YarnApplicationState appState = appReport.getYarnApplicationState();
        YarnApplicationState oldState = appState;
        LOG.info("Application state: " + appState);
        while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) {
            // wait for 200ms
            Thread.sleep(APP_STATE_INTERVAL);
            appReport = yarnClient.getApplicationReport(appId);
            appState = appReport.getYarnApplicationState();
            if (appState != oldState) {
                oldState = appState;
                LOG.info("Application state: " + appState);
            }
        }
        // check final status (failed or succeeded)
        FinalApplicationStatus finalState = appReport.getFinalApplicationStatus();
        LOG.info("Application final status: " + finalState);
        // show application and total runtime
        double appRuntime = (double) (appReport.getFinishTime() - appReport.getStartTime()) / 1000;
        LOG.info("Application runtime: " + appRuntime + " sec.");
        LOG.info("Total runtime: " + String.format("%.3f", time.stop() / 1000) + " sec.");
        // raised script-level error in case of failed final status
        if (finalState != FinalApplicationStatus.SUCCEEDED) {
            // propagate script-level stop call message
            String stop_msg = readMessageToHDFSWorkingDir(_dmlConfig, yconf, appId);
            if (stop_msg != null)
                throw new DMLScriptException(stop_msg);
            // generic failure message
            throw new DMLRuntimeException("DML yarn app master finished with final status: " + finalState + ".");
        }
        ret = true;
    } catch (DMLScriptException ex) {
        // rethrow DMLScriptException to propagate stop call
        throw ex;
    } catch (Exception ex) {
        LOG.error("Failed to run DML yarn app master.", ex);
        ret = false;
    } finally {
        // cleanup working directory
        if (hdfsWD != null)
            MapReduceTool.deleteFileIfExistOnHDFS(hdfsWD);
    }
    return ret;
}

Example 18 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project incubator-systemml by apache.

the class ResourceOptimizer method optimizeResourceConfig.

public static synchronized ResourceConfig optimizeResourceConfig(ArrayList<ProgramBlock> prog, YarnClusterConfig cc, GridEnumType cptype, GridEnumType mrtype) {
    ResourceConfig ROpt = null;
    try {
        // init statistics and counters
        Timing time = new Timing(true);
        initStatistics();
        // get constraints (yarn-specific: force higher min to limit degree of parallelism)
        long max = (long) (YarnOptimizerUtils.toB(cc.getMaxAllocationMB()) / DMLYarnClient.MEM_FACTOR);
        long minCP = (long) Math.max(YarnOptimizerUtils.toB(cc.getMinAllocationMB()) / DMLYarnClient.MEM_FACTOR, MIN_CP_BUDGET);
        long minMR = YarnOptimizerUtils.computeMinContraint(minCP, max, cc.getAvgNumCores());
        // enumerate grid points for given types (refers to jvm max heap)
        ArrayList<Long> SRc = enumerateGridPoints(prog, minCP, max, cptype);
        ArrayList<Long> SRm = enumerateGridPoints(prog, minMR, max, mrtype);
        // init resource config and global costs
        ROpt = new ResourceConfig(prog, minMR);
        double costOpt = Double.MAX_VALUE;
        for (// enumerate CP memory rc
        Long rc : // enumerate CP memory rc
        SRc) {
            // baseline compile and pruning
            // unrolled Bp
            ArrayList<ProgramBlock> B = compileProgram(prog, null, rc, minMR);
            ArrayList<ProgramBlock> Bp = pruneProgramBlocks(B);
            LOG.debug("Enum (rc=" + rc + "): |B|=" + B.size() + ", |Bp|=" + Bp.size());
            // init local memo table [resource, cost]
            double[][] memo = initLocalMemoTable(Bp, minMR);
            for (// for all relevant blocks
            int i = 0; // for all relevant blocks
            i < Bp.size(); // for all relevant blocks
            i++) {
                ProgramBlock pb = Bp.get(i);
                for (// for each MR memory
                Long rm : // for each MR memory
                SRm) {
                    // recompile program block
                    recompileProgramBlock(pb, rc, rm);
                    // local costing and memo table maintenance (cost entire program to account for
                    // in-memory status of variables and loops)
                    double lcost = getProgramCosts(pb);
                    if (lcost < memo[i][1]) {
                        // accept new local opt
                        memo[i][0] = rm;
                        memo[i][1] = lcost;
                    // LOG.debug("Enum (rc="+rc+"): found new local opt w/ cost="+lcost);
                    }
                // LOG.debug("Enum (rc="+rc+", rm="+rm+"): lcost="+lcost+", mincost="+memo[i][1]);
                }
            }
            // global costing
            double[][] gmemo = initGlobalMemoTable(B, Bp, memo, minMR);
            recompileProgramBlocks(B, rc, gmemo);
            double gcost = getProgramCosts(B.get(0).getProgram());
            if (gcost < costOpt) {
                // accept new global opt
                ROpt.setCPResource(rc.longValue());
                ROpt.setMRResources(B, gmemo);
                costOpt = gcost;
                LOG.debug("Enum (rc=" + rc + "): found new opt w/ cost=" + gcost);
            }
        }
        // print optimization summary
        LOG.info("Optimization summary:");
        LOG.info("-- optimal plan (rc, rm): " + YarnOptimizerUtils.toMB(ROpt.getCPResource()) + "MB, " + YarnOptimizerUtils.toMB(ROpt.getMaxMRResource()) + "MB");
        LOG.info("-- costs of optimal plan: " + costOpt);
        LOG.info("-- # of block compiles:   " + _cntCompilePB);
        LOG.info("-- # of block costings:   " + _cntCostPB);
        LOG.info("-- optimization time:     " + String.format("%.3f", (double) time.stop() / 1000) + " sec.");
        LOG.info("-- optimal plan details:  " + ROpt.serialize());
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    return ROpt;
}

Example 19 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project incubator-systemml by apache.

the class FrameCastingTest method execDMLScriptviaJMLC.

private static ArrayList<String[][]> execDMLScriptviaJMLC(String testname, String[][] F1, boolean modelReuse) throws IOException {
    Timing time = new Timing(true);
    ArrayList<String[][]> ret = new ArrayList<String[][]>();
    // establish connection to SystemML
    Connection conn = new Connection();
    try {
        // prepare input arguments
        HashMap<String, String> args = new HashMap<String, String>();
        args.put("$TRANSFORM_SPEC", "{ \"ids\": true ,\"recode\": [ 1, 2, 3] }");
        // read and precompile script
        String script = conn.readScript(SCRIPT_DIR + TEST_DIR + testname + ".dml");
        PreparedScript pstmt = conn.prepareScript(script, args, new String[] { "F1", "M" }, new String[] { "F2" }, false);
        if (modelReuse)
            pstmt.setFrame("F1", F1, true);
        // execute script multiple times
        for (int i = 0; i < nRuns; i++) {
            // bind input parameters
            if (!modelReuse)
                pstmt.setFrame("F1", F1);
            // execute script
            ResultVariables rs = pstmt.executeScript();
            // get output parameter
            String[][] Y = rs.getFrame("F2");
            // keep result for comparison
            ret.add(Y);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new IOException(ex);
    } finally {
        IOUtilFunctions.closeSilently(conn);
    }
    System.out.println("JMLC scoring w/ " + nRuns + " runs in " + time.stop() + "ms.");
    return ret;
}

Example 20 with Timing

use of org.apache.sysml.runtime.controlprogram.parfor.stat.Timing in project incubator-systemml by apache.

the class FrameDecodeTest method execDMLScriptviaJMLC.

private static ArrayList<String[][]> execDMLScriptviaJMLC(String testname, String[][] F1, String[][] M, boolean modelReuse) throws IOException {
    Timing time = new Timing(true);
    ArrayList<String[][]> ret = new ArrayList<String[][]>();
    // establish connection to SystemML
    Connection conn = new Connection();
    try {
        // prepare input arguments
        HashMap<String, String> args = new HashMap<String, String>();
        args.put("$TRANSFORM_SPEC", "{ \"ids\": true ,\"recode\": [ 1, 2, 3] }");
        // read and precompile script
        String script = conn.readScript(SCRIPT_DIR + TEST_DIR + testname + ".dml");
        PreparedScript pstmt = conn.prepareScript(script, args, new String[] { "F1", "M" }, new String[] { "F2" }, false);
        if (modelReuse)
            pstmt.setFrame("M", M, true);
        // execute script multiple times
        for (int i = 0; i < nRuns; i++) {
            // bind input parameters
            if (!modelReuse)
                pstmt.setFrame("M", M);
            pstmt.setFrame("F1", F1);
            // execute script
            ResultVariables rs = pstmt.executeScript();
            // get output parameter
            String[][] Y = rs.getFrame("F2");
            // keep result for comparison
            ret.add(Y);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new IOException(ex);
    } finally {
        IOUtilFunctions.closeSilently(conn);
    }
    System.out.println("JMLC scoring w/ " + nRuns + " runs in " + time.stop() + "ms.");
    return ret;
}