Examples with ResultVar org.apache.sysml.parser.ParForStatementBlock.ResultVar used on opensource projects

Example 11 with ResultVar

use of org.apache.sysml.parser.ParForStatementBlock.ResultVar in project incubator-systemml by apache.

the class ParForProgramBlock method consolidateAndCheckResults.

private void consolidateAndCheckResults(ExecutionContext ec, long expIters, long expTasks, long numIters, long numTasks, LocalVariableMap[] results) {
    Timing time = new Timing(true);
    // result merge
    if (checkParallelRemoteResultMerge()) {
        // execute result merge in parallel for all result vars
        int par = Math.min(_resultVars.size(), InfrastructureAnalyzer.getLocalParallelism());
        if (InfrastructureAnalyzer.isLocalMode()) {
            int parmem = (int) Math.floor(OptimizerUtils.getLocalMemBudget() / InfrastructureAnalyzer.getRemoteMaxMemorySortBuffer());
            // reduce k if necessary
            par = Math.min(par, Math.max(parmem, 1));
        }
        try {
            // enqueue all result vars as tasks
            LocalTaskQueue<ResultVar> q = new LocalTaskQueue<>();
            for (ResultVar var : _resultVars) {
                // foreach non-local write
                if (// robustness scalars
                ec.getVariable(var._name) instanceof MatrixObject)
                    q.enqueueTask(var);
            }
            q.closeInput();
            // run result merge workers
            ResultMergeWorker[] rmWorkers = new ResultMergeWorker[par];
            for (int i = 0; i < par; i++) rmWorkers[i] = new ResultMergeWorker(q, results, ec);
            for (// start all
            int i = 0; // start all
            i < par; // start all
            i++) rmWorkers[i].start();
            for (int i = 0; i < par; i++) {
                // wait for all
                rmWorkers[i].join();
                if (!rmWorkers[i].finishedNoError())
                    throw new DMLRuntimeException("Error occured in parallel result merge worker.");
            }
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
    } else {
        // execute result merge sequentially for all result vars
        for (// foreach non-local write
        ResultVar var : // foreach non-local write
        _resultVars) {
            Data dat = ec.getVariable(var._name);
            if (// robustness scalars
            dat instanceof MatrixObject) {
                MatrixObject out = (MatrixObject) dat;
                MatrixObject[] in = new MatrixObject[results.length];
                for (int i = 0; i < results.length; i++) in[i] = (MatrixObject) results[i].get(var._name);
                String fname = constructResultMergeFileName();
                ResultMerge rm = createResultMerge(_resultMerge, out, in, fname, var._isAccum, ec);
                MatrixObject outNew = null;
                if (USE_PARALLEL_RESULT_MERGE)
                    outNew = rm.executeParallelMerge(_numThreads);
                else
                    outNew = rm.executeSerialMerge();
                // cleanup existing var
                Data exdata = ec.removeVariable(var._name);
                if (exdata != null && exdata != outNew && exdata instanceof MatrixObject)
                    ec.cleanupCacheableData((MatrixObject) exdata);
                // cleanup of intermediate result variables
                cleanWorkerResultVariables(ec, out, in);
                // set merged result variable
                ec.setVariable(var._name, outNew);
            }
        }
    }
    // handle unscoped variables (vars created in parfor, but potentially used afterwards)
    ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
    if (// sb might be null for nested parallelism
    CREATE_UNSCOPED_RESULTVARS && sb != null && ec.getVariables() != null)
        createEmptyUnscopedVariables(ec.getVariables(), sb);
    // check expected counters
    if (// consistency check
    numTasks != expTasks || numIters != expIters)
        throw new DMLRuntimeException("PARFOR: Number of executed tasks does not match the number of created tasks: tasks " + numTasks + "/" + expTasks + ", iters " + numIters + "/" + expIters + ".");
    if (DMLScript.STATISTICS)
        Statistics.incrementParForMergeTime((long) time.stop());
}

Example 12 with ResultVar

use of org.apache.sysml.parser.ParForStatementBlock.ResultVar in project incubator-systemml by apache.

the class ProgramConverter method parseResultVariables.

private static ArrayList<ResultVar> parseResultVariables(String in) {
    ArrayList<ResultVar> ret = new ArrayList<>();
    for (String var : parseStringArrayList(in)) {
        boolean accum = var.endsWith("+");
        ret.add(new ResultVar(accum ? var.substring(0, var.length() - 1) : var, accum));
    }
    return ret;
}

Example 13 with ResultVar

use of org.apache.sysml.parser.ParForStatementBlock.ResultVar in project incubator-systemml by apache.

the class RemoteParForUtils method exportResultVariables.

/**
 * For remote Spark parfor workers. This is a simplified version compared to MR.
 *
 * @param workerID worker id
 * @param vars local variable map
 * @param resultVars list of result variables
 * @return list of result variables
 * @throws IOException if IOException occurs
 */
public static ArrayList<String> exportResultVariables(long workerID, LocalVariableMap vars, ArrayList<ResultVar> resultVars) throws IOException {
    ArrayList<String> ret = new ArrayList<>();
    // foreach result variables probe if export necessary
    for (ResultVar rvar : resultVars) {
        Data dat = vars.get(rvar._name);
        // export output variable to HDFS (see RunMRJobs)
        if (dat != null && dat.getDataType() == DataType.MATRIX) {
            MatrixObject mo = (MatrixObject) dat;
            if (mo.isDirty()) {
                // export result var (iff actually modified in parfor)
                mo.exportData();
                // pass output vars (scalars by value, matrix by ref) to result
                // (only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
                ret.add(ProgramConverter.serializeDataObject(rvar._name, mo));
            }
        }
    }
    return ret;
}

Example 14 with ResultVar

use of org.apache.sysml.parser.ParForStatementBlock.ResultVar in project incubator-systemml by apache.

the class RemoteParForUtils method exportResultVariables.

/**
 * For remote MR parfor workers.
 *
 * @param workerID worker id
 * @param vars local variable map
 * @param resultVars list of result variables
 * @param rvarFnames ?
 * @param out output collectors
 * @throws IOException if IOException occurs
 */
public static void exportResultVariables(long workerID, LocalVariableMap vars, ArrayList<ResultVar> resultVars, HashMap<String, String> rvarFnames, OutputCollector<Writable, Writable> out) throws IOException {
    // create key and value for reuse
    LongWritable okey = new LongWritable(workerID);
    Text ovalue = new Text();
    // foreach result variables probe if export necessary
    for (ResultVar rvar : resultVars) {
        Data dat = vars.get(rvar._name);
        // export output variable to HDFS (see RunMRJobs)
        if (dat != null && dat.getDataType() == DataType.MATRIX) {
            MatrixObject mo = (MatrixObject) dat;
            if (mo.isDirty()) {
                if (ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER && rvarFnames != null) {
                    String fname = rvarFnames.get(rvar._name);
                    if (fname != null)
                        mo.setFileName(fname);
                    // export result var (iff actually modified in parfor)
                    // note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
                    mo.exportData();
                    rvarFnames.put(rvar._name, mo.getFileName());
                } else {
                    // export result var (iff actually modified in parfor)
                    // note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
                    mo.exportData();
                }
                // pass output vars (scalars by value, matrix by ref) to result
                // (only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
                String datStr = ProgramConverter.serializeDataObject(rvar._name, mo);
                ovalue.set(datStr);
                out.collect(okey, ovalue);
            }
        }
    }
}