Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 46 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class RemoteParForUtils method exportResultVariables.

/**
	 * For remote MR parfor workers.
	 * 
	 * @param workerID worker id
	 * @param vars local variable map
	 * @param resultVars list of result variables
	 * @param rvarFnames ?
	 * @param out output collector
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 * @throws IOException if IOException occurs
	 */
public static void exportResultVariables(long workerID, LocalVariableMap vars, ArrayList<String> resultVars, HashMap<String, String> rvarFnames, OutputCollector<Writable, Writable> out) throws DMLRuntimeException, IOException {
    //create key and value for reuse
    LongWritable okey = new LongWritable(workerID);
    Text ovalue = new Text();
    //foreach result variables probe if export necessary
    for (String rvar : resultVars) {
        Data dat = vars.get(rvar);
        //export output variable to HDFS (see RunMRJobs)
        if (dat != null && dat.getDataType() == DataType.MATRIX) {
            MatrixObject mo = (MatrixObject) dat;
            if (mo.isDirty()) {
                if (ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER && rvarFnames != null) {
                    String fname = rvarFnames.get(rvar);
                    if (fname != null)
                        mo.setFileName(fname);
                    //export result var (iff actually modified in parfor)
                    //note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)		
                    mo.exportData();
                    rvarFnames.put(rvar, mo.getFileName());
                } else {
                    //export result var (iff actually modified in parfor)
                    //note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
                    mo.exportData();
                }
                //pass output vars (scalars by value, matrix by ref) to result
                //(only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
                String datStr = ProgramConverter.serializeDataObject(rvar, mo);
                ovalue.set(datStr);
                out.collect(okey, ovalue);
            }
        }
    }
}

Example 47 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class RemoteParForUtils method exportResultVariables.

/**
	 * For remote Spark parfor workers. This is a simplified version compared to MR.
	 * 
	 * @param workerID worker id
	 * @param vars local variable map
	 * @param resultVars list of result variables
	 * @return list of result variables
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 * @throws IOException if IOException occurs
	 */
public static ArrayList<String> exportResultVariables(long workerID, LocalVariableMap vars, ArrayList<String> resultVars) throws DMLRuntimeException, IOException {
    ArrayList<String> ret = new ArrayList<String>();
    //foreach result variables probe if export necessary
    for (String rvar : resultVars) {
        Data dat = vars.get(rvar);
        //export output variable to HDFS (see RunMRJobs)
        if (dat != null && dat.getDataType() == DataType.MATRIX) {
            MatrixObject mo = (MatrixObject) dat;
            if (mo.isDirty()) {
                //export result var (iff actually modified in parfor)
                mo.exportData();
                //pass output vars (scalars by value, matrix by ref) to result
                //(only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
                ret.add(ProgramConverter.serializeDataObject(rvar, mo));
            }
        }
    }
    return ret;
}