use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.
the class ProgramConverter method createDeepCopyExecutionContext.
// //////////////////////////////
// CREATION of DEEP COPIES
// //////////////////////////////
/**
* Creates a deep copy of the given execution context.
* For rt_platform=Hadoop, execution context has a symbol table.
*
* @param ec execution context
* @return execution context
* @throws CloneNotSupportedException if CloneNotSupportedException occurs
*/
public static ExecutionContext createDeepCopyExecutionContext(ExecutionContext ec) throws CloneNotSupportedException {
ExecutionContext cpec = ExecutionContextFactory.createContext(false, ec.getProgram());
cpec.setVariables((LocalVariableMap) ec.getVariables().clone());
// (each worker requires its own copy of the empty matrix object)
for (String var : cpec.getVariables().keySet()) {
Data dat = cpec.getVariables().get(var);
if (dat instanceof MatrixObject && ((MatrixObject) dat).getUpdateType().isInPlace()) {
MatrixObject mo = (MatrixObject) dat;
MatrixObject moNew = new MatrixObject(mo);
if (mo.getNnz() != 0) {
// If output matrix is not empty (NNZ != 0), then local copy is created so that
// update in place operation can be applied.
MatrixBlock mbVar = mo.acquireRead();
moNew.acquireModify(new MatrixBlock(mbVar));
mo.release();
} else {
// create empty matrix block w/ dense representation (preferred for update in-place)
// Creating a dense matrix block is valid because empty block not allocated and transfer
// to sparse representation happens in left indexing in place operation.
moNew.acquireModify(new MatrixBlock((int) mo.getNumRows(), (int) mo.getNumColumns(), false));
}
moNew.release();
cpec.setVariable(var, moNew);
}
}
return cpec;
}
use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.
the class ProgramConverter method parseDataObject.
/**
* NOTE: MRJobConfiguration cannot be used for the general case because program blocks and
* related symbol tables can be hierarchically structured.
*
* @param in data object as string
* @return array of objects
*/
public static Object[] parseDataObject(String in) {
Object[] ret = new Object[2];
StringTokenizer st = new StringTokenizer(in, DATA_FIELD_DELIM);
String name = st.nextToken();
DataType datatype = DataType.valueOf(st.nextToken());
ValueType valuetype = ValueType.valueOf(st.nextToken());
String valString = st.hasMoreTokens() ? st.nextToken() : "";
Data dat = null;
switch(datatype) {
case SCALAR:
{
switch(valuetype) {
case INT:
dat = new IntObject(Long.parseLong(valString));
break;
case DOUBLE:
dat = new DoubleObject(Double.parseDouble(valString));
break;
case BOOLEAN:
dat = new BooleanObject(Boolean.parseBoolean(valString));
break;
case STRING:
dat = new StringObject(valString);
break;
default:
throw new DMLRuntimeException("Unable to parse valuetype " + valuetype);
}
break;
}
case MATRIX:
{
MatrixObject mo = new MatrixObject(valuetype, valString);
long rows = Long.parseLong(st.nextToken());
long cols = Long.parseLong(st.nextToken());
int brows = Integer.parseInt(st.nextToken());
int bcols = Integer.parseInt(st.nextToken());
long nnz = Long.parseLong(st.nextToken());
InputInfo iin = InputInfo.stringToInputInfo(st.nextToken());
OutputInfo oin = OutputInfo.stringToOutputInfo(st.nextToken());
PartitionFormat partFormat = PartitionFormat.valueOf(st.nextToken());
UpdateType inplace = UpdateType.valueOf(st.nextToken());
MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, brows, bcols, nnz);
MetaDataFormat md = new MetaDataFormat(mc, oin, iin);
mo.setMetaData(md);
if (partFormat._dpf != PDataPartitionFormat.NONE)
mo.setPartitioned(partFormat._dpf, partFormat._N);
mo.setUpdateType(inplace);
dat = mo;
break;
}
default:
throw new DMLRuntimeException("Unable to parse datatype " + datatype);
}
ret[0] = name;
ret[1] = dat;
return ret;
}
use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.
the class RemoteParForUtils method exportResultVariables.
/**
* For remote Spark parfor workers. This is a simplified version compared to MR.
*
* @param workerID worker id
* @param vars local variable map
* @param resultVars list of result variables
* @return list of result variables
* @throws IOException if IOException occurs
*/
public static ArrayList<String> exportResultVariables(long workerID, LocalVariableMap vars, ArrayList<ResultVar> resultVars) throws IOException {
ArrayList<String> ret = new ArrayList<>();
// foreach result variables probe if export necessary
for (ResultVar rvar : resultVars) {
Data dat = vars.get(rvar._name);
// export output variable to HDFS (see RunMRJobs)
if (dat != null && dat.getDataType() == DataType.MATRIX) {
MatrixObject mo = (MatrixObject) dat;
if (mo.isDirty()) {
// export result var (iff actually modified in parfor)
mo.exportData();
// pass output vars (scalars by value, matrix by ref) to result
// (only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
ret.add(ProgramConverter.serializeDataObject(rvar._name, mo));
}
}
}
return ret;
}
use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.
the class RemoteParForUtils method exportResultVariables.
/**
* For remote MR parfor workers.
*
* @param workerID worker id
* @param vars local variable map
* @param resultVars list of result variables
* @param rvarFnames ?
* @param out output collectors
* @throws IOException if IOException occurs
*/
public static void exportResultVariables(long workerID, LocalVariableMap vars, ArrayList<ResultVar> resultVars, HashMap<String, String> rvarFnames, OutputCollector<Writable, Writable> out) throws IOException {
// create key and value for reuse
LongWritable okey = new LongWritable(workerID);
Text ovalue = new Text();
// foreach result variables probe if export necessary
for (ResultVar rvar : resultVars) {
Data dat = vars.get(rvar._name);
// export output variable to HDFS (see RunMRJobs)
if (dat != null && dat.getDataType() == DataType.MATRIX) {
MatrixObject mo = (MatrixObject) dat;
if (mo.isDirty()) {
if (ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER && rvarFnames != null) {
String fname = rvarFnames.get(rvar._name);
if (fname != null)
mo.setFileName(fname);
// export result var (iff actually modified in parfor)
// note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
mo.exportData();
rvarFnames.put(rvar._name, mo.getFileName());
} else {
// export result var (iff actually modified in parfor)
// note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
mo.exportData();
}
// pass output vars (scalars by value, matrix by ref) to result
// (only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
String datStr = ProgramConverter.serializeDataObject(rvar._name, mo);
ovalue.set(datStr);
out.collect(okey, ovalue);
}
}
}
}
use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.
the class RemoteParWorkerMapper method close.
@Override
public void close() throws IOException {
// cleanup cache and local tmp dir
RemoteParForUtils.cleanupWorkingDirectories();
// delete evicted matrices as well.
if (ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER) {
for (RemoteParWorkerMapper pw : _sCache.values()) {
LocalVariableMap vars = pw._ec.getVariables();
for (String varName : vars.keySet()) {
Data dat = vars.get(varName);
if (dat instanceof MatrixObject)
((MatrixObject) dat).setEmptyStatus();
}
}
}
// ensure caching is not disabled for CP in local mode
CacheableData.enableCaching();
}
Aggregations