use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.
the class InterProceduralAnalysis method extractFunctionCallReturnStatistics.
/**
* Extract return variable statistics from this function into the
* calling program.
*
* @param fstmt The function statement.
* @param fop The function op.
* @param tmpVars Function's map of variables eligible for
* extraction.
* @param callVars Calling program's map of variables.
* @param overwrite Whether or not to overwrite variables in the
* calling program's variable map.
*/
private static void extractFunctionCallReturnStatistics(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap tmpVars, LocalVariableMap callVars, boolean overwrite) {
ArrayList<DataIdentifier> foutputOps = fstmt.getOutputParams();
String[] outputVars = fop.getOutputVariableNames();
String fkey = fop.getFunctionKey();
try {
for (int i = 0; i < foutputOps.size(); i++) {
DataIdentifier di = foutputOps.get(i);
// name in function signature
String fvarname = di.getName();
// name in calling program
String pvarname = outputVars[i];
// output, remove that variable from the calling program's variable map.
if (callVars.keySet().contains(pvarname)) {
DataType fdataType = di.getDataType();
DataType pdataType = callVars.get(pvarname).getDataType();
if (fdataType != pdataType) {
// datatype has changed, and the calling program is reassigning the
// the variable, so remove it from the calling variable map
callVars.remove(pvarname);
}
}
// Update or add to the calling program's variable map.
if (di.getDataType() == DataType.MATRIX && tmpVars.keySet().contains(fvarname)) {
MatrixObject moIn = (MatrixObject) tmpVars.get(fvarname);
if (// not existing so far
!callVars.keySet().contains(pvarname) || overwrite) {
MatrixObject moOut = createOutputMatrix(moIn.getNumRows(), moIn.getNumColumns(), moIn.getNnz());
callVars.put(pvarname, moOut);
} else // already existing: take largest
{
Data dat = callVars.get(pvarname);
if (dat instanceof MatrixObject) {
MatrixObject moOut = (MatrixObject) dat;
MatrixCharacteristics mc = moOut.getMatrixCharacteristics();
if (OptimizerUtils.estimateSizeExactSparsity(mc.getRows(), mc.getCols(), (mc.getNonZeros() > 0) ? OptimizerUtils.getSparsity(mc) : 1.0) < OptimizerUtils.estimateSize(moIn.getNumRows(), moIn.getNumColumns())) {
// update statistics if necessary
mc.setDimension(moIn.getNumRows(), moIn.getNumColumns());
mc.setNonZeros(moIn.getNnz());
}
}
}
}
}
} catch (Exception ex) {
throw new HopsException("Failed to extract output statistics of function " + fkey + ".", ex);
}
}
use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.
the class InterProceduralAnalysis method populateLocalVariableMapForFunctionCall.
private static void populateLocalVariableMapForFunctionCall(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callvars, LocalVariableMap vars, FunctionCallSizeInfo fcallSizes) {
ArrayList<DataIdentifier> inputVars = fstmt.getInputParams();
ArrayList<Hop> inputOps = fop.getInput();
String fkey = fop.getFunctionKey();
for (int i = 0; i < inputVars.size(); i++) {
// create mapping between input hops and vars
DataIdentifier dat = inputVars.get(i);
Hop input = inputOps.get(i);
if (input.getDataType() == DataType.MATRIX) {
// propagate matrix characteristics
MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics(input.getDim1(), input.getDim2(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), fcallSizes.isSafeNnz(fkey, i) ? input.getNnz() : -1);
MetaDataFormat meta = new MetaDataFormat(mc, null, null);
mo.setMetaData(meta);
vars.put(dat.getName(), mo);
} else if (input.getDataType() == DataType.SCALAR) {
// (for multiple calls, literal equivalence already checked)
if (input instanceof LiteralOp) {
vars.put(dat.getName(), ScalarObjectFactory.createScalarObject(input.getValueType(), (LiteralOp) input));
} else // and input scalar is existing variable in symbol table
if (PROPAGATE_SCALAR_VARS_INTO_FUN && fcallSizes.getFunctionCallCount(fkey) == 1 && input instanceof DataOp) {
Data scalar = callvars.get(input.getName());
if (scalar != null && scalar instanceof ScalarObject) {
vars.put(dat.getName(), scalar);
}
}
}
}
}
use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.
the class Recompiler method reconcileUpdatedCallVarsLoops.
public static boolean reconcileUpdatedCallVarsLoops(LocalVariableMap oldCallVars, LocalVariableMap callVars, StatementBlock sb) {
boolean requiresRecompile = false;
// handle matrices
for (String varname : sb.variablesUpdated().getVariableNames()) {
Data dat1 = oldCallVars.get(varname);
Data dat2 = callVars.get(varname);
if (dat1 != null && dat1 instanceof MatrixObject && dat2 != null && dat2 instanceof MatrixObject) {
MatrixObject moOld = (MatrixObject) dat1;
MatrixObject mo = (MatrixObject) dat2;
MatrixCharacteristics mcOld = moOld.getMatrixCharacteristics();
MatrixCharacteristics mc = mo.getMatrixCharacteristics();
if (mcOld.getRows() != mc.getRows() || mcOld.getCols() != mc.getCols() || mcOld.getNonZeros() != mc.getNonZeros()) {
long ldim1 = mc.getRows(), ldim2 = mc.getCols(), lnnz = mc.getNonZeros();
// handle row dimension change in body
if (mcOld.getRows() != mc.getRows()) {
// unknown
ldim1 = -1;
requiresRecompile = true;
}
// handle column dimension change in body
if (mcOld.getCols() != mc.getCols()) {
// unknown
ldim2 = -1;
requiresRecompile = true;
}
// handle sparsity change
if (mcOld.getNonZeros() != mc.getNonZeros()) {
// unknown
lnnz = -1;
requiresRecompile = true;
}
MatrixObject moNew = createOutputMatrix(ldim1, ldim2, lnnz);
callVars.put(varname, moNew);
}
}
}
return requiresRecompile;
}
use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.
the class Recompiler method extractDAGOutputStatistics.
public static void extractDAGOutputStatistics(Hop hop, LocalVariableMap vars, boolean overwrite) {
if (// for all writes to symbol table
hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTWRITE) {
String varName = hop.getName();
if (// not existing so far
!vars.keySet().contains(varName) || overwrite) {
// extract matrix sizes for size propagation
if (hop.getDataType() == DataType.MATRIX) {
MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics(hop.getDim1(), hop.getDim2(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), hop.getNnz());
MetaDataFormat meta = new MetaDataFormat(mc, null, null);
mo.setMetaData(meta);
vars.put(varName, mo);
} else // extract scalar constants for second constant propagation
if (hop.getDataType() == DataType.SCALAR) {
// extract literal assignments
if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof LiteralOp) {
ScalarObject constant = HopRewriteUtils.getScalarObject((LiteralOp) hop.getInput().get(0));
if (constant != null)
vars.put(varName, constant);
} else // extract constant variable assignments
if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof DataOp) {
DataOp dop = (DataOp) hop.getInput().get(0);
String dopvarname = dop.getName();
if (dop.isRead() && vars.keySet().contains(dopvarname)) {
ScalarObject constant = (ScalarObject) vars.get(dopvarname);
// no clone because constant
vars.put(varName, constant);
}
} else // extract ncol/nrow variable assignments
if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof UnaryOp && (((UnaryOp) hop.getInput().get(0)).getOp() == OpOp1.NROW || ((UnaryOp) hop.getInput().get(0)).getOp() == OpOp1.NCOL)) {
UnaryOp uop = (UnaryOp) hop.getInput().get(0);
if (uop.getOp() == OpOp1.NROW && uop.getInput().get(0).getDim1() > 0)
vars.put(varName, new IntObject(uop.getInput().get(0).getDim1()));
else if (uop.getOp() == OpOp1.NCOL && uop.getInput().get(0).getDim2() > 0)
vars.put(varName, new IntObject(uop.getInput().get(0).getDim2()));
} else // remove other updated scalars
{
// we need to remove other updated scalars in order to ensure result
// correctness of recompilation w/o being too conservative
vars.remove(varName);
}
}
} else // already existing: take largest
{
Data dat = vars.get(varName);
if (dat instanceof MatrixObject) {
MatrixObject mo = (MatrixObject) dat;
MatrixCharacteristics mc = mo.getMatrixCharacteristics();
if (OptimizerUtils.estimateSizeExactSparsity(mc.getRows(), mc.getCols(), (mc.getNonZeros() >= 0) ? ((double) mc.getNonZeros()) / mc.getRows() / mc.getCols() : 1.0) < OptimizerUtils.estimateSize(hop.getDim1(), hop.getDim2())) {
// update statistics if necessary
mc.setDimension(hop.getDim1(), hop.getDim2());
mc.setNonZeros(hop.getNnz());
}
} else // scalar (just overwrite)
{
if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof LiteralOp) {
ScalarObject constant = HopRewriteUtils.getScalarObject((LiteralOp) hop.getInput().get(0));
if (constant != null)
vars.put(varName, constant);
}
}
}
}
}
use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.
the class Recompiler method rUpdateStatistics.
public static void rUpdateStatistics(Hop hop, LocalVariableMap vars) {
if (hop.isVisited())
return;
// recursively process children
if (hop.getInput() != null)
for (Hop c : hop.getInput()) rUpdateStatistics(c, vars);
boolean updatedSizeExpr = false;
// (with awareness not to override persistent reads to an existing name)
if (hop instanceof DataOp && ((DataOp) hop).getDataOpType() != DataOpTypes.PERSISTENTREAD) {
DataOp d = (DataOp) hop;
String varName = d.getName();
if (vars.keySet().contains(varName)) {
Data dat = vars.get(varName);
if (dat instanceof MatrixObject) {
MatrixObject mo = (MatrixObject) dat;
d.setDim1(mo.getNumRows());
d.setDim2(mo.getNumColumns());
d.setNnz(mo.getNnz());
} else if (dat instanceof FrameObject) {
FrameObject fo = (FrameObject) dat;
d.setDim1(fo.getNumRows());
d.setDim2(fo.getNumColumns());
}
}
} else // special case for persistent reads with unknown size (read-after-write)
if (hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.PERSISTENTREAD && !hop.dimsKnown() && ((DataOp) hop).getInputFormatType() != FileFormatTypes.CSV && !ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_READ_WRITE_METADATA)) {
// update hop with read meta data
DataOp dop = (DataOp) hop;
tryReadMetaDataFileMatrixCharacteristics(dop);
} else // update size expression for rand/seq according to symbol table entries
if (hop instanceof DataGenOp) {
DataGenOp d = (DataGenOp) hop;
HashMap<String, Integer> params = d.getParamIndexMap();
if (d.getOp() == DataGenMethod.RAND || d.getOp() == DataGenMethod.SINIT || d.getOp() == DataGenMethod.SAMPLE) {
boolean initUnknown = !d.dimsKnown();
int ix1 = params.get(DataExpression.RAND_ROWS);
int ix2 = params.get(DataExpression.RAND_COLS);
// update rows/cols by evaluating simple expression of literals, nrow, ncol, scalars, binaryops
HashMap<Long, Long> memo = new HashMap<>();
d.refreshRowsParameterInformation(d.getInput().get(ix1), vars, memo);
d.refreshColsParameterInformation(d.getInput().get(ix2), vars, memo);
updatedSizeExpr = initUnknown & d.dimsKnown();
} else if (d.getOp() == DataGenMethod.SEQ) {
boolean initUnknown = !d.dimsKnown();
int ix1 = params.get(Statement.SEQ_FROM);
int ix2 = params.get(Statement.SEQ_TO);
int ix3 = params.get(Statement.SEQ_INCR);
HashMap<Long, Double> memo = new HashMap<>();
double from = d.computeBoundsInformation(d.getInput().get(ix1), vars, memo);
double to = d.computeBoundsInformation(d.getInput().get(ix2), vars, memo);
double incr = d.computeBoundsInformation(d.getInput().get(ix3), vars, memo);
// special case increment
if (from != Double.MAX_VALUE && to != Double.MAX_VALUE) {
incr *= ((from > to && incr > 0) || (from < to && incr < 0)) ? -1.0 : 1.0;
}
if (from != Double.MAX_VALUE && to != Double.MAX_VALUE && incr != Double.MAX_VALUE) {
d.setDim1(UtilFunctions.getSeqLength(from, to, incr));
d.setDim2(1);
d.setIncrementValue(incr);
}
updatedSizeExpr = initUnknown & d.dimsKnown();
} else {
throw new DMLRuntimeException("Unexpected data generation method: " + d.getOp());
}
} else // update size expression for reshape according to symbol table entries
if (hop instanceof ReorgOp && ((ReorgOp) (hop)).getOp() == Hop.ReOrgOp.RESHAPE) {
ReorgOp d = (ReorgOp) hop;
boolean initUnknown = !d.dimsKnown();
HashMap<Long, Long> memo = new HashMap<>();
d.refreshRowsParameterInformation(d.getInput().get(1), vars, memo);
d.refreshColsParameterInformation(d.getInput().get(2), vars, memo);
updatedSizeExpr = initUnknown & d.dimsKnown();
} else // update size expression for indexing according to symbol table entries
if (hop instanceof IndexingOp) {
IndexingOp iop = (IndexingOp) hop;
// inpRowL
Hop input2 = iop.getInput().get(1);
// inpRowU
Hop input3 = iop.getInput().get(2);
// inpColL
Hop input4 = iop.getInput().get(3);
// inpColU
Hop input5 = iop.getInput().get(4);
boolean initUnknown = !iop.dimsKnown();
HashMap<Long, Double> memo = new HashMap<>();
double rl = iop.computeBoundsInformation(input2, vars, memo);
double ru = iop.computeBoundsInformation(input3, vars, memo);
double cl = iop.computeBoundsInformation(input4, vars, memo);
double cu = iop.computeBoundsInformation(input5, vars, memo);
if (rl != Double.MAX_VALUE && ru != Double.MAX_VALUE)
iop.setDim1((long) (ru - rl + 1));
if (cl != Double.MAX_VALUE && cu != Double.MAX_VALUE)
iop.setDim2((long) (cu - cl + 1));
updatedSizeExpr = initUnknown & iop.dimsKnown();
}
// without overwriting inferred size expressions
if (!updatedSizeExpr) {
hop.refreshSizeInformation();
}
hop.setVisited();
}
Aggregations