use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.
the class OptTreePlanChecker method checkFunctionNames.
private static void checkFunctionNames(Program prog, DMLProgram dprog, Hop root, ArrayList<Instruction> inst, Set<String> fnStack) throws DMLRuntimeException, HopsException {
//reset visit status of dag
root.resetVisitStatus();
//get all function op in this dag
HashMap<String, FunctionOp> fops = new HashMap<String, FunctionOp>();
getAllFunctionOps(root, fops);
for (Instruction linst : inst) if (linst instanceof FunctionCallCPInstruction) {
FunctionCallCPInstruction flinst = (FunctionCallCPInstruction) linst;
String fnamespace = flinst.getNamespace();
String fname = flinst.getFunctionName();
String key = DMLProgram.constructFunctionKey(fnamespace, fname);
//check 1: instruction name equal to hop name
if (!fops.containsKey(key))
throw new DMLRuntimeException("Function Check: instruction and hop names differ (" + key + ", " + fops.keySet() + ")");
//check 2: function exists
if (!prog.getFunctionProgramBlocks().containsKey(key))
throw new DMLRuntimeException("Function Check: function does not exits (" + key + ")");
//check 3: recursive program check
FunctionProgramBlock fpb = prog.getFunctionProgramBlock(fnamespace, fname);
FunctionStatementBlock fsb = dprog.getFunctionStatementBlock(fnamespace, fname);
if (!fnStack.contains(key)) {
fnStack.add(key);
checkProgramCorrectness(fpb, fsb, fnStack);
fnStack.remove(key);
}
}
}
use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.
the class DMLTranslator method constructHops.
public void constructHops(StatementBlock sb) throws ParseException, LanguageException {
if (sb instanceof WhileStatementBlock) {
constructHopsForWhileControlBlock((WhileStatementBlock) sb);
return;
}
if (sb instanceof IfStatementBlock) {
constructHopsForIfControlBlock((IfStatementBlock) sb);
return;
}
if (sb instanceof ForStatementBlock) {
//NOTE: applies to ForStatementBlock and ParForStatementBlock
constructHopsForForControlBlock((ForStatementBlock) sb);
return;
}
if (sb instanceof FunctionStatementBlock) {
constructHopsForFunctionControlBlock((FunctionStatementBlock) sb);
return;
}
HashMap<String, Hop> ids = new HashMap<String, Hop>();
ArrayList<Hop> output = new ArrayList<Hop>();
VariableSet liveIn = sb.liveIn();
VariableSet liveOut = sb.liveOut();
VariableSet updated = sb._updated;
VariableSet gen = sb._gen;
VariableSet updatedLiveOut = new VariableSet();
// handle liveout variables that are updated --> target identifiers for Assignment
HashMap<String, Integer> liveOutToTemp = new HashMap<String, Integer>();
for (int i = 0; i < sb.getNumStatements(); i++) {
Statement current = sb.getStatement(i);
if (current instanceof AssignmentStatement) {
AssignmentStatement as = (AssignmentStatement) current;
DataIdentifier target = as.getTarget();
if (target != null) {
if (liveOut.containsVariable(target.getName())) {
liveOutToTemp.put(target.getName(), Integer.valueOf(i));
}
}
}
if (current instanceof MultiAssignmentStatement) {
MultiAssignmentStatement mas = (MultiAssignmentStatement) current;
for (DataIdentifier target : mas.getTargetList()) {
if (liveOut.containsVariable(target.getName())) {
liveOutToTemp.put(target.getName(), Integer.valueOf(i));
}
}
}
}
// (i.e., from LV analysis, updated and gen sets)
if (!liveIn.getVariables().values().isEmpty()) {
for (String varName : liveIn.getVariables().keySet()) {
if (updated.containsVariable(varName) || gen.containsVariable(varName)) {
DataIdentifier var = liveIn.getVariables().get(varName);
long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim1() : var.getDim1();
long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim2() : var.getDim2();
DataOp read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
read.setAllPositions(var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn());
ids.put(varName, read);
}
}
}
for (int i = 0; i < sb.getNumStatements(); i++) {
Statement current = sb.getStatement(i);
if (current instanceof OutputStatement) {
OutputStatement os = (OutputStatement) current;
DataExpression source = os.getSource();
DataIdentifier target = os.getIdentifier();
//error handling unsupported indexing expression in write statement
if (target instanceof IndexedIdentifier) {
throw new LanguageException(source.printErrorLocation() + ": Unsupported indexing expression in write statement. " + "Please, assign the right indexing result to a variable and write this variable.");
}
DataOp ae = (DataOp) processExpression(source, target, ids);
String formatName = os.getExprParam(DataExpression.FORMAT_TYPE).toString();
ae.setInputFormatType(Expression.convertFormatType(formatName));
if (ae.getDataType() == DataType.SCALAR) {
ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), -1, -1);
} else {
switch(ae.getInputFormatType()) {
case TEXT:
case MM:
case CSV:
// write output in textcell format
ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), -1, -1);
break;
case BINARY:
// write output in binary block format
ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
break;
default:
throw new LanguageException("Unrecognized file format: " + ae.getInputFormatType());
}
}
output.add(ae);
}
if (current instanceof PrintStatement) {
DataIdentifier target = createTarget();
target.setDataType(DataType.SCALAR);
target.setValueType(ValueType.STRING);
target.setAllPositions(current.getFilename(), current.getBeginLine(), target.getBeginColumn(), current.getEndLine(), current.getEndColumn());
PrintStatement ps = (PrintStatement) current;
PRINTTYPE ptype = ps.getType();
try {
if (ptype == PRINTTYPE.PRINT) {
Hop.OpOp1 op = Hop.OpOp1.PRINT;
Expression source = ps.getExpressions().get(0);
Hop ae = processExpression(source, target, ids);
Hop printHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae);
printHop.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
output.add(printHop);
} else if (ptype == PRINTTYPE.STOP) {
Hop.OpOp1 op = Hop.OpOp1.STOP;
Expression source = ps.getExpressions().get(0);
Hop ae = processExpression(source, target, ids);
Hop stopHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae);
stopHop.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
output.add(stopHop);
} else if (ptype == PRINTTYPE.PRINTF) {
List<Expression> expressions = ps.getExpressions();
Hop[] inHops = new Hop[expressions.size()];
// Hop (ie, MultipleOp) as input Hops
for (int j = 0; j < expressions.size(); j++) {
Hop inHop = processExpression(expressions.get(j), target, ids);
inHops[j] = inHop;
}
target.setValueType(ValueType.STRING);
Hop printfHop = new MultipleOp(target.getName(), target.getDataType(), target.getValueType(), MultiInputOp.PRINTF, inHops);
output.add(printfHop);
}
} catch (HopsException e) {
throw new LanguageException(e);
}
}
if (current instanceof AssignmentStatement) {
AssignmentStatement as = (AssignmentStatement) current;
DataIdentifier target = as.getTarget();
Expression source = as.getSource();
// CASE: regular assignment statement -- source is DML expression that is NOT user-defined or external function
if (!(source instanceof FunctionCallIdentifier)) {
// CASE: target is regular data identifier
if (!(target instanceof IndexedIdentifier)) {
Hop ae = processExpression(source, target, ids);
ids.put(target.getName(), ae);
target.setProperties(source.getOutput());
Integer statementId = liveOutToTemp.get(target.getName());
if ((statementId != null) && (statementId.intValue() == i)) {
DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, DataOpTypes.TRANSIENTWRITE, null);
transientwrite.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), ae.getRowsInBlock(), ae.getColsInBlock());
transientwrite.setAllPositions(target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndLine());
updatedLiveOut.addVariable(target.getName(), target);
output.add(transientwrite);
}
} else // end if (!(target instanceof IndexedIdentifier)) {
// CASE: target is indexed identifier (left-hand side indexed expression)
{
Hop ae = processLeftIndexedExpression(source, (IndexedIdentifier) target, ids);
ids.put(target.getName(), ae);
// obtain origDim values BEFORE they are potentially updated during setProperties call
// (this is incorrect for LHS Indexing)
long origDim1 = ((IndexedIdentifier) target).getOrigDim1();
long origDim2 = ((IndexedIdentifier) target).getOrigDim2();
target.setProperties(source.getOutput());
((IndexedIdentifier) target).setOriginalDimensions(origDim1, origDim2);
// (required for scalar input to left indexing)
if (target.getDataType() != DataType.MATRIX) {
target.setDataType(DataType.MATRIX);
target.setValueType(ValueType.DOUBLE);
target.setBlockDimensions(ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
}
Integer statementId = liveOutToTemp.get(target.getName());
if ((statementId != null) && (statementId.intValue() == i)) {
DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, DataOpTypes.TRANSIENTWRITE, null);
transientwrite.setOutputParams(origDim1, origDim2, ae.getNnz(), ae.getUpdateType(), ae.getRowsInBlock(), ae.getColsInBlock());
transientwrite.setAllPositions(target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn());
updatedLiveOut.addVariable(target.getName(), target);
output.add(transientwrite);
}
}
} else {
//assignment, function call
FunctionCallIdentifier fci = (FunctionCallIdentifier) source;
FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(), fci.getName());
//error handling missing function
if (fsb == null) {
String error = source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace();
LOG.error(error);
throw new LanguageException(error);
}
//error handling unsupported function call in indexing expression
if (target instanceof IndexedIdentifier) {
String fkey = DMLProgram.constructFunctionKey(fci.getNamespace(), fci.getName());
throw new LanguageException("Unsupported function call to '" + fkey + "' in left indexing expression. " + "Please, assign the function output to a variable.");
}
ArrayList<Hop> finputs = new ArrayList<Hop>();
for (ParameterExpression paramName : fci.getParamExprs()) {
Hop in = processExpression(paramName.getExpr(), null, ids);
finputs.add(in);
}
//create function op
FunctionType ftype = fsb.getFunctionOpType();
FunctionOp fcall = null;
if (target == null) {
fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new String[] {});
} else {
fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new String[] { target.getName() });
}
output.add(fcall);
//TODO function output dataops (phase 3)
//DataOp trFoutput = new DataOp(target.getName(), target.getDataType(), target.getValueType(), fcall, DataOpTypes.FUNCTIONOUTPUT, null);
//DataOp twFoutput = new DataOp(target.getName(), target.getDataType(), target.getValueType(), trFoutput, DataOpTypes.TRANSIENTWRITE, null);
}
} else if (current instanceof MultiAssignmentStatement) {
//multi-assignment, by definition a function call
MultiAssignmentStatement mas = (MultiAssignmentStatement) current;
Expression source = mas.getSource();
if (source instanceof FunctionCallIdentifier) {
FunctionCallIdentifier fci = (FunctionCallIdentifier) source;
FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(), fci.getName());
FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
if (fstmt == null) {
LOG.error(source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace());
throw new LanguageException(source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace());
}
ArrayList<Hop> finputs = new ArrayList<Hop>();
for (ParameterExpression paramName : fci.getParamExprs()) {
Hop in = processExpression(paramName.getExpr(), null, ids);
finputs.add(in);
}
//create function op
String[] foutputs = new String[mas.getTargetList().size()];
int count = 0;
for (DataIdentifier paramName : mas.getTargetList()) {
foutputs[count++] = paramName.getName();
}
FunctionType ftype = fsb.getFunctionOpType();
FunctionOp fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, foutputs);
output.add(fcall);
//TODO function output dataops (phase 3)
/*for ( DataIdentifier paramName : mas.getTargetList() ){
DataOp twFoutput = new DataOp(paramName.getName(), paramName.getDataType(), paramName.getValueType(), fcall, DataOpTypes.TRANSIENTWRITE, null);
output.add(twFoutput);
}*/
} else if (source instanceof BuiltinFunctionExpression && ((BuiltinFunctionExpression) source).multipleReturns()) {
// construct input hops
Hop fcall = processMultipleReturnBuiltinFunctionExpression((BuiltinFunctionExpression) source, mas.getTargetList(), ids);
output.add(fcall);
} else if (source instanceof ParameterizedBuiltinFunctionExpression && ((ParameterizedBuiltinFunctionExpression) source).multipleReturns()) {
// construct input hops
Hop fcall = processMultipleReturnParameterizedBuiltinFunctionExpression((ParameterizedBuiltinFunctionExpression) source, mas.getTargetList(), ids);
output.add(fcall);
} else
throw new LanguageException("Class \"" + source.getClass() + "\" is not supported in Multiple Assignment statements");
}
}
sb.updateLiveVariablesOut(updatedLiveOut);
sb.set_hops(output);
}
use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.
the class DMLTranslator method processMultipleReturnParameterizedBuiltinFunctionExpression.
private Hop processMultipleReturnParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFunctionExpression source, ArrayList<DataIdentifier> targetList, HashMap<String, Hop> hops) throws ParseException {
FunctionType ftype = FunctionType.MULTIRETURN_BUILTIN;
String nameSpace = DMLProgram.INTERNAL_NAMESPACE;
// Create an array list to hold the outputs of this lop.
// Exact list of outputs are added based on opcode.
ArrayList<Hop> outputs = new ArrayList<Hop>();
// Construct Hop for current builtin function expression based on its type
Hop currBuiltinOp = null;
switch(source.getOpCode()) {
case TRANSFORMENCODE:
ArrayList<Hop> inputs = new ArrayList<Hop>();
inputs.add(processExpression(source.getVarParam("target"), null, hops));
inputs.add(processExpression(source.getVarParam("spec"), null, hops));
String[] outputNames = new String[targetList.size()];
outputNames[0] = ((DataIdentifier) targetList.get(0)).getName();
outputNames[1] = ((DataIdentifier) targetList.get(1)).getName();
outputs.add(new DataOp(outputNames[0], DataType.MATRIX, ValueType.DOUBLE, inputs.get(0), DataOpTypes.FUNCTIONOUTPUT, outputNames[0]));
outputs.add(new DataOp(outputNames[1], DataType.FRAME, ValueType.STRING, inputs.get(0), DataOpTypes.FUNCTIONOUTPUT, outputNames[1]));
currBuiltinOp = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), inputs, outputNames, outputs);
break;
default:
throw new ParseException("Invaid Opcode in DMLTranslator:processMultipleReturnParameterizedBuiltinFunctionExpression(): " + source.getOpCode());
}
// set properties for created hops based on outputs of source expression
for (int i = 0; i < source.getOutputs().length; i++) {
setIdentifierParams(outputs.get(i), source.getOutputs()[i]);
outputs.get(i).setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
}
currBuiltinOp.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
return currBuiltinOp;
}
use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.
the class OptimizerRuleBased method rReplaceFunctionNames.
protected void rReplaceFunctionNames(OptNode n, String oldName, String newName) throws DMLRuntimeException, HopsException {
if (n.getNodeType() == NodeType.FUNCCALL) {
FunctionOp fop = (FunctionOp) OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
String[] names = n.getParam(ParamType.OPSTRING).split(Program.KEY_DELIM);
String fnamespace = names[0];
String fname = names[1];
if (//newName if shared hop
fname.equals(oldName) || fname.equals(newName)) {
//set opttree function name
n.addParam(ParamType.OPSTRING, DMLProgram.constructFunctionKey(fnamespace, newName));
//set instruction function name
long parentID = OptTreeConverter.getAbstractPlanMapping().getMappedParentID(n.getID());
ProgramBlock pb = (ProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(parentID)[1];
ArrayList<Instruction> instArr = pb.getInstructions();
for (int i = 0; i < instArr.size(); i++) {
Instruction inst = instArr.get(i);
if (inst instanceof FunctionCallCPInstruction) {
FunctionCallCPInstruction fci = (FunctionCallCPInstruction) inst;
if (oldName.equals(fci.getFunctionName()))
instArr.set(i, FunctionCallCPInstruction.parseInstruction(fci.toString().replaceAll(oldName, newName)));
}
}
//set hop name (for recompile)
if (fop.getFunctionName().equals(oldName))
fop.setFunctionName(newName);
}
}
//recursive invocation
if (!n.isLeaf())
for (OptNode c : n.getChilds()) rReplaceFunctionNames(c, oldName, newName);
}
use of org.apache.sysml.hops.FunctionOp in project incubator-systemml by apache.
the class RewriteBlockSizeAndReblock method rule_BlockSizeAndReblock.
private void rule_BlockSizeAndReblock(Hop hop, final int blocksize) throws HopsException {
// Go to the source(s) of the DAG
for (Hop hi : hop.getInput()) {
if (!hi.isVisited())
rule_BlockSizeAndReblock(hi, blocksize);
}
boolean canReblock = isReblockValid();
if (hop instanceof DataOp) {
DataOp dop = (DataOp) hop;
// if block size does not match
if (canReblock && ((dop.getDataType() == DataType.MATRIX && (dop.getRowsInBlock() != blocksize || dop.getColsInBlock() != blocksize)) || (dop.getDataType() == DataType.FRAME && OptimizerUtils.isSparkExecutionMode() && (dop.getInputFormatType() == FileFormatTypes.TEXT || dop.getInputFormatType() == FileFormatTypes.CSV && OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK)))) {
if (dop.getDataOpType() == DataOp.DataOpTypes.PERSISTENTREAD) {
// insert reblock after the hop
dop.setRequiresReblock(true);
dop.setOutputBlocksizes(blocksize, blocksize);
} else if (dop.getDataOpType() == DataOp.DataOpTypes.PERSISTENTWRITE) {
if (dop.getRowsInBlock() == -1 && dop.getColsInBlock() == -1) {
// if this dataop is for cell output, then no reblock is needed
// as (A) all jobtypes can produce block2cell and cell2cell and
// (B) we don't generate an explicit instruction for it (the info
// is conveyed through OutputInfo.
} else if (dop.getInput().get(0).requiresReblock() && dop.getInput().get(0).getParent().size() == 1) {
// if a reblock is feeding into this, then use it if this is
// the only parent, otherwise new Reblock
dop.getInput().get(0).setOutputBlocksizes(dop.getRowsInBlock(), dop.getColsInBlock());
} else {
// insert reblock after the hop
dop.setRequiresReblock(true);
dop.setOutputBlocksizes(blocksize, blocksize);
}
} else if (dop.getDataOpType() == DataOp.DataOpTypes.TRANSIENTWRITE || dop.getDataOpType() == DataOp.DataOpTypes.TRANSIENTREAD) {
if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE) {
// simply copy the values from its input
dop.setRowsInBlock(hop.getInput().get(0).getRowsInBlock());
dop.setColsInBlock(hop.getInput().get(0).getColsInBlock());
} else {
// by default, all transient reads and writes are in blocked format
dop.setRowsInBlock(blocksize);
dop.setColsInBlock(blocksize);
}
} else {
throw new HopsException(hop.printErrorLocation() + "unexpected non-scalar Data HOP in reblock.\n");
}
}
} else //TODO remove once transform rebased to frames
if ((hop instanceof ParameterizedBuiltinOp && ((ParameterizedBuiltinOp) hop).getOp() == ParamBuiltinOp.TRANSFORM)) {
// check if there exists a non-csv-write output. If yes, add reblock
boolean rblk = false;
for (Hop out : hop.getParent()) {
if (!(out instanceof DataOp && ((DataOp) out).getDataOpType() == DataOpTypes.PERSISTENTWRITE && ((DataOp) out).getInputFormatType() == FileFormatTypes.CSV)) {
rblk = true;
break;
}
}
if (rblk) {
hop.setRequiresReblock(true);
hop.setOutputBlocksizes(blocksize, blocksize);
}
} else //NO DATAOP
{
if (hop.requiresReblock()) {
hop.setRowsInBlock(blocksize);
hop.setColsInBlock(blocksize);
} else // Constraint C2:
if (hop.getDataType() == DataType.SCALAR) {
hop.setRowsInBlock(-1);
hop.setColsInBlock(-1);
} else // Constraint C3:
{
if (!canReblock) {
hop.setRowsInBlock(-1);
hop.setColsInBlock(-1);
} else {
hop.setRowsInBlock(blocksize);
hop.setColsInBlock(blocksize);
// Reblock properties need to be set for each output.
if (hop instanceof FunctionOp) {
FunctionOp fop = (FunctionOp) hop;
if (fop.getOutputs() != null) {
for (Hop out : fop.getOutputs()) {
out.setRowsInBlock(blocksize);
out.setColsInBlock(blocksize);
}
}
}
}
// if any input is not blocked then the output of current Hop should not be blocked
for (Hop h : hop.getInput()) {
if (h.getDataType() == DataType.MATRIX && h.getRowsInBlock() == -1 && h.getColsInBlock() == -1) {
hop.setRowsInBlock(-1);
hop.setColsInBlock(-1);
break;
}
}
}
}
hop.setVisited();
}
Aggregations