use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.
the class DMLProgram method createRuntimeProgramBlock.
public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, DMLConfig config) throws IOException, LopsException, DMLRuntimeException {
Dag<Lop> dag = null;
Dag<Lop> pred_dag = null;
ArrayList<Instruction> instruct;
ArrayList<Instruction> pred_instruct = null;
ProgramBlock retPB = null;
// process While Statement - add runtime program blocks to program
if (sb instanceof WhileStatementBlock) {
// create DAG for loop predicates
pred_dag = new Dag<Lop>();
((WhileStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
// create instructions for loop predicates
pred_instruct = new ArrayList<Instruction>();
ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
for (Instruction i : pInst) {
pred_instruct.add(i);
}
// create while program block
WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
if (rtpb.getPredicateResultVar() == null) {
// e.g case : WHILE(continue)
if (((WhileStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data) {
String resultVar = ((WhileStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
rtpb.setPredicateResultVar(resultVar);
} else {
LOG.error(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
}
}
//// process the body of the while statement block ////
WhileStatementBlock wsb = (WhileStatementBlock) sb;
if (wsb.getNumStatements() > 1) {
LOG.error(wsb.printBlockErrorLocation() + "WhileStatementBlock should only have 1 statement");
throw new LopsException(wsb.printBlockErrorLocation() + "WhileStatementBlock should only have 1 statement");
}
WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
for (StatementBlock sblock : wstmt.getBody()) {
// process the body
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlock(childBlock);
}
// check there are actually Lops in to process (loop stmt body will not have any)
if (wsb.getLops() != null && !wsb.getLops().isEmpty()) {
LOG.error(wsb.printBlockErrorLocation() + "WhileStatementBlock should have no Lops");
throw new LopsException(wsb.printBlockErrorLocation() + "WhileStatementBlock should have no Lops");
}
retPB = rtpb;
//post processing for generating missing instructions
//retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
} else // process If Statement - add runtime program blocks to program
if (sb instanceof IfStatementBlock) {
// create DAG for loop predicates
pred_dag = new Dag<Lop>();
((IfStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
// create instructions for loop predicates
pred_instruct = new ArrayList<Instruction>();
ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
for (Instruction i : pInst) {
pred_instruct.add(i);
}
// create if program block
IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
if (rtpb.getPredicateResultVar() == null) {
// e.g case : If(continue)
if (((IfStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data) {
String resultVar = ((IfStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
rtpb.setPredicateResultVar(resultVar);
} else {
LOG.error(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
}
}
// process the body of the if statement block
IfStatementBlock isb = (IfStatementBlock) sb;
if (isb.getNumStatements() > 1) {
LOG.error(isb.printBlockErrorLocation() + "IfStatementBlock should have only 1 statement");
throw new LopsException(isb.printBlockErrorLocation() + "IfStatementBlock should have only 1 statement");
}
IfStatement istmt = (IfStatement) isb.getStatement(0);
// process the if body
for (StatementBlock sblock : istmt.getIfBody()) {
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlockIfBody(childBlock);
}
// process the else body
for (StatementBlock sblock : istmt.getElseBody()) {
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlockElseBody(childBlock);
}
// check there are actually Lops in to process (loop stmt body will not have any)
if (isb.getLops() != null && !isb.getLops().isEmpty()) {
LOG.error(isb.printBlockErrorLocation() + "IfStatementBlock should have no Lops");
throw new LopsException(isb.printBlockErrorLocation() + "IfStatementBlock should have no Lops");
}
retPB = rtpb;
//post processing for generating missing instructions
//retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
} else // NOTE: applies to ForStatementBlock and ParForStatementBlock
if (sb instanceof ForStatementBlock) {
ForStatementBlock fsb = (ForStatementBlock) sb;
// create DAGs for loop predicates
Dag<Lop> fromDag = new Dag<Lop>();
Dag<Lop> toDag = new Dag<Lop>();
Dag<Lop> incrementDag = new Dag<Lop>();
if (fsb.getFromHops() != null)
fsb.getFromLops().addToDag(fromDag);
if (fsb.getToHops() != null)
fsb.getToLops().addToDag(toDag);
if (fsb.getIncrementHops() != null)
fsb.getIncrementLops().addToDag(incrementDag);
// create instructions for loop predicates
ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
// create for program block
String sbName = null;
ForProgramBlock rtpb = null;
IterablePredicate iterPred = fsb.getIterPredicate();
String[] iterPredData = IterablePredicate.createIterablePredicateVariables(iterPred.getIterVar().getName(), fsb.getFromLops(), fsb.getToLops(), fsb.getIncrementLops());
if (sb instanceof ParForStatementBlock) {
sbName = "ParForStatementBlock";
rtpb = new ParForProgramBlock(prog, iterPredData, iterPred.getParForParams());
ParForProgramBlock pfrtpb = (ParForProgramBlock) rtpb;
pfrtpb.setResultVariables(((ParForStatementBlock) sb).getResultVariables());
//used for optimization and creating unscoped variables
pfrtpb.setStatementBlock((ParForStatementBlock) sb);
} else //ForStatementBlock
{
sbName = "ForStatementBlock";
rtpb = new ForProgramBlock(prog, iterPredData);
}
rtpb.setFromInstructions(fromInstructions);
rtpb.setToInstructions(toInstructions);
rtpb.setIncrementInstructions(incrementInstructions);
rtpb.setIterablePredicateVars(iterPredData);
// process the body of the for statement block
if (fsb.getNumStatements() > 1) {
LOG.error(fsb.printBlockErrorLocation() + " " + sbName + " should have 1 statement");
throw new LopsException(fsb.printBlockErrorLocation() + " " + sbName + " should have 1 statement");
}
ForStatement fs = (ForStatement) fsb.getStatement(0);
for (StatementBlock sblock : fs.getBody()) {
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlock(childBlock);
}
// check there are actually Lops in to process (loop stmt body will not have any)
if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
LOG.error(fsb.printBlockErrorLocation() + sbName + " should have no Lops");
throw new LopsException(fsb.printBlockErrorLocation() + sbName + " should have no Lops");
}
retPB = rtpb;
//post processing for generating missing instructions
//retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
} else // process function statement block - add runtime program blocks to program
if (sb instanceof FunctionStatementBlock) {
FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
if (fsb.getNumStatements() > 1) {
LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should only have 1 statement");
throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should only have 1 statement");
}
FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
FunctionProgramBlock rtpb = null;
if (fstmt instanceof ExternalFunctionStatement) {
// create external function program block
String execType = ((ExternalFunctionStatement) fstmt).getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
boolean isCP = (execType.equals(ExternalFunctionStatement.IN_MEMORY)) ? true : false;
String scratchSpaceLoc = null;
try {
scratchSpaceLoc = config.getTextValue(DMLConfig.SCRATCH_SPACE);
} catch (Exception e) {
LOG.error(fsb.printBlockErrorLocation() + "could not retrieve parameter " + DMLConfig.SCRATCH_SPACE + " from DMLConfig");
}
StringBuilder buff = new StringBuilder();
buff.append(scratchSpaceLoc);
buff.append(Lop.FILE_SEPARATOR);
buff.append(Lop.PROCESS_PREFIX);
buff.append(DMLScript.getUUID());
buff.append(Lop.FILE_SEPARATOR);
buff.append(ProgramConverter.CP_ROOT_THREAD_ID);
buff.append(Lop.FILE_SEPARATOR);
buff.append("PackageSupport");
buff.append(Lop.FILE_SEPARATOR);
String basedir = buff.toString();
if (isCP) {
rtpb = new ExternalFunctionProgramBlockCP(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
} else {
rtpb = new ExternalFunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
}
if (!fstmt.getBody().isEmpty()) {
LOG.error(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
throw new LopsException(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
}
} else {
// create function program block
rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams());
// process the function statement body
for (StatementBlock sblock : fstmt.getBody()) {
// process the body
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlock(childBlock);
}
}
// check there are actually Lops in to process (loop stmt body will not have any)
if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
}
retPB = rtpb;
// add location information
retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
} else {
// handle general case
ProgramBlock rtpb = new ProgramBlock(prog);
// DAGs for Lops
dag = new Dag<Lop>();
// check there are actually Lops in to process (loop stmt body will not have any)
if (sb.getLops() != null && !sb.getLops().isEmpty()) {
for (Lop l : sb.getLops()) {
l.addToDag(dag);
}
// Instructions for Lobs DAGs
instruct = dag.getJobs(sb, config);
rtpb.addInstructions(instruct);
}
/*// TODO: check with Doug
// add instruction for a function call
if (sb.getFunctionCallInst() != null){
rtpb.addInstruction(sb.getFunctionCallInst());
}*/
retPB = rtpb;
//post processing for generating missing instructions
//retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
}
return retPB;
}
use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.
the class LiteralOp method constructLops.
@Override
public Lop constructLops() {
// return already created lops
if (getLops() != null)
return getLops();
try {
Lop l = null;
switch(getValueType()) {
case DOUBLE:
l = Data.createLiteralLop(ValueType.DOUBLE, Double.toString(value_double));
break;
case BOOLEAN:
l = Data.createLiteralLop(ValueType.BOOLEAN, Boolean.toString(value_boolean));
break;
case STRING:
l = Data.createLiteralLop(ValueType.STRING, value_string);
break;
case INT:
l = Data.createLiteralLop(ValueType.INT, Long.toString(value_long));
break;
default:
throw new HopsException(this.printErrorLocation() + "unexpected value type constructing lops for LiteralOp.\n");
}
l.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
setLineNumbers(l);
setLops(l);
} catch (LopsException e) {
throw new HopsException(e);
}
return getLops();
}
use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.
the class NaryOp method constructLops.
/**
* Construct the corresponding Lops for this Hop
*/
@Override
public Lop constructLops() {
// reuse existing lop
if (getLops() != null)
return getLops();
try {
Lop[] inLops = new Lop[getInput().size()];
for (int i = 0; i < getInput().size(); i++) inLops[i] = getInput().get(i).constructLops();
Nary.OperationType opType = HopsOpOpNLops.get(_op);
if (opType == null)
throw new HopsException("Unknown Nary Lop type for '" + _op + "'");
ExecType et = optFindExecType();
Nary multipleCPLop = new Nary(opType, getDataType(), getValueType(), inLops, et);
setOutputDimensions(multipleCPLop);
setLineNumbers(multipleCPLop);
setLops(multipleCPLop);
} catch (Exception e) {
throw new HopsException(this.printErrorLocation() + "error constructing Lops for NaryOp -- \n ", e);
}
// add reblock/checkpoint lops if necessary
constructAndSetLopsDataFlowProperties();
return getLops();
}
use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.
the class ParameterizedBuiltinOp method constructLopsRemoveEmpty.
private void constructLopsRemoveEmpty(HashMap<String, Lop> inputlops, ExecType et) {
Hop targetHop = getTargetHop();
Hop marginHop = getParameterHop("margin");
Hop selectHop = getParameterHop("select");
Hop emptyRet = getParameterHop("empty.return");
if (et == ExecType.CP || et == ExecType.CP_FILE) {
ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops, HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
setOutputDimensions(pbilop);
setLineNumbers(pbilop);
setLops(pbilop);
/*DISABLED CP PMM (see for example, MDA Bivar test, requires size propagation on recompile)
if( et == ExecType.CP && isTargetDiagInput() && marginHop instanceof LiteralOp
&& ((LiteralOp)marginHop).getStringValue().equals("rows")
&& _outputPermutationMatrix ) //SPECIAL CASE SELECTION VECTOR
{
//TODO this special case could be taken into account for memory estimates in order
// to reduce the estimates for the input diag and subsequent matrix multiply
//get input vector (without materializing diag())
Hop input = targetHop.getInput().get(0);
long brlen = input.getRowsInBlock();
long bclen = input.getColsInBlock();
MemoTable memo = new MemoTable();
boolean isPPredInput = (input instanceof BinaryOp && ((BinaryOp)input).isPPredOperation());
//step1: compute index vectors
Hop ppred0 = input;
if( !isPPredInput ) { //ppred only if required
ppred0 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, OpOp2.NOTEQUAL, input, new LiteralOp("0",0));
HopRewriteUtils.setOutputBlocksizes(ppred0, brlen, bclen);
ppred0.refreshSizeInformation();
ppred0.computeMemEstimate(memo); //select exec type
HopRewriteUtils.copyLineNumbers(this, ppred0);
}
UnaryOp cumsum = new UnaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, OpOp1.CUMSUM, ppred0);
HopRewriteUtils.setOutputBlocksizes(cumsum, brlen, bclen);
cumsum.refreshSizeInformation();
cumsum.computeMemEstimate(memo); //select exec type
HopRewriteUtils.copyLineNumbers(this, cumsum);
BinaryOp sel = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, ppred0, cumsum);
HopRewriteUtils.setOutputBlocksizes(sel, brlen, bclen);
sel.refreshSizeInformation();
sel.computeMemEstimate(memo); //select exec type
HopRewriteUtils.copyLineNumbers(this, sel);
Lop loutput = sel.constructLops();
//Step 4: cleanup hops (allow for garbage collection)
HopRewriteUtils.removeChildReference(ppred0, input);
setLops( loutput );
}
else //GENERAL CASE
{
ParameterizedBuiltin pbilop = new ParameterizedBuiltin( et, inputlops,
HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType());
pbilop.getOutputParameters().setDimensions(getDim1(),getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
setLineNumbers(pbilop);
setLops(pbilop);
}
*/
} else if (et == ExecType.MR) {
// special compile for mr removeEmpty-diag
if (isTargetDiagInput() && HopRewriteUtils.isLiteralOfValue(marginHop, "rows")) {
// get input vector (without materializing diag())
Hop input = targetHop.getInput().get(0);
int brlen = input.getRowsInBlock();
int bclen = input.getColsInBlock();
MemoTable memo = new MemoTable();
boolean isPPredInput = (input instanceof BinaryOp && ((BinaryOp) input).isPPredOperation());
// step1: compute index vectors
Hop ppred0 = input;
if (!isPPredInput) {
// ppred only if required
ppred0 = HopRewriteUtils.createBinary(input, new LiteralOp(0), OpOp2.NOTEQUAL);
HopRewriteUtils.updateHopCharacteristics(ppred0, brlen, bclen, memo, this);
}
UnaryOp cumsum = HopRewriteUtils.createUnary(ppred0, OpOp1.CUMSUM);
HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, memo, this);
Lop loutput = null;
double mest = AggBinaryOp.getMapmmMemEstimate(input.getDim1(), 1, brlen, bclen, -1, brlen, bclen, brlen, bclen, -1, 1, true);
double mbudget = OptimizerUtils.getRemoteMemBudgetMap(true);
if (// SPECIAL CASE: SELECTION VECTOR
_outputPermutationMatrix && mest < mbudget) {
BinaryOp sel = HopRewriteUtils.createBinary(ppred0, cumsum, OpOp2.MULT);
HopRewriteUtils.updateHopCharacteristics(sel, brlen, bclen, memo, this);
loutput = sel.constructLops();
} else // GENERAL CASE: GENERAL PERMUTATION MATRIX
{
// max ensures non-zero entries and at least one output row
BinaryOp max = HopRewriteUtils.createBinary(cumsum, new LiteralOp(1), OpOp2.MAX);
HopRewriteUtils.updateHopCharacteristics(max, brlen, bclen, memo, this);
DataGenOp seq = HopRewriteUtils.createSeqDataGenOp(input);
seq.setName("tmp4");
HopRewriteUtils.updateHopCharacteristics(seq, brlen, bclen, memo, this);
// step 2: compute removeEmpty(rows) output via table, seq guarantees right column dimension
// note: weights always the input (even if isPPredInput) because input also includes 0s
TernaryOp table = new TernaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp3.CTABLE, max, seq, input);
table.setOutputBlocksizes(brlen, bclen);
table.refreshSizeInformation();
// force MR
table.setForcedExecType(ExecType.MR);
HopRewriteUtils.copyLineNumbers(this, table);
table.setDisjointInputs(true);
table.setOutputEmptyBlocks(_outputEmptyBlocks);
loutput = table.constructLops();
HopRewriteUtils.removeChildReference(table, input);
}
// Step 4: cleanup hops (allow for garbage collection)
HopRewriteUtils.removeChildReference(ppred0, input);
setLops(loutput);
} else // default mr remove empty
if (et == ExecType.MR) {
if (!(marginHop instanceof LiteralOp))
throw new HopsException("Parameter 'margin' must be a literal argument.");
Hop input = targetHop;
long rlen = input.getDim1();
long clen = input.getDim2();
int brlen = input.getRowsInBlock();
int bclen = input.getColsInBlock();
long nnz = input.getNnz();
boolean rmRows = ((LiteralOp) marginHop).getStringValue().equals("rows");
// construct lops via new partial hop dag and subsequent lops construction
// in order to reuse of operator selection decisions
BinaryOp ppred0 = null;
Hop emptyInd = null;
if (selectHop == null) {
// Step1: compute row/col non-empty indicators
ppred0 = HopRewriteUtils.createBinary(input, new LiteralOp(0), OpOp2.NOTEQUAL);
// always MR
ppred0.setForcedExecType(ExecType.MR);
emptyInd = ppred0;
if (!((rmRows && clen == 1) || (!rmRows && rlen == 1))) {
emptyInd = HopRewriteUtils.createAggUnaryOp(ppred0, AggOp.MAX, rmRows ? Direction.Row : Direction.Col);
// always MR
emptyInd.setForcedExecType(ExecType.MR);
HopRewriteUtils.copyLineNumbers(this, emptyInd);
}
} else {
emptyInd = selectHop;
}
// Step 2: compute row offsets for non-empty rows
Hop cumsumInput = emptyInd;
if (!rmRows) {
cumsumInput = HopRewriteUtils.createTranspose(emptyInd);
HopRewriteUtils.updateHopCharacteristics(cumsumInput, brlen, bclen, this);
}
UnaryOp cumsum = HopRewriteUtils.createUnary(cumsumInput, OpOp1.CUMSUM);
HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, this);
Hop cumsumOutput = cumsum;
if (!rmRows) {
cumsumOutput = HopRewriteUtils.createTranspose(cumsum);
HopRewriteUtils.updateHopCharacteristics(cumsumOutput, brlen, bclen, this);
}
// alternative: right indexing
Hop maxDim = HopRewriteUtils.createAggUnaryOp(cumsumOutput, AggOp.MAX, Direction.RowCol);
HopRewriteUtils.updateHopCharacteristics(maxDim, brlen, bclen, this);
BinaryOp offsets = HopRewriteUtils.createBinary(cumsumOutput, emptyInd, OpOp2.MULT);
HopRewriteUtils.updateHopCharacteristics(offsets, brlen, bclen, this);
// Step 3: gather non-empty rows/cols into final results
Lop linput = input.constructLops();
Lop loffset = offsets.constructLops();
Lop lmaxdim = maxDim.constructLops();
double mestPM = OptimizerUtils.estimatePartitionedSizeExactSparsity(rlen, 1, brlen, bclen, 1.0);
Lop rmEmpty = null;
// a) broadcast-based PMM (permutation matrix mult)
if (rmRows && rlen >= 0 && mestPM < OptimizerUtils.getRemoteMemBudgetMap() && HopRewriteUtils.isLiteralOfValue(emptyRet, false)) {
boolean needPart = !offsets.dimsKnown() || offsets.getDim1() > DistributedCacheInput.PARTITION_SIZE;
if (needPart) {
// requires partitioning
loffset = new DataPartition(loffset, DataType.MATRIX, ValueType.DOUBLE, (mestPM > OptimizerUtils.getLocalMemBudget()) ? ExecType.MR : ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
loffset.getOutputParameters().setDimensions(rlen, 1, brlen, bclen, rlen);
setLineNumbers(loffset);
}
rmEmpty = new PMMJ(loffset, linput, lmaxdim, getDataType(), getValueType(), needPart, true, ExecType.MR);
setOutputDimensions(rmEmpty);
setLineNumbers(rmEmpty);
} else // b) general case: repartition-based rmempty
{
boolean requiresRep = ((clen > bclen || clen <= 0) && rmRows) || ((rlen > brlen || rlen <= 0) && !rmRows);
if (requiresRep) {
// ncol of left input (determines num replicates)
Lop pos = createOffsetLop(input, rmRows);
loffset = new RepMat(loffset, pos, rmRows, DataType.MATRIX, ValueType.DOUBLE);
loffset.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, nnz);
setLineNumbers(loffset);
}
Group group1 = new Group(linput, Group.OperationTypes.Sort, getDataType(), getValueType());
setLineNumbers(group1);
group1.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, nnz);
Group group2 = new Group(loffset, Group.OperationTypes.Sort, getDataType(), getValueType());
setLineNumbers(group2);
group2.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, nnz);
HashMap<String, Lop> inMap = new HashMap<>();
inMap.put("target", group1);
inMap.put("offset", group2);
inMap.put("maxdim", lmaxdim);
inMap.put("margin", inputlops.get("margin"));
inMap.put("empty.return", inputlops.get("empty.return"));
rmEmpty = new ParameterizedBuiltin(inMap, HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
setOutputDimensions(rmEmpty);
setLineNumbers(rmEmpty);
}
Group group3 = new Group(rmEmpty, Group.OperationTypes.Sort, getDataType(), getValueType());
setLineNumbers(group3);
group3.getOutputParameters().setDimensions(-1, -1, brlen, bclen, -1);
Aggregate finalagg = new Aggregate(group3, Aggregate.OperationTypes.Sum, DataType.MATRIX, getValueType(), ExecType.MR);
setOutputDimensions(finalagg);
setLineNumbers(finalagg);
// Step 4: cleanup hops (allow for garbage collection)
if (selectHop == null)
HopRewriteUtils.removeChildReference(ppred0, input);
setLops(finalagg);
}
} else if (et == ExecType.SPARK) {
if (!(marginHop instanceof LiteralOp))
throw new HopsException("Parameter 'margin' must be a literal argument.");
Hop input = targetHop;
long rlen = input.getDim1();
long clen = input.getDim2();
int brlen = input.getRowsInBlock();
int bclen = input.getColsInBlock();
boolean rmRows = ((LiteralOp) marginHop).getStringValue().equals("rows");
// construct lops via new partial hop dag and subsequent lops construction
// in order to reuse of operator selection decisions
BinaryOp ppred0 = null;
Hop emptyInd = null;
if (selectHop == null) {
// Step1: compute row/col non-empty indicators
ppred0 = HopRewriteUtils.createBinary(input, new LiteralOp(0), OpOp2.NOTEQUAL);
// always Spark
ppred0.setForcedExecType(ExecType.SPARK);
emptyInd = ppred0;
if (!((rmRows && clen == 1) || (!rmRows && rlen == 1))) {
emptyInd = HopRewriteUtils.createAggUnaryOp(ppred0, AggOp.MAX, rmRows ? Direction.Row : Direction.Col);
// always Spark
emptyInd.setForcedExecType(ExecType.SPARK);
}
} else {
emptyInd = selectHop;
}
// Step 2: compute row offsets for non-empty rows
Hop cumsumInput = emptyInd;
if (!rmRows) {
cumsumInput = HopRewriteUtils.createTranspose(emptyInd);
HopRewriteUtils.updateHopCharacteristics(cumsumInput, brlen, bclen, this);
}
UnaryOp cumsum = HopRewriteUtils.createUnary(cumsumInput, OpOp1.CUMSUM);
HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, this);
Hop cumsumOutput = cumsum;
if (!rmRows) {
cumsumOutput = HopRewriteUtils.createTranspose(cumsum);
HopRewriteUtils.updateHopCharacteristics(cumsumOutput, brlen, bclen, this);
}
// alternative: right indexing
Hop maxDim = HopRewriteUtils.createAggUnaryOp(cumsumOutput, AggOp.MAX, Direction.RowCol);
HopRewriteUtils.updateHopCharacteristics(maxDim, brlen, bclen, this);
BinaryOp offsets = HopRewriteUtils.createBinary(cumsumOutput, emptyInd, OpOp2.MULT);
HopRewriteUtils.updateHopCharacteristics(offsets, brlen, bclen, this);
// Step 3: gather non-empty rows/cols into final results
Lop linput = input.constructLops();
Lop loffset = offsets.constructLops();
Lop lmaxdim = maxDim.constructLops();
HashMap<String, Lop> inMap = new HashMap<>();
inMap.put("target", linput);
inMap.put("offset", loffset);
inMap.put("maxdim", lmaxdim);
inMap.put("margin", inputlops.get("margin"));
inMap.put("empty.return", inputlops.get("empty.return"));
if (!FORCE_DIST_RM_EMPTY && isRemoveEmptyBcSP())
_bRmEmptyBC = true;
ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inMap, HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et, _bRmEmptyBC);
setOutputDimensions(pbilop);
setLineNumbers(pbilop);
// Step 4: cleanup hops (allow for garbage collection)
if (selectHop == null)
HopRewriteUtils.removeChildReference(ppred0, input);
setLops(pbilop);
// NOTE: in contrast to mr, replication and aggregation handled instruction-local
}
}
use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.
the class ParameterizedBuiltinOp method constructLops.
@Override
public Lop constructLops() {
// return already created lops
if (getLops() != null)
return getLops();
// construct lops for all input parameters
HashMap<String, Lop> inputlops = new HashMap<>();
for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) inputlops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops());
switch(_op) {
case GROUPEDAGG:
{
ExecType et = optFindExecType();
constructLopsGroupedAggregate(inputlops, et);
break;
}
case RMEMPTY:
{
ExecType et = optFindExecType();
et = (et == ExecType.MR && !COMPILE_PARALLEL_REMOVEEMPTY) ? ExecType.CP_FILE : et;
constructLopsRemoveEmpty(inputlops, et);
break;
}
case REXPAND:
{
ExecType et = optFindExecType();
constructLopsRExpand(inputlops, et);
break;
}
case CDF:
case INVCDF:
case REPLACE:
case TRANSFORMAPPLY:
case TRANSFORMDECODE:
case TRANSFORMCOLMAP:
case TRANSFORMMETA:
case TOSTRING:
{
ExecType et = optFindExecType();
ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops, HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
setOutputDimensions(pbilop);
setLineNumbers(pbilop);
setLops(pbilop);
break;
}
default:
throw new HopsException("Unknown ParamBuiltinOp: " + _op);
}
// add reblock/checkpoint lops if necessary
constructAndSetLopsDataFlowProperties();
return getLops();
}
Aggregations