use of org.apache.sysml.parser.WhileStatement in project systemml by apache.
the class InterProceduralAnalysis method propagateStatisticsAcrossBlock.
// ///////////////////////////
// INTRA-PROCEDURE ANALYSIS
// ////
private void propagateStatisticsAcrossBlock(StatementBlock sb, LocalVariableMap callVars, FunctionCallSizeInfo fcallSizes, Set<String> fnStack) {
if (sb instanceof FunctionStatementBlock) {
FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
for (StatementBlock sbi : fstmt.getBody()) propagateStatisticsAcrossBlock(sbi, callVars, fcallSizes, fnStack);
} else if (sb instanceof WhileStatementBlock) {
WhileStatementBlock wsb = (WhileStatementBlock) sb;
WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
// old stats into predicate
propagateStatisticsAcrossPredicateDAG(wsb.getPredicateHops(), callVars);
// remove updated constant scalars
Recompiler.removeUpdatedScalars(callVars, wsb);
// check and propagate stats into body
LocalVariableMap oldCallVars = (LocalVariableMap) callVars.clone();
for (StatementBlock sbi : wstmt.getBody()) propagateStatisticsAcrossBlock(sbi, callVars, fcallSizes, fnStack);
if (Recompiler.reconcileUpdatedCallVarsLoops(oldCallVars, callVars, wsb)) {
// second pass if required
propagateStatisticsAcrossPredicateDAG(wsb.getPredicateHops(), callVars);
for (StatementBlock sbi : wstmt.getBody()) propagateStatisticsAcrossBlock(sbi, callVars, fcallSizes, fnStack);
}
// remove updated constant scalars
Recompiler.removeUpdatedScalars(callVars, sb);
} else if (sb instanceof IfStatementBlock) {
IfStatementBlock isb = (IfStatementBlock) sb;
IfStatement istmt = (IfStatement) isb.getStatement(0);
// old stats into predicate
propagateStatisticsAcrossPredicateDAG(isb.getPredicateHops(), callVars);
// check and propagate stats into body
LocalVariableMap oldCallVars = (LocalVariableMap) callVars.clone();
LocalVariableMap callVarsElse = (LocalVariableMap) callVars.clone();
for (StatementBlock sbi : istmt.getIfBody()) propagateStatisticsAcrossBlock(sbi, callVars, fcallSizes, fnStack);
for (StatementBlock sbi : istmt.getElseBody()) propagateStatisticsAcrossBlock(sbi, callVarsElse, fcallSizes, fnStack);
callVars = Recompiler.reconcileUpdatedCallVarsIf(oldCallVars, callVars, callVarsElse, isb);
// remove updated constant scalars
Recompiler.removeUpdatedScalars(callVars, sb);
} else if (// incl parfor
sb instanceof ForStatementBlock) {
ForStatementBlock fsb = (ForStatementBlock) sb;
ForStatement fstmt = (ForStatement) fsb.getStatement(0);
// old stats into predicate
propagateStatisticsAcrossPredicateDAG(fsb.getFromHops(), callVars);
propagateStatisticsAcrossPredicateDAG(fsb.getToHops(), callVars);
propagateStatisticsAcrossPredicateDAG(fsb.getIncrementHops(), callVars);
// remove updated constant scalars
Recompiler.removeUpdatedScalars(callVars, fsb);
// check and propagate stats into body
LocalVariableMap oldCallVars = (LocalVariableMap) callVars.clone();
for (StatementBlock sbi : fstmt.getBody()) propagateStatisticsAcrossBlock(sbi, callVars, fcallSizes, fnStack);
if (Recompiler.reconcileUpdatedCallVarsLoops(oldCallVars, callVars, fsb))
for (StatementBlock sbi : fstmt.getBody()) propagateStatisticsAcrossBlock(sbi, callVars, fcallSizes, fnStack);
// remove updated constant scalars
Recompiler.removeUpdatedScalars(callVars, sb);
} else // generic (last-level)
{
// remove updated constant scalars
Recompiler.removeUpdatedScalars(callVars, sb);
// old stats in, new stats out if updated
ArrayList<Hop> roots = sb.getHops();
DMLProgram prog = sb.getDMLProg();
// replace scalar reads with literals
Hop.resetVisitStatus(roots);
propagateScalarsAcrossDAG(roots, callVars);
// refresh stats across dag
Hop.resetVisitStatus(roots);
propagateStatisticsAcrossDAG(roots, callVars);
// propagate stats into function calls
Hop.resetVisitStatus(roots);
propagateStatisticsIntoFunctions(prog, roots, callVars, fcallSizes, fnStack);
}
}
use of org.apache.sysml.parser.WhileStatement in project systemml by apache.
the class ProgramRecompiler method isApplicableForReuseVariable.
private static boolean isApplicableForReuseVariable(StatementBlock sb, StatementBlock parforSB, String var) {
boolean ret = false;
if (sb instanceof IfStatementBlock) {
IfStatement is = (IfStatement) sb.getStatement(0);
for (StatementBlock lsb : is.getIfBody()) ret |= isApplicableForReuseVariable(lsb, parforSB, var);
for (StatementBlock lsb : is.getElseBody()) ret |= isApplicableForReuseVariable(lsb, parforSB, var);
} else if (sb instanceof WhileStatementBlock) {
WhileStatement ws = (WhileStatement) sb.getStatement(0);
for (StatementBlock lsb : ws.getBody()) ret |= isApplicableForReuseVariable(lsb, parforSB, var);
} else if (sb instanceof ForStatementBlock) {
// for or parfor
ForStatementBlock fsb = (ForStatementBlock) sb;
ForStatement fs = (ForStatement) fsb.getStatement(0);
if (fsb == parforSB) {
// found parfor statement
ret = true;
} else {
for (StatementBlock lsb : fs.getBody()) ret |= isApplicableForReuseVariable(lsb, parforSB, var);
}
}
return ret && !sb.variablesUpdated().containsVariable(var);
}
use of org.apache.sysml.parser.WhileStatement in project systemml by apache.
the class ProgramRecompiler method replaceConstantScalarVariables.
public static void replaceConstantScalarVariables(StatementBlock sb, LocalVariableMap vars) {
if (sb instanceof IfStatementBlock) {
IfStatementBlock isb = (IfStatementBlock) sb;
IfStatement is = (IfStatement) sb.getStatement(0);
replacePredicateLiterals(isb.getPredicateHops(), vars);
for (StatementBlock lsb : is.getIfBody()) replaceConstantScalarVariables(lsb, vars);
for (StatementBlock lsb : is.getElseBody()) replaceConstantScalarVariables(lsb, vars);
} else if (sb instanceof WhileStatementBlock) {
WhileStatementBlock wsb = (WhileStatementBlock) sb;
WhileStatement ws = (WhileStatement) sb.getStatement(0);
replacePredicateLiterals(wsb.getPredicateHops(), vars);
for (StatementBlock lsb : ws.getBody()) replaceConstantScalarVariables(lsb, vars);
} else if (// for or parfor
sb instanceof ForStatementBlock) {
ForStatementBlock fsb = (ForStatementBlock) sb;
ForStatement fs = (ForStatement) fsb.getStatement(0);
replacePredicateLiterals(fsb.getFromHops(), vars);
replacePredicateLiterals(fsb.getToHops(), vars);
replacePredicateLiterals(fsb.getIncrementHops(), vars);
for (StatementBlock lsb : fs.getBody()) replaceConstantScalarVariables(lsb, vars);
} else // last level block
{
ArrayList<Hop> hops = sb.getHops();
if (hops != null) {
// replace constant literals
Hop.resetVisitStatus(hops);
for (Hop hopRoot : hops) Recompiler.rReplaceLiterals(hopRoot, vars, true);
}
}
}
use of org.apache.sysml.parser.WhileStatement in project systemml by apache.
the class ProgramRecompiler method rFindAndRecompileIndexingHOP.
/**
* NOTE: if force is set, we set and recompile the respective indexing hops;
* otherwise, we release the forced exec type and recompile again. Hence,
* any changes can be exactly reverted with the same access behavior.
*
* @param sb statement block
* @param pb program block
* @param var variable
* @param ec execution context
* @param force if true, set and recompile the respective indexing hops
*/
public static void rFindAndRecompileIndexingHOP(StatementBlock sb, ProgramBlock pb, String var, ExecutionContext ec, boolean force) {
if (pb instanceof IfProgramBlock && sb instanceof IfStatementBlock) {
IfProgramBlock ipb = (IfProgramBlock) pb;
IfStatementBlock isb = (IfStatementBlock) sb;
IfStatement is = (IfStatement) sb.getStatement(0);
// process if condition
if (isb.getPredicateHops() != null)
ipb.setPredicate(rFindAndRecompileIndexingHOP(isb.getPredicateHops(), ipb.getPredicate(), var, ec, force));
// process if branch
int len = is.getIfBody().size();
for (int i = 0; i < ipb.getChildBlocksIfBody().size() && i < len; i++) {
ProgramBlock lpb = ipb.getChildBlocksIfBody().get(i);
StatementBlock lsb = is.getIfBody().get(i);
rFindAndRecompileIndexingHOP(lsb, lpb, var, ec, force);
}
// process else branch
if (ipb.getChildBlocksElseBody() != null) {
int len2 = is.getElseBody().size();
for (int i = 0; i < ipb.getChildBlocksElseBody().size() && i < len2; i++) {
ProgramBlock lpb = ipb.getChildBlocksElseBody().get(i);
StatementBlock lsb = is.getElseBody().get(i);
rFindAndRecompileIndexingHOP(lsb, lpb, var, ec, force);
}
}
} else if (pb instanceof WhileProgramBlock && sb instanceof WhileStatementBlock) {
WhileProgramBlock wpb = (WhileProgramBlock) pb;
WhileStatementBlock wsb = (WhileStatementBlock) sb;
WhileStatement ws = (WhileStatement) sb.getStatement(0);
// process while condition
if (wsb.getPredicateHops() != null)
wpb.setPredicate(rFindAndRecompileIndexingHOP(wsb.getPredicateHops(), wpb.getPredicate(), var, ec, force));
// process body
// robustness for potentially added problem blocks
int len = ws.getBody().size();
for (int i = 0; i < wpb.getChildBlocks().size() && i < len; i++) {
ProgramBlock lpb = wpb.getChildBlocks().get(i);
StatementBlock lsb = ws.getBody().get(i);
rFindAndRecompileIndexingHOP(lsb, lpb, var, ec, force);
}
} else if (// for or parfor
pb instanceof ForProgramBlock && sb instanceof ForStatementBlock) {
ForProgramBlock fpb = (ForProgramBlock) pb;
ForStatementBlock fsb = (ForStatementBlock) sb;
ForStatement fs = (ForStatement) fsb.getStatement(0);
if (fsb.getFromHops() != null)
fpb.setFromInstructions(rFindAndRecompileIndexingHOP(fsb.getFromHops(), fpb.getFromInstructions(), var, ec, force));
if (fsb.getToHops() != null)
fpb.setToInstructions(rFindAndRecompileIndexingHOP(fsb.getToHops(), fpb.getToInstructions(), var, ec, force));
if (fsb.getIncrementHops() != null)
fpb.setIncrementInstructions(rFindAndRecompileIndexingHOP(fsb.getIncrementHops(), fpb.getIncrementInstructions(), var, ec, force));
// process body
// robustness for potentially added problem blocks
int len = fs.getBody().size();
for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
ProgramBlock lpb = fpb.getChildBlocks().get(i);
StatementBlock lsb = fs.getBody().get(i);
rFindAndRecompileIndexingHOP(lsb, lpb, var, ec, force);
}
} else // last level program block
{
try {
// process actual hops
boolean ret = false;
Hop.resetVisitStatus(sb.getHops());
if (force) {
// set forced execution type
for (Hop h : sb.getHops()) ret |= rFindAndSetCPIndexingHOP(h, var);
} else {
// release forced execution type
for (Hop h : sb.getHops()) ret |= rFindAndReleaseIndexingHOP(h, var);
}
// recompilation on-demand
if (ret) {
// construct new instructions
ArrayList<Instruction> newInst = Recompiler.recompileHopsDag(sb, sb.getHops(), ec.getVariables(), null, true, false, 0);
pb.setInstructions(newInst);
}
} catch (Exception ex) {
throw new DMLRuntimeException(ex);
}
}
}
use of org.apache.sysml.parser.WhileStatement in project systemml by apache.
the class OptTreeConverter method rCreateAbstractOptNode.
public static OptNode rCreateAbstractOptNode(StatementBlock sb, ProgramBlock pb, LocalVariableMap vars, boolean topLevel, Set<String> memo) {
OptNode node = null;
if (pb instanceof IfProgramBlock && sb instanceof IfStatementBlock) {
IfProgramBlock ipb = (IfProgramBlock) pb;
IfStatementBlock isb = (IfStatementBlock) sb;
IfStatement is = (IfStatement) isb.getStatement(0);
node = new OptNode(NodeType.IF);
_hlMap.putProgMapping(sb, pb, node);
node.setExecType(ExecType.CP);
node.setLineNumbers(isb.getBeginLine(), isb.getEndLine());
// handle predicate
isb.getPredicateHops().resetVisitStatus();
node.addChilds(rCreateAbstractOptNodes(isb.getPredicateHops(), vars, memo));
// process if branch
OptNode ifn = new OptNode(NodeType.GENERIC);
_hlMap.putProgMapping(sb, pb, ifn);
ifn.setExecType(ExecType.CP);
node.addChild(ifn);
int len = is.getIfBody().size();
for (int i = 0; i < ipb.getChildBlocksIfBody().size() && i < len; i++) {
ProgramBlock lpb = ipb.getChildBlocksIfBody().get(i);
StatementBlock lsb = is.getIfBody().get(i);
ifn.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
}
// process else branch
if (ipb.getChildBlocksElseBody() != null) {
OptNode efn = new OptNode(NodeType.GENERIC);
_hlMap.putProgMapping(sb, pb, efn);
efn.setExecType(ExecType.CP);
node.addChild(efn);
int len2 = is.getElseBody().size();
for (int i = 0; i < ipb.getChildBlocksElseBody().size() && i < len2; i++) {
ProgramBlock lpb = ipb.getChildBlocksElseBody().get(i);
StatementBlock lsb = is.getElseBody().get(i);
efn.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
}
}
} else if (pb instanceof WhileProgramBlock && sb instanceof WhileStatementBlock) {
WhileProgramBlock wpb = (WhileProgramBlock) pb;
WhileStatementBlock wsb = (WhileStatementBlock) sb;
WhileStatement ws = (WhileStatement) wsb.getStatement(0);
node = new OptNode(NodeType.WHILE);
_hlMap.putProgMapping(sb, pb, node);
node.setExecType(ExecType.CP);
node.setLineNumbers(wsb.getBeginLine(), wsb.getEndLine());
// handle predicate
wsb.getPredicateHops().resetVisitStatus();
node.addChilds(rCreateAbstractOptNodes(wsb.getPredicateHops(), vars, memo));
// process body
int len = ws.getBody().size();
for (int i = 0; i < wpb.getChildBlocks().size() && i < len; i++) {
ProgramBlock lpb = wpb.getChildBlocks().get(i);
StatementBlock lsb = ws.getBody().get(i);
node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
}
} else if (pb instanceof ForProgramBlock && sb instanceof ForStatementBlock && !(pb instanceof ParForProgramBlock)) {
ForProgramBlock fpb = (ForProgramBlock) pb;
ForStatementBlock fsb = (ForStatementBlock) sb;
ForStatement fs = (ForStatement) fsb.getStatement(0);
node = new OptNode(NodeType.FOR);
_hlMap.putProgMapping(sb, pb, node);
node.setExecType(ExecType.CP);
node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
// determine number of iterations
long N = OptimizerUtils.getNumIterations(fpb, vars, CostEstimator.FACTOR_NUM_ITERATIONS);
node.addParam(ParamType.NUM_ITERATIONS, String.valueOf(N));
// handle predicate
fsb.getFromHops().resetVisitStatus();
fsb.getToHops().resetVisitStatus();
if (fsb.getIncrementHops() != null)
fsb.getIncrementHops().resetVisitStatus();
node.addChilds(rCreateAbstractOptNodes(fsb.getFromHops(), vars, memo));
node.addChilds(rCreateAbstractOptNodes(fsb.getToHops(), vars, memo));
if (fsb.getIncrementHops() != null)
node.addChilds(rCreateAbstractOptNodes(fsb.getIncrementHops(), vars, memo));
// process body
int len = fs.getBody().size();
for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
ProgramBlock lpb = fpb.getChildBlocks().get(i);
StatementBlock lsb = fs.getBody().get(i);
node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
}
} else if (pb instanceof ParForProgramBlock && sb instanceof ParForStatementBlock) {
ParForProgramBlock fpb = (ParForProgramBlock) pb;
ParForStatementBlock fsb = (ParForStatementBlock) sb;
ParForStatement fs = (ParForStatement) fsb.getStatement(0);
node = new OptNode(NodeType.PARFOR);
node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
_hlMap.putProgMapping(sb, pb, node);
node.setK(fpb.getDegreeOfParallelism());
long N = fpb.getNumIterations();
node.addParam(ParamType.NUM_ITERATIONS, (N != -1) ? String.valueOf(N) : String.valueOf(CostEstimator.FACTOR_NUM_ITERATIONS));
switch(fpb.getExecMode()) {
case LOCAL:
node.setExecType(ExecType.CP);
break;
case REMOTE_MR:
case REMOTE_MR_DP:
node.setExecType(ExecType.MR);
break;
case REMOTE_SPARK:
case REMOTE_SPARK_DP:
node.setExecType(ExecType.SPARK);
break;
case UNSPECIFIED:
node.setExecType(null);
}
if (!topLevel) {
fsb.getFromHops().resetVisitStatus();
fsb.getToHops().resetVisitStatus();
if (fsb.getIncrementHops() != null)
fsb.getIncrementHops().resetVisitStatus();
node.addChilds(rCreateAbstractOptNodes(fsb.getFromHops(), vars, memo));
node.addChilds(rCreateAbstractOptNodes(fsb.getToHops(), vars, memo));
if (fsb.getIncrementHops() != null)
node.addChilds(rCreateAbstractOptNodes(fsb.getIncrementHops(), vars, memo));
}
// process body
int len = fs.getBody().size();
for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
ProgramBlock lpb = fpb.getChildBlocks().get(i);
StatementBlock lsb = fs.getBody().get(i);
node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
}
// parameters, add required parameters
Map<String, String> lparams = fpb.getParForParams();
node.addParam(ParamType.DATA_PARTITIONER, lparams.get(ParForStatementBlock.DATA_PARTITIONER));
node.addParam(ParamType.TASK_PARTITIONER, lparams.get(ParForStatementBlock.TASK_PARTITIONER));
node.addParam(ParamType.RESULT_MERGE, lparams.get(ParForStatementBlock.RESULT_MERGE));
// TODO task size
} else // last level program block
{
sb = pb.getStatementBlock();
// process all hops
node = new OptNode(NodeType.GENERIC);
_hlMap.putProgMapping(sb, pb, node);
node.addChilds(createAbstractOptNodes(sb.getHops(), vars, memo));
node.setExecType(ExecType.CP);
node.setLineNumbers(sb.getBeginLine(), sb.getEndLine());
// TODO remove this workaround once this information can be obtained from hops/lops compiler
if (node.isCPOnly()) {
boolean isSparkExec = OptimizerUtils.isSparkExecutionMode();
if (!isSparkExec && containsMRJobInstruction(pb, false, false))
node.setExecType(ExecType.MR);
else if (isSparkExec && containsMRJobInstruction(pb, false, true))
node.setExecType(ExecType.SPARK);
}
}
// final cleanup
// NOTE: required because this function is also used to create subtrees
node.checkAndCleanupLeafNodes();
return node;
}
Aggregations