use of org.apache.sysml.runtime.controlprogram.ForProgramBlock in project systemml by apache.
the class Explain method explainProgramBlock.
// ////////////
// internal explain RUNTIME
private static String explainProgramBlock(ProgramBlock pb, int level) {
StringBuilder sb = new StringBuilder();
String offset = createOffset(level);
if (pb instanceof FunctionProgramBlock) {
FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
for (ProgramBlock pbc : fpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
} else if (pb instanceof WhileProgramBlock) {
WhileProgramBlock wpb = (WhileProgramBlock) pb;
StatementBlock wsb = pb.getStatementBlock();
sb.append(offset);
if (wsb != null && !wsb.getUpdateInPlaceVars().isEmpty())
sb.append("WHILE (lines " + wpb.getBeginLine() + "-" + wpb.getEndLine() + ") [in-place=" + wsb.getUpdateInPlaceVars().toString() + "]\n");
else
sb.append("WHILE (lines " + wpb.getBeginLine() + "-" + wpb.getEndLine() + ")\n");
sb.append(explainInstructions(wpb.getPredicate(), level + 1));
for (ProgramBlock pbc : wpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
} else if (pb instanceof IfProgramBlock) {
IfProgramBlock ipb = (IfProgramBlock) pb;
sb.append(offset);
sb.append("IF (lines " + ipb.getBeginLine() + "-" + ipb.getEndLine() + ")\n");
sb.append(explainInstructions(ipb.getPredicate(), level + 1));
for (ProgramBlock pbc : ipb.getChildBlocksIfBody()) sb.append(explainProgramBlock(pbc, level + 1));
if (!ipb.getChildBlocksElseBody().isEmpty()) {
sb.append(offset);
sb.append("ELSE\n");
for (ProgramBlock pbc : ipb.getChildBlocksElseBody()) sb.append(explainProgramBlock(pbc, level + 1));
}
} else if (// incl parfor
pb instanceof ForProgramBlock) {
ForProgramBlock fpb = (ForProgramBlock) pb;
StatementBlock fsb = pb.getStatementBlock();
sb.append(offset);
if (pb instanceof ParForProgramBlock)
sb.append("PARFOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ")\n");
else {
if (fsb != null && !fsb.getUpdateInPlaceVars().isEmpty())
sb.append("FOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ") [in-place=" + fsb.getUpdateInPlaceVars().toString() + "]\n");
else
sb.append("FOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ")\n");
}
sb.append(explainInstructions(fpb.getFromInstructions(), level + 1));
sb.append(explainInstructions(fpb.getToInstructions(), level + 1));
sb.append(explainInstructions(fpb.getIncrementInstructions(), level + 1));
for (ProgramBlock pbc : fpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
} else {
sb.append(offset);
if (pb.getStatementBlock() != null)
sb.append("GENERIC (lines " + pb.getBeginLine() + "-" + pb.getEndLine() + ") [recompile=" + pb.getStatementBlock().requiresRecompilation() + "]\n");
else
sb.append("GENERIC (lines " + pb.getBeginLine() + "-" + pb.getEndLine() + ") \n");
sb.append(explainInstructions(pb.getInstructions(), level + 1));
}
return sb.toString();
}
use of org.apache.sysml.runtime.controlprogram.ForProgramBlock in project systemml by apache.
the class GridEnumerationMemory method getMemoryEstimates.
private void getMemoryEstimates(ProgramBlock pb, ArrayList<Long> mem) {
if (pb instanceof FunctionProgramBlock) {
FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
getMemoryEstimates(fpb.getChildBlocks(), mem);
} else if (pb instanceof WhileProgramBlock) {
WhileProgramBlock fpb = (WhileProgramBlock) pb;
getMemoryEstimates(fpb.getChildBlocks(), mem);
} else if (pb instanceof IfProgramBlock) {
IfProgramBlock fpb = (IfProgramBlock) pb;
getMemoryEstimates(fpb.getChildBlocksIfBody(), mem);
getMemoryEstimates(fpb.getChildBlocksElseBody(), mem);
} else if (// incl parfor
pb instanceof ForProgramBlock) {
ForProgramBlock fpb = (ForProgramBlock) pb;
getMemoryEstimates(fpb.getChildBlocks(), mem);
} else {
StatementBlock sb = pb.getStatementBlock();
if (sb != null && sb.getHops() != null) {
Hop.resetVisitStatus(sb.getHops());
for (Hop hop : sb.getHops()) getMemoryEstimates(hop, mem);
}
}
}
use of org.apache.sysml.runtime.controlprogram.ForProgramBlock in project systemml by apache.
the class ResourceOptimizer method recompileProgramBlock.
private static void recompileProgramBlock(ProgramBlock pb, long cp, long mr) {
// init compiler memory budget
InfrastructureAnalyzer.setLocalMaxMemory(cp);
InfrastructureAnalyzer.setRemoteMaxMemoryMap(mr);
InfrastructureAnalyzer.setRemoteMaxMemoryReduce(mr);
// dependent on cp, mr
OptimizerUtils.resetDefaultSize();
// recompile instructions (incl predicates)
if (pb instanceof WhileProgramBlock) {
WhileProgramBlock wpb = (WhileProgramBlock) pb;
WhileStatementBlock sb = (WhileStatementBlock) pb.getStatementBlock();
if (INCLUDE_PREDICATES && sb != null && sb.getPredicateHops() != null) {
ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getPredicateHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
wpb.setPredicate(inst);
}
} else if (pb instanceof IfProgramBlock) {
IfProgramBlock ipb = (IfProgramBlock) pb;
IfStatementBlock sb = (IfStatementBlock) ipb.getStatementBlock();
if (INCLUDE_PREDICATES && sb != null && sb.getPredicateHops() != null) {
ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getPredicateHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
ipb.setPredicate(inst);
}
} else if (pb instanceof ForProgramBlock) {
// incl parfor
ForProgramBlock fpb = (ForProgramBlock) pb;
ForStatementBlock sb = (ForStatementBlock) fpb.getStatementBlock();
if (INCLUDE_PREDICATES && sb != null) {
if (sb.getFromHops() != null) {
ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getFromHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
fpb.setFromInstructions(inst);
}
if (sb.getToHops() != null) {
ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getToHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
fpb.setToInstructions(inst);
}
if (sb.getIncrementHops() != null) {
ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getIncrementHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
fpb.setIncrementInstructions(inst);
}
}
} else {
// last-level program blocks
StatementBlock sb = pb.getStatementBlock();
ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb, sb.getHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
pb.setInstructions(inst);
}
_cntCompilePB++;
}
use of org.apache.sysml.runtime.controlprogram.ForProgramBlock in project systemml by apache.
the class Recompiler method rRecompileProgramBlock2Forced.
private static void rRecompileProgramBlock2Forced(ProgramBlock pb, long tid, HashSet<String> fnStack, ExecType et) {
if (pb instanceof WhileProgramBlock) {
WhileProgramBlock pbTmp = (WhileProgramBlock) pb;
WhileStatementBlock sbTmp = (WhileStatementBlock) pbTmp.getStatementBlock();
// recompile predicate
if (sbTmp != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getPredicate(), true, true)))
pbTmp.setPredicate(Recompiler.recompileHopsDag2Forced(sbTmp.getPredicateHops(), tid, et));
// recompile body
for (ProgramBlock pb2 : pbTmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
} else if (pb instanceof IfProgramBlock) {
IfProgramBlock pbTmp = (IfProgramBlock) pb;
IfStatementBlock sbTmp = (IfStatementBlock) pbTmp.getStatementBlock();
// recompile predicate
if (sbTmp != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getPredicate(), true, true)))
pbTmp.setPredicate(Recompiler.recompileHopsDag2Forced(sbTmp.getPredicateHops(), tid, et));
// recompile body
for (ProgramBlock pb2 : pbTmp.getChildBlocksIfBody()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
for (ProgramBlock pb2 : pbTmp.getChildBlocksElseBody()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
} else if (// includes ParFORProgramBlock
pb instanceof ForProgramBlock) {
ForProgramBlock pbTmp = (ForProgramBlock) pb;
ForStatementBlock sbTmp = (ForStatementBlock) pbTmp.getStatementBlock();
// recompile predicate
if (sbTmp != null && sbTmp.getFromHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getFromInstructions(), true, true)))
pbTmp.setFromInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getFromHops(), tid, et));
if (sbTmp != null && sbTmp.getToHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getToInstructions(), true, true)))
pbTmp.setToInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getToHops(), tid, et));
if (sbTmp != null && sbTmp.getIncrementHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getIncrementInstructions(), true, true)))
pbTmp.setIncrementInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getIncrementHops(), tid, et));
// recompile body
for (ProgramBlock pb2 : pbTmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
} else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
pb instanceof FunctionProgramBlock) {
FunctionProgramBlock tmp = (FunctionProgramBlock) pb;
for (ProgramBlock pb2 : tmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
} else {
StatementBlock sb = pb.getStatementBlock();
// would be invalid with permutation matrix mult across multiple dags)
if (sb != null) {
ArrayList<Instruction> tmp = pb.getInstructions();
tmp = Recompiler.recompileHopsDag2Forced(sb, sb.getHops(), tid, et);
pb.setInstructions(tmp);
}
// recompile functions
if (OptTreeConverter.containsFunctionCallInstruction(pb)) {
ArrayList<Instruction> tmp = pb.getInstructions();
for (Instruction inst : tmp) if (inst instanceof FunctionCallCPInstruction) {
FunctionCallCPInstruction func = (FunctionCallCPInstruction) inst;
String fname = func.getFunctionName();
String fnamespace = func.getNamespace();
String fKey = DMLProgram.constructFunctionKey(fnamespace, fname);
if (// memoization for multiple calls, recursion
!fnStack.contains(fKey)) {
fnStack.add(fKey);
FunctionProgramBlock fpb = pb.getProgram().getFunctionProgramBlock(fnamespace, fname);
// recompile chains of functions
rRecompileProgramBlock2Forced(fpb, tid, fnStack, et);
}
}
}
}
}
use of org.apache.sysml.runtime.controlprogram.ForProgramBlock in project systemml by apache.
the class CostEstimator method rGetTimeEstimate.
private double rGetTimeEstimate(ProgramBlock pb, HashMap<String, VarStats> stats, HashSet<String> memoFunc, boolean recursive) {
double ret = 0;
if (pb instanceof WhileProgramBlock) {
WhileProgramBlock tmp = (WhileProgramBlock) pb;
if (recursive)
for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
ret *= DEFAULT_NUMITER;
} else if (pb instanceof IfProgramBlock) {
IfProgramBlock tmp = (IfProgramBlock) pb;
if (recursive) {
for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
if (tmp.getChildBlocksElseBody() != null)
for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) {
ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
// weighted sum
ret /= 2;
}
}
} else if (// includes ParFORProgramBlock
pb instanceof ForProgramBlock) {
ForProgramBlock tmp = (ForProgramBlock) pb;
if (recursive)
for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
ret *= getNumIterations(stats, tmp);
} else if (pb instanceof FunctionProgramBlock && // see generic
!(pb instanceof ExternalFunctionProgramBlock)) {
FunctionProgramBlock tmp = (FunctionProgramBlock) pb;
if (recursive)
for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
} else {
ArrayList<Instruction> tmp = pb.getInstructions();
for (Instruction inst : tmp) {
if (// CP
inst instanceof CPInstruction) {
// obtain stats from createvar, cpvar, rmvar, rand
maintainCPInstVariableStatistics((CPInstruction) inst, stats);
// extract statistics (instruction-specific)
Object[] o = extractCPInstStatistics(inst, stats);
VarStats[] vs = (VarStats[]) o[0];
String[] attr = (String[]) o[1];
// if(LOG.isDebugEnabled())
// LOG.debug(inst);
// call time estimation for inst
ret += getCPInstTimeEstimate(inst, vs, attr);
if (// functions
inst instanceof FunctionCallCPInstruction) {
FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst;
String fkey = DMLProgram.constructFunctionKey(finst.getNamespace(), finst.getFunctionName());
// awareness of recursive functions, missing program
if (!memoFunc.contains(fkey) && pb.getProgram() != null) {
if (LOG.isDebugEnabled())
LOG.debug("Begin Function " + fkey);
memoFunc.add(fkey);
Program prog = pb.getProgram();
FunctionProgramBlock fpb = prog.getFunctionProgramBlock(finst.getNamespace(), finst.getFunctionName());
ret += rGetTimeEstimate(fpb, stats, memoFunc, recursive);
memoFunc.remove(fkey);
if (LOG.isDebugEnabled())
LOG.debug("End Function " + fkey);
}
}
} else if (// MR
inst instanceof MRJobInstruction) {
// obtain stats for job
maintainMRJobInstVariableStatistics(inst, stats);
// extract input statistics
Object[] o = extractMRJobInstStatistics(inst, stats);
VarStats[] vs = (VarStats[]) o[0];
if (LOG.isDebugEnabled())
LOG.debug("Begin MRJob type=" + ((MRJobInstruction) inst).getJobType());
// call time estimation for complex MR inst
ret += getMRJobInstTimeEstimate(inst, vs, null);
if (LOG.isDebugEnabled())
LOG.debug("End MRJob");
// cleanup stats for job
cleanupMRJobVariableStatistics(inst, stats);
}
}
}
return ret;
}
Aggregations