use of org.apache.sysml.lops.compile.Dag in project incubator-systemml by apache.
the class Recompiler method recompileHopsDag2Forced.
/**
* D) Recompile predicate hop DAG (single root), but forced to CP.
*
* This happens always 'inplace', without statistics updates, and
* without dynamic rewrites.
*
* @param hops list of high-level operators
* @param tid thread id
* @param et execution type
* @return list of instructions
* @throws DMLRuntimeException if DMLRuntimeException occurs
* @throws HopsException if HopsException occurs
* @throws LopsException if LopsException occurs
* @throws IOException if IOException occurs
*/
public static ArrayList<Instruction> recompileHopsDag2Forced(Hop hops, long tid, ExecType et) throws DMLRuntimeException, HopsException, LopsException, IOException {
ArrayList<Instruction> newInst = null;
//need for synchronization as we do temp changes in shared hops/lops
synchronized (hops) {
LOG.debug("\n**************** Optimizer (Recompile) *************\nMemory Budget = " + OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
// clear existing lops
hops.resetVisitStatus();
rClearLops(hops);
// update exec type
hops.resetVisitStatus();
rSetExecType(hops, et);
hops.resetVisitStatus();
// construct lops
Dag<Lop> dag = new Dag<Lop>();
Lop lops = hops.constructLops();
lops.addToDag(dag);
// generate runtime instructions (incl piggybacking)
newInst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
}
// replace thread ids in new instructions
if (//only in parfor context
tid != 0)
newInst = ProgramConverter.createDeepCopyInstructionSet(newInst, tid, -1, null, null, null, false, false);
return newInst;
}
use of org.apache.sysml.lops.compile.Dag in project incubator-systemml by apache.
the class Recompiler method recompileHopsDagInstructions.
public static ArrayList<Instruction> recompileHopsDagInstructions(Hop hops) throws DMLRuntimeException, HopsException, LopsException, IOException {
ArrayList<Instruction> newInst = null;
//need for synchronization as we do temp changes in shared hops/lops
synchronized (hops) {
LOG.debug("\n**************** Optimizer (Recompile) *************\nMemory Budget = " + OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
// clear existing lops
hops.resetVisitStatus();
rClearLops(hops);
// construct lops
Dag<Lop> dag = new Dag<Lop>();
Lop lops = hops.constructLops();
lops.addToDag(dag);
// generate runtime instructions (incl piggybacking)
newInst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
}
// explain recompiled instructions
if (DMLScript.EXPLAIN == ExplainType.RECOMPILE_HOPS)
LOG.info("EXPLAIN RECOMPILE \nPRED (line " + hops.getBeginLine() + "):\n" + Explain.explain(hops, 1));
if (DMLScript.EXPLAIN == ExplainType.RECOMPILE_RUNTIME)
LOG.info("EXPLAIN RECOMPILE \nPRED (line " + hops.getBeginLine() + "):\n" + Explain.explain(newInst, 1));
return newInst;
}
use of org.apache.sysml.lops.compile.Dag in project incubator-systemml by apache.
the class Recompiler method recompileHopsDagInstructions.
public static ArrayList<Instruction> recompileHopsDagInstructions(StatementBlock sb, ArrayList<Hop> hops) throws HopsException, LopsException, DMLRuntimeException, IOException {
ArrayList<Instruction> newInst = null;
//however, we create deep copies for most dags to allow for concurrent recompile
synchronized (hops) {
LOG.debug("\n**************** Optimizer (Recompile) *************\nMemory Budget = " + OptimizerUtils.toMB(OptimizerUtils.getLocalMemBudget()) + " MB");
// clear existing lops
Hop.resetVisitStatus(hops);
for (Hop hopRoot : hops) rClearLops(hopRoot);
// construct lops
Dag<Lop> dag = new Dag<Lop>();
for (Hop hopRoot : hops) {
Lop lops = hopRoot.constructLops();
lops.addToDag(dag);
}
// generate runtime instructions (incl piggybacking)
newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
}
// explain recompiled hops / instructions
if (DMLScript.EXPLAIN == ExplainType.RECOMPILE_HOPS) {
LOG.info("EXPLAIN RECOMPILE \nGENERIC (lines " + sb.getBeginLine() + "-" + sb.getEndLine() + "):\n" + Explain.explainHops(hops, 1));
}
if (DMLScript.EXPLAIN == ExplainType.RECOMPILE_RUNTIME) {
LOG.info("EXPLAIN RECOMPILE \nGENERIC (lines " + sb.getBeginLine() + "-" + sb.getEndLine() + "):\n" + Explain.explain(newInst, 1));
}
return newInst;
}
use of org.apache.sysml.lops.compile.Dag in project incubator-systemml by apache.
the class DMLTranslator method createRuntimeProgramBlock.
public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, DMLConfig config) {
Dag<Lop> dag = null;
Dag<Lop> pred_dag = null;
ArrayList<Instruction> instruct;
ArrayList<Instruction> pred_instruct = null;
ProgramBlock retPB = null;
// process While Statement - add runtime program blocks to program
if (sb instanceof WhileStatementBlock) {
// create DAG for loop predicates
pred_dag = new Dag<>();
((WhileStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
// create instructions for loop predicates
pred_instruct = new ArrayList<>();
ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
for (Instruction i : pInst) {
pred_instruct.add(i);
}
// create while program block
WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
// // process the body of the while statement block ////
WhileStatementBlock wsb = (WhileStatementBlock) sb;
WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
for (StatementBlock sblock : wstmt.getBody()) {
// process the body
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlock(childBlock);
}
retPB = rtpb;
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setParseInfo(sb);
} else // process If Statement - add runtime program blocks to program
if (sb instanceof IfStatementBlock) {
// create DAG for loop predicates
pred_dag = new Dag<>();
((IfStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
// create instructions for loop predicates
pred_instruct = new ArrayList<>();
ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
for (Instruction i : pInst) {
pred_instruct.add(i);
}
// create if program block
IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
// process the body of the if statement block
IfStatementBlock isb = (IfStatementBlock) sb;
IfStatement istmt = (IfStatement) isb.getStatement(0);
// process the if body
for (StatementBlock sblock : istmt.getIfBody()) {
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlockIfBody(childBlock);
}
// process the else body
for (StatementBlock sblock : istmt.getElseBody()) {
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlockElseBody(childBlock);
}
retPB = rtpb;
// post processing for generating missing instructions
// retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setParseInfo(sb);
} else // NOTE: applies to ForStatementBlock and ParForStatementBlock
if (sb instanceof ForStatementBlock) {
ForStatementBlock fsb = (ForStatementBlock) sb;
// create DAGs for loop predicates
Dag<Lop> fromDag = new Dag<>();
Dag<Lop> toDag = new Dag<>();
Dag<Lop> incrementDag = new Dag<>();
if (fsb.getFromHops() != null)
fsb.getFromLops().addToDag(fromDag);
if (fsb.getToHops() != null)
fsb.getToLops().addToDag(toDag);
if (fsb.getIncrementHops() != null)
fsb.getIncrementLops().addToDag(incrementDag);
// create instructions for loop predicates
ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
// create for program block
ForProgramBlock rtpb = null;
IterablePredicate iterPred = fsb.getIterPredicate();
if (sb instanceof ParForStatementBlock && ConfigurationManager.isParallelParFor()) {
rtpb = new ParForProgramBlock(prog, iterPred.getIterVar().getName(), iterPred.getParForParams(), ((ParForStatementBlock) sb).getResultVariables());
ParForProgramBlock pfrtpb = (ParForProgramBlock) rtpb;
// used for optimization and creating unscoped variables
pfrtpb.setStatementBlock((ParForStatementBlock) sb);
} else {
// ForStatementBlock
rtpb = new ForProgramBlock(prog, iterPred.getIterVar().getName());
}
rtpb.setFromInstructions(fromInstructions);
rtpb.setToInstructions(toInstructions);
rtpb.setIncrementInstructions(incrementInstructions);
// process the body of the for statement block
ForStatement fs = (ForStatement) fsb.getStatement(0);
for (StatementBlock sblock : fs.getBody()) {
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlock(childBlock);
}
retPB = rtpb;
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setParseInfo(sb);
} else // process function statement block - add runtime program blocks to program
if (sb instanceof FunctionStatementBlock) {
FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
FunctionProgramBlock rtpb = null;
if (fstmt instanceof ExternalFunctionStatement) {
// create external function program block
String execType = ((ExternalFunctionStatement) fstmt).getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
boolean isCP = (execType.equals(ExternalFunctionStatement.IN_MEMORY)) ? true : false;
StringBuilder buff = new StringBuilder();
buff.append(config.getTextValue(DMLConfig.SCRATCH_SPACE));
buff.append(Lop.FILE_SEPARATOR);
buff.append(Lop.PROCESS_PREFIX);
buff.append(DMLScript.getUUID());
buff.append(Lop.FILE_SEPARATOR);
buff.append(ProgramConverter.CP_ROOT_THREAD_ID);
buff.append(Lop.FILE_SEPARATOR);
buff.append("PackageSupport");
buff.append(Lop.FILE_SEPARATOR);
String basedir = buff.toString();
if (isCP) {
rtpb = new ExternalFunctionProgramBlockCP(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
} else {
rtpb = new ExternalFunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
}
if (!fstmt.getBody().isEmpty()) {
LOG.error(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
throw new LopsException(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
}
} else {
// create function program block
rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams());
// process the function statement body
for (StatementBlock sblock : fstmt.getBody()) {
// process the body
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlock(childBlock);
}
}
// check there are actually Lops in to process (loop stmt body will not have any)
if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
}
retPB = rtpb;
// add location information
retPB.setParseInfo(sb);
} else {
// handle general case
ProgramBlock rtpb = new ProgramBlock(prog);
// DAGs for Lops
dag = new Dag<>();
// check there are actually Lops in to process (loop stmt body will not have any)
if (sb.getLops() != null && !sb.getLops().isEmpty()) {
for (Lop l : sb.getLops()) {
l.addToDag(dag);
}
// Instructions for Lops DAGs
instruct = dag.getJobs(sb, config);
rtpb.addInstructions(instruct);
}
retPB = rtpb;
// post processing for generating missing instructions
// retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setParseInfo(sb);
}
return retPB;
}
use of org.apache.sysml.lops.compile.Dag in project incubator-systemml by apache.
the class RewriteConstantFolding method evalScalarOperation.
/**
* In order to (1) prevent unexpected side effects from constant folding and
* (2) for simplicity with regard to arbitrary value type combinations,
* we use the same compilation and runtime for constant folding as we would
* use for actual instruction execution.
*
* @param bop high-level operator
* @return literal op
*/
private LiteralOp evalScalarOperation(Hop bop) {
// Timing time = new Timing( true );
DataOp tmpWrite = new DataOp(TMP_VARNAME, bop.getDataType(), bop.getValueType(), bop, DataOpTypes.TRANSIENTWRITE, TMP_VARNAME);
// generate runtime instruction
Dag<Lop> dag = new Dag<>();
// prevent lops reuse
Recompiler.rClearLops(tmpWrite);
// reconstruct lops
Lop lops = tmpWrite.constructLops();
lops.addToDag(dag);
ArrayList<Instruction> inst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
// execute instructions
ExecutionContext ec = getExecutionContext();
ProgramBlock pb = getProgramBlock();
pb.setInstructions(inst);
pb.execute(ec);
// get scalar result (check before invocation) and create literal according
// to observed scalar output type (not hop type) for runtime consistency
ScalarObject so = (ScalarObject) ec.getVariable(TMP_VARNAME);
LiteralOp literal = null;
switch(so.getValueType()) {
case DOUBLE:
literal = new LiteralOp(so.getDoubleValue());
break;
case INT:
literal = new LiteralOp(so.getLongValue());
break;
case BOOLEAN:
literal = new LiteralOp(so.getBooleanValue());
break;
case STRING:
literal = new LiteralOp(so.getStringValue());
break;
default:
throw new HopsException("Unsupported literal value type: " + bop.getValueType());
}
// cleanup
tmpWrite.getInput().clear();
bop.getParent().remove(tmpWrite);
pb.setInstructions(null);
ec.getVariables().removeAll();
// set literal properties (scalar)
HopRewriteUtils.setOutputParametersForScalar(literal);
return literal;
}
Aggregations