use of org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlockCP in project incubator-systemml by apache.
the class ProgramConverter method rParseExternalFunctionProgramBlock.
private static ExternalFunctionProgramBlock rParseExternalFunctionProgramBlock(String in, Program prog, int id) {
String lin = in.substring(PARFOR_PB_EFC.length(), in.length() - PARFOR_PB_END.length());
HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
// LocalVariableMap vars = parseVariables(st.nextToken());
// inputs, outputs and params
ArrayList<DataIdentifier> dat1 = parseDataIdentifiers(st.nextToken());
ArrayList<DataIdentifier> dat2 = parseDataIdentifiers(st.nextToken());
HashMap<String, String> dat3 = parseStringHashMap(st.nextToken());
// basedir
String basedir = st.nextToken();
// instructions (required for removing INST BEGIN, END)
parseInstructions(st.nextToken(), id);
// program blocks
ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id);
ArrayList<DataIdentifier> tmp1 = new ArrayList<>(dat1);
ArrayList<DataIdentifier> tmp2 = new ArrayList<>(dat2);
// only CP external functions, because no nested MR jobs for reblocks
ExternalFunctionProgramBlockCP efpb = new ExternalFunctionProgramBlockCP(prog, tmp1, tmp2, dat3, basedir);
efpb.setChildBlocks(pbs);
return efpb;
}
use of org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlockCP in project systemml by apache.
the class DMLTranslator method createRuntimeProgramBlock.
public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, DMLConfig config) {
Dag<Lop> dag = null;
Dag<Lop> pred_dag = null;
ArrayList<Instruction> instruct;
ArrayList<Instruction> pred_instruct = null;
ProgramBlock retPB = null;
// process While Statement - add runtime program blocks to program
if (sb instanceof WhileStatementBlock) {
// create DAG for loop predicates
pred_dag = new Dag<>();
((WhileStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
// create instructions for loop predicates
pred_instruct = new ArrayList<>();
ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
for (Instruction i : pInst) {
pred_instruct.add(i);
}
// create while program block
WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
// // process the body of the while statement block ////
WhileStatementBlock wsb = (WhileStatementBlock) sb;
WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
for (StatementBlock sblock : wstmt.getBody()) {
// process the body
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlock(childBlock);
}
retPB = rtpb;
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setParseInfo(sb);
} else // process If Statement - add runtime program blocks to program
if (sb instanceof IfStatementBlock) {
// create DAG for loop predicates
pred_dag = new Dag<>();
((IfStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
// create instructions for loop predicates
pred_instruct = new ArrayList<>();
ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
for (Instruction i : pInst) {
pred_instruct.add(i);
}
// create if program block
IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
// process the body of the if statement block
IfStatementBlock isb = (IfStatementBlock) sb;
IfStatement istmt = (IfStatement) isb.getStatement(0);
// process the if body
for (StatementBlock sblock : istmt.getIfBody()) {
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlockIfBody(childBlock);
}
// process the else body
for (StatementBlock sblock : istmt.getElseBody()) {
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlockElseBody(childBlock);
}
retPB = rtpb;
// post processing for generating missing instructions
// retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setParseInfo(sb);
} else // NOTE: applies to ForStatementBlock and ParForStatementBlock
if (sb instanceof ForStatementBlock) {
ForStatementBlock fsb = (ForStatementBlock) sb;
// create DAGs for loop predicates
Dag<Lop> fromDag = new Dag<>();
Dag<Lop> toDag = new Dag<>();
Dag<Lop> incrementDag = new Dag<>();
if (fsb.getFromHops() != null)
fsb.getFromLops().addToDag(fromDag);
if (fsb.getToHops() != null)
fsb.getToLops().addToDag(toDag);
if (fsb.getIncrementHops() != null)
fsb.getIncrementLops().addToDag(incrementDag);
// create instructions for loop predicates
ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
// create for program block
ForProgramBlock rtpb = null;
IterablePredicate iterPred = fsb.getIterPredicate();
if (sb instanceof ParForStatementBlock && ConfigurationManager.isParallelParFor()) {
rtpb = new ParForProgramBlock(prog, iterPred.getIterVar().getName(), iterPred.getParForParams(), ((ParForStatementBlock) sb).getResultVariables());
ParForProgramBlock pfrtpb = (ParForProgramBlock) rtpb;
// used for optimization and creating unscoped variables
pfrtpb.setStatementBlock((ParForStatementBlock) sb);
} else {
// ForStatementBlock
rtpb = new ForProgramBlock(prog, iterPred.getIterVar().getName());
}
rtpb.setFromInstructions(fromInstructions);
rtpb.setToInstructions(toInstructions);
rtpb.setIncrementInstructions(incrementInstructions);
// process the body of the for statement block
ForStatement fs = (ForStatement) fsb.getStatement(0);
for (StatementBlock sblock : fs.getBody()) {
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlock(childBlock);
}
retPB = rtpb;
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setParseInfo(sb);
} else // process function statement block - add runtime program blocks to program
if (sb instanceof FunctionStatementBlock) {
FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
FunctionProgramBlock rtpb = null;
if (fstmt instanceof ExternalFunctionStatement) {
// create external function program block
String execType = ((ExternalFunctionStatement) fstmt).getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
boolean isCP = (execType.equals(ExternalFunctionStatement.IN_MEMORY)) ? true : false;
StringBuilder buff = new StringBuilder();
buff.append(config.getTextValue(DMLConfig.SCRATCH_SPACE));
buff.append(Lop.FILE_SEPARATOR);
buff.append(Lop.PROCESS_PREFIX);
buff.append(DMLScript.getUUID());
buff.append(Lop.FILE_SEPARATOR);
buff.append(ProgramConverter.CP_ROOT_THREAD_ID);
buff.append(Lop.FILE_SEPARATOR);
buff.append("PackageSupport");
buff.append(Lop.FILE_SEPARATOR);
String basedir = buff.toString();
if (isCP) {
rtpb = new ExternalFunctionProgramBlockCP(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
} else {
rtpb = new ExternalFunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
}
if (!fstmt.getBody().isEmpty()) {
LOG.error(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
throw new LopsException(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
}
} else {
// create function program block
rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams());
// process the function statement body
for (StatementBlock sblock : fstmt.getBody()) {
// process the body
ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
rtpb.addProgramBlock(childBlock);
}
}
// check there are actually Lops in to process (loop stmt body will not have any)
if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
}
retPB = rtpb;
// add location information
retPB.setParseInfo(sb);
} else {
// handle general case
ProgramBlock rtpb = new ProgramBlock(prog);
// DAGs for Lops
dag = new Dag<>();
// check there are actually Lops in to process (loop stmt body will not have any)
if (sb.getLops() != null && !sb.getLops().isEmpty()) {
for (Lop l : sb.getLops()) {
l.addToDag(dag);
}
// Instructions for Lops DAGs
instruct = dag.getJobs(sb, config);
rtpb.addInstructions(instruct);
}
retPB = rtpb;
// post processing for generating missing instructions
// retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
// add statement block
retPB.setStatementBlock(sb);
// add location information
retPB.setParseInfo(sb);
}
return retPB;
}
use of org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlockCP in project systemml by apache.
the class ProgramConverter method createDeepCopyFunctionProgramBlock.
/**
* This creates a deep copy of a function program block. The central reference to singletons of function program blocks
* poses the need for explicit copies in order to prevent conflicting writes of temporary variables (see ExternalFunctionProgramBlock.
*
* @param namespace function namespace
* @param oldName ?
* @param pid ?
* @param IDPrefix ?
* @param prog runtime program
* @param fnStack ?
* @param fnCreated ?
* @param plain ?
*/
public static void createDeepCopyFunctionProgramBlock(String namespace, String oldName, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain) {
// fpb guaranteed to be non-null (checked inside getFunctionProgramBlock)
FunctionProgramBlock fpb = prog.getFunctionProgramBlock(namespace, oldName);
String fnameNew = (plain) ? oldName : (oldName + CP_CHILD_THREAD + pid);
String fnameNewKey = DMLProgram.constructFunctionKey(namespace, fnameNew);
if (prog.getFunctionProgramBlocks().containsKey(fnameNewKey))
// prevent redundant deep copy if already existent
return;
// create deep copy
FunctionProgramBlock copy = null;
ArrayList<DataIdentifier> tmp1 = new ArrayList<>();
ArrayList<DataIdentifier> tmp2 = new ArrayList<>();
if (fpb.getInputParams() != null)
tmp1.addAll(fpb.getInputParams());
if (fpb.getOutputParams() != null)
tmp2.addAll(fpb.getOutputParams());
if (fpb instanceof ExternalFunctionProgramBlockCP) {
ExternalFunctionProgramBlockCP efpb = (ExternalFunctionProgramBlockCP) fpb;
HashMap<String, String> tmp3 = efpb.getOtherParams();
if (IDPrefix != -1)
copy = new ExternalFunctionProgramBlockCP(prog, tmp1, tmp2, tmp3, saveReplaceFilenameThreadID(efpb.getBaseDir(), CP_CHILD_THREAD + IDPrefix, CP_CHILD_THREAD + pid));
else
copy = new ExternalFunctionProgramBlockCP(prog, tmp1, tmp2, tmp3, saveReplaceFilenameThreadID(efpb.getBaseDir(), CP_ROOT_THREAD_ID, CP_CHILD_THREAD + pid));
} else if (fpb instanceof ExternalFunctionProgramBlock) {
ExternalFunctionProgramBlock efpb = (ExternalFunctionProgramBlock) fpb;
HashMap<String, String> tmp3 = efpb.getOtherParams();
if (IDPrefix != -1)
copy = new ExternalFunctionProgramBlock(prog, tmp1, tmp2, tmp3, saveReplaceFilenameThreadID(efpb.getBaseDir(), CP_CHILD_THREAD + IDPrefix, CP_CHILD_THREAD + pid));
else
copy = new ExternalFunctionProgramBlock(prog, tmp1, tmp2, tmp3, saveReplaceFilenameThreadID(efpb.getBaseDir(), CP_ROOT_THREAD_ID, CP_CHILD_THREAD + pid));
} else {
if (!fnStack.contains(fnameNewKey)) {
fnStack.add(fnameNewKey);
copy = new FunctionProgramBlock(prog, tmp1, tmp2);
copy.setChildBlocks(rcreateDeepCopyProgramBlocks(fpb.getChildBlocks(), pid, IDPrefix, fnStack, fnCreated, plain, fpb.isRecompileOnce()));
copy.setRecompileOnce(fpb.isRecompileOnce());
copy.setThreadID(pid);
fnStack.remove(fnameNewKey);
} else
// stop deep copy for recursive function calls
copy = fpb;
}
// copy.setVariables( (LocalVariableMap) fpb.getVariables() ); //implicit cloning
// note: instructions not used by function program block
// put
prog.addFunctionProgramBlock(namespace, fnameNew, copy);
fnCreated.add(DMLProgram.constructFunctionKey(namespace, fnameNew));
}
use of org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlockCP in project systemml by apache.
the class ProgramConverter method rParseExternalFunctionProgramBlock.
private static ExternalFunctionProgramBlock rParseExternalFunctionProgramBlock(String in, Program prog, int id) {
String lin = in.substring(PARFOR_PB_EFC.length(), in.length() - PARFOR_PB_END.length());
HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
// LocalVariableMap vars = parseVariables(st.nextToken());
// inputs, outputs and params
ArrayList<DataIdentifier> dat1 = parseDataIdentifiers(st.nextToken());
ArrayList<DataIdentifier> dat2 = parseDataIdentifiers(st.nextToken());
HashMap<String, String> dat3 = parseStringHashMap(st.nextToken());
// basedir
String basedir = st.nextToken();
// instructions (required for removing INST BEGIN, END)
parseInstructions(st.nextToken(), id);
// program blocks
ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id);
ArrayList<DataIdentifier> tmp1 = new ArrayList<>(dat1);
ArrayList<DataIdentifier> tmp2 = new ArrayList<>(dat2);
// only CP external functions, because no nested MR jobs for reblocks
ExternalFunctionProgramBlockCP efpb = new ExternalFunctionProgramBlockCP(prog, tmp1, tmp2, dat3, basedir);
efpb.setChildBlocks(pbs);
return efpb;
}
Aggregations