Examples with VariableSet org.apache.sysml.parser.VariableSet used on opensource projects

Example 6 with VariableSet

use of org.apache.sysml.parser.VariableSet in project incubator-systemml by apache.

the class RewriteMarkLoopVariablesUpdateInPlace method rewriteStatementBlock.

@Override
public List<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus status) {
    if (DMLScript.rtplatform == RUNTIME_PLATFORM.HADOOP || DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK) {
        // nothing to do here, return original statement block
        return Arrays.asList(sb);
    }
    if (// incl parfor
    sb instanceof WhileStatementBlock || sb instanceof ForStatementBlock) {
        ArrayList<String> candidates = new ArrayList<>();
        VariableSet updated = sb.variablesUpdated();
        VariableSet liveout = sb.liveOut();
        for (String varname : updated.getVariableNames()) {
            if (updated.getVariable(varname).getDataType() == DataType.MATRIX && // exclude local vars
            liveout.containsVariable(varname)) {
                if (sb instanceof WhileStatementBlock) {
                    WhileStatement wstmt = (WhileStatement) sb.getStatement(0);
                    if (rIsApplicableForUpdateInPlace(wstmt.getBody(), varname))
                        candidates.add(varname);
                } else if (sb instanceof ForStatementBlock) {
                    ForStatement wstmt = (ForStatement) sb.getStatement(0);
                    if (rIsApplicableForUpdateInPlace(wstmt.getBody(), varname))
                        candidates.add(varname);
                }
            }
        }
        sb.setUpdateInPlaceVars(candidates);
    }
    // return modified statement block
    return Arrays.asList(sb);
}

Example 7 with VariableSet

use of org.apache.sysml.parser.VariableSet in project incubator-systemml by apache.

the class RewriteSplitDagDataDependentOperators method rewriteStatementBlock.

@Override
public List<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus state) {
    // DAG splits not required for forced single node
    if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE || !HopRewriteUtils.isLastLevelStatementBlock(sb))
        return Arrays.asList(sb);
    ArrayList<StatementBlock> ret = new ArrayList<>();
    // collect all unknown csv reads hops
    ArrayList<Hop> cand = new ArrayList<>();
    collectDataDependentOperators(sb.getHops(), cand);
    Hop.resetVisitStatus(sb.getHops());
    // split hop dag on demand
    if (!cand.isEmpty()) {
        // collect child operators of candidates (to prevent rewrite anomalies)
        HashSet<Hop> candChilds = new HashSet<>();
        collectCandidateChildOperators(cand, candChilds);
        try {
            // duplicate sb incl live variable sets
            StatementBlock sb1 = new StatementBlock();
            sb1.setDMLProg(sb.getDMLProg());
            sb1.setParseInfo(sb);
            sb1.setLiveIn(new VariableSet());
            sb1.setLiveOut(new VariableSet());
            // move data-dependent ops incl transient writes to new statement block
            // (and replace original persistent read with transient read)
            ArrayList<Hop> sb1hops = new ArrayList<>();
            for (Hop c : cand) {
                // if there are already transient writes use them and don't introduce artificial variables;
                // unless there are transient reads w/ the same variable name in the current dag which can
                // lead to invalid reordering if variable consumers are not feeding into the candidate op.
                boolean hasTWrites = hasTransientWriteParents(c);
                boolean moveTWrite = hasTWrites ? HopRewriteUtils.rHasSimpleReadChain(c, getFirstTransientWriteParent(c).getName()) : false;
                String varname = null;
                long rlen = c.getDim1();
                long clen = c.getDim2();
                long nnz = c.getNnz();
                UpdateType update = c.getUpdateType();
                int brlen = c.getRowsInBlock();
                int bclen = c.getColsInBlock();
                if (// reuse existing transient_write
                hasTWrites && moveTWrite) {
                    Hop twrite = getFirstTransientWriteParent(c);
                    varname = twrite.getName();
                    // create new transient read
                    DataOp tread = new DataOp(varname, c.getDataType(), c.getValueType(), DataOpTypes.TRANSIENTREAD, null, rlen, clen, nnz, update, brlen, bclen);
                    tread.setVisited();
                    HopRewriteUtils.copyLineNumbers(c, tread);
                    // replace data-dependent operator with transient read
                    ArrayList<Hop> parents = new ArrayList<>(c.getParent());
                    for (int i = 0; i < parents.size(); i++) {
                        // prevent concurrent modification by index access
                        Hop parent = parents.get(i);
                        if (!candChilds.contains(parent)) {
                            // anomaly filter
                            if (parent != twrite)
                                HopRewriteUtils.replaceChildReference(parent, c, tread);
                            else
                                sb.getHops().remove(parent);
                        }
                    }
                    // add data-dependent operator sub dag to first statement block
                    sb1hops.add(twrite);
                } else // create transient write to artificial variables
                {
                    varname = createCutVarName(false);
                    // create new transient read
                    DataOp tread = new DataOp(varname, c.getDataType(), c.getValueType(), DataOpTypes.TRANSIENTREAD, null, rlen, clen, nnz, update, brlen, bclen);
                    tread.setVisited();
                    HopRewriteUtils.copyLineNumbers(c, tread);
                    // replace data-dependent operator with transient read
                    ArrayList<Hop> parents = new ArrayList<>(c.getParent());
                    for (int i = 0; i < parents.size(); i++) {
                        // prevent concurrent modification by index access
                        Hop parent = parents.get(i);
                        if (// anomaly filter
                        !candChilds.contains(parent))
                            HopRewriteUtils.replaceChildReference(parent, c, tread);
                    }
                    // add data-dependent operator sub dag to first statement block
                    DataOp twrite = new DataOp(varname, c.getDataType(), c.getValueType(), c, DataOpTypes.TRANSIENTWRITE, null);
                    twrite.setVisited();
                    twrite.setOutputParams(rlen, clen, nnz, update, brlen, bclen);
                    HopRewriteUtils.copyLineNumbers(c, twrite);
                    sb1hops.add(twrite);
                }
                // update live in and out of new statement block (for piggybacking)
                DataIdentifier diVar = new DataIdentifier(varname);
                diVar.setDimensions(rlen, clen);
                diVar.setBlockDimensions(brlen, bclen);
                diVar.setDataType(c.getDataType());
                diVar.setValueType(c.getValueType());
                sb1.liveOut().addVariable(varname, new DataIdentifier(diVar));
                sb.liveIn().addVariable(varname, new DataIdentifier(diVar));
            }
            // ensure disjoint operators across DAGs (prevent replicated operations)
            handleReplicatedOperators(sb1hops, sb.getHops(), sb1.liveOut(), sb.liveIn());
            // deep copy new dag (in order to prevent any dangling references)
            sb1.setHops(Recompiler.deepCopyHopsDag(sb1hops));
            sb1.updateRecompilationFlag();
            // avoid later merge by other rewrites
            sb1.setSplitDag(true);
            // recursive application of rewrite rule (in case of multiple data dependent operators
            // with data dependencies in between each other)
            List<StatementBlock> tmp = rewriteStatementBlock(sb1, state);
            // add new statement blocks to output
            // statement block with data dependent hops
            ret.addAll(tmp);
            // statement block with remaining hops
            ret.add(sb);
            // avoid later merge by other rewrites
            sb.setSplitDag(true);
        } catch (Exception ex) {
            throw new HopsException("Failed to split hops dag for data dependent operators with unknown size.", ex);
        }
        LOG.debug("Applied splitDagDataDependentOperators (lines " + sb.getBeginLine() + "-" + sb.getEndLine() + ").");
    } else // keep original hop dag
    {
        ret.add(sb);
    }
    return ret;
}

Example 8 with VariableSet

use of org.apache.sysml.parser.VariableSet in project incubator-systemml by apache.

the class OptimizerRuleBased method rValidateUIPConsumerList.

/* 	
	 * This will validate candidate's consumer list.
	 * 
	 * @param pn:				OpNode of parfor loop
	 * @param uipCandHopHM:		Hashmap of UIPCandidateHop with name as a key.		
	 * @throws DMLRuntimeException
	 */
private void rValidateUIPConsumerList(OptNode pn, HashMap<String, ArrayList<UIPCandidateHop>> uipCandHopHM) throws DMLRuntimeException {
    if (!pn.isLeaf()) {
        if (pn.getNodeType() == OptNode.NodeType.FUNCCALL) {
            Hop hop = (Hop) OptTreeConverter.getAbstractPlanMapping().getMappedHop(pn.getID());
            rValidateUIPConsumerList(hop.getInput(), uipCandHopHM);
            return;
        }
        ProgramBlock pb = (ProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
        VariableSet varRead = pb.getStatementBlock().variablesRead();
        boolean bUIPCandHopRead = false;
        for (Entry<String, ArrayList<UIPCandidateHop>> entry : uipCandHopHM.entrySet()) {
            ArrayList<UIPCandidateHop> uipCandHopList = entry.getValue();
            if (uipCandHopList != null) {
                for (UIPCandidateHop uipCandHop : uipCandHopList) {
                    ArrayList<Hop> consumerHops = uipCandHop.getConsumerHops();
                    if (consumerHops != null) {
                        // remove candidate from the list.
                        for (Hop consumerHop : consumerHops) {
                            if (varRead.containsVariable(consumerHop.getName())) {
                                bUIPCandHopRead = true;
                                break;
                            }
                        }
                    }
                }
            }
        }
        // As none of the UIP candidates updated in this DAG, no need for further processing within this DAG
        if (!bUIPCandHopRead)
            return;
        for (OptNode optNode : pn.getChilds()) rValidateUIPConsumerList(optNode, uipCandHopHM);
    } else {
        OptTreePlanMappingAbstract map = OptTreeConverter.getAbstractPlanMapping();
        long ppid = map.getMappedParentID(map.getMappedParentID(pn.getID()));
        Object[] o = map.getMappedProg(ppid);
        ProgramBlock pb = (ProgramBlock) o[1];
        if (pb instanceof IfProgramBlock || pb instanceof WhileProgramBlock || //TODO
        (pb instanceof ForProgramBlock && !(pb instanceof ParForProgramBlock)))
            rValidateUIPConsumerList(pb, uipCandHopHM);
        long pid = map.getMappedParentID(pn.getID());
        o = map.getMappedProg(pid);
        pb = (ProgramBlock) o[1];
        Hop hop = map.getMappedHop(pn.getID());
        rValidateUIPConsumerList(hop, uipCandHopHM, pb.getStatementBlock().variablesRead());
    }
}

Example 9 with VariableSet

use of org.apache.sysml.parser.VariableSet in project incubator-systemml by apache.

the class OptimizerRuleBased method rGetUIPConsumerList.

/* 	
	 * This will get consumer list for candidate LeftIndexingOp.
	 * 
	 * @param pn:				OpNode of parfor loop
	 * @param uipCandHopHM:		Hashmap of UIPCandidateHop with name as a key.		
	 * @throws DMLRuntimeException
	 */
private void rGetUIPConsumerList(OptNode pn, HashMap<String, ArrayList<UIPCandidateHop>> uipCandHopHM) throws DMLRuntimeException {
    if (!pn.isLeaf()) {
        if (pn.getNodeType() == OptNode.NodeType.FUNCCALL)
            return;
        ProgramBlock pb = (ProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
        VariableSet varRead = pb.getStatementBlock().variablesRead();
        boolean bUIPCandHopRead = false;
        for (Entry<String, ArrayList<UIPCandidateHop>> entry : uipCandHopHM.entrySet()) {
            String uipCandHopID = entry.getKey();
            if (varRead.containsVariable(uipCandHopID)) {
                bUIPCandHopRead = true;
                break;
            }
        }
        // As none of the UIP candidates updated in this DAG, no need for further processing within this DAG
        if (!bUIPCandHopRead)
            return;
        for (OptNode optNode : pn.getChilds()) rGetUIPConsumerList(optNode, uipCandHopHM);
    } else {
        OptTreePlanMappingAbstract map = OptTreeConverter.getAbstractPlanMapping();
        long ppid = map.getMappedParentID(map.getMappedParentID(pn.getID()));
        Object[] o = map.getMappedProg(ppid);
        ProgramBlock pb = (ProgramBlock) o[1];
        Hop hop = (Hop) OptTreeConverter.getAbstractPlanMapping().getMappedHop(pn.getID());
        rGetUIPConsumerList(hop, uipCandHopHM);
        if (pb instanceof IfProgramBlock || pb instanceof WhileProgramBlock || //TODO
        (pb instanceof ForProgramBlock && !(pb instanceof ParForProgramBlock)))
            rGetUIPConsumerList(pb, uipCandHopHM);
    }
}