Examples with DataOp org.apache.sysml.hops.DataOp used on opensource projects

Example 86 with DataOp

use of org.apache.sysml.hops.DataOp in project systemml by apache.

the class RewriteSplitDagUnknownCSVRead method rewriteStatementBlock.

@Override
public List<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus state) {
    // DAG splits not required for forced single node
    if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE || !HopRewriteUtils.isLastLevelStatementBlock(sb))
        return Arrays.asList(sb);
    ArrayList<StatementBlock> ret = new ArrayList<>();
    // collect all unknown csv reads hops
    ArrayList<Hop> cand = new ArrayList<>();
    collectCSVReadHopsUnknownSize(sb.getHops(), cand);
    // split hop dag on demand
    if (!cand.isEmpty()) {
        try {
            // duplicate sb incl live variable sets
            StatementBlock sb1 = new StatementBlock();
            sb1.setDMLProg(sb.getDMLProg());
            sb1.setParseInfo(sb);
            sb1.setLiveIn(new VariableSet());
            sb1.setLiveOut(new VariableSet());
            // move csv reads incl reblock to new statement block
            // (and replace original persistent read with transient read)
            ArrayList<Hop> sb1hops = new ArrayList<>();
            for (Hop reblock : cand) {
                long rlen = reblock.getDim1();
                long clen = reblock.getDim2();
                long nnz = reblock.getNnz();
                UpdateType update = reblock.getUpdateType();
                int brlen = reblock.getRowsInBlock();
                int bclen = reblock.getColsInBlock();
                // (otherwise, for instance, literal ops are shared across dags)
                for (int i = 0; i < reblock.getInput().size(); i++) if (reblock.getInput().get(i) instanceof LiteralOp)
                    HopRewriteUtils.replaceChildReference(reblock, reblock.getInput().get(i), new LiteralOp((LiteralOp) reblock.getInput().get(i)));
                // create new transient read
                DataOp tread = new DataOp(reblock.getName(), reblock.getDataType(), reblock.getValueType(), DataOpTypes.TRANSIENTREAD, null, rlen, clen, nnz, update, brlen, bclen);
                HopRewriteUtils.copyLineNumbers(reblock, tread);
                // replace reblock with transient read
                ArrayList<Hop> parents = new ArrayList<>(reblock.getParent());
                for (int i = 0; i < parents.size(); i++) {
                    Hop parent = parents.get(i);
                    HopRewriteUtils.replaceChildReference(parent, reblock, tread);
                }
                // add reblock sub dag to first statement block
                DataOp twrite = new DataOp(reblock.getName(), reblock.getDataType(), reblock.getValueType(), reblock, DataOpTypes.TRANSIENTWRITE, null);
                twrite.setOutputParams(rlen, clen, nnz, update, brlen, bclen);
                HopRewriteUtils.copyLineNumbers(reblock, twrite);
                sb1hops.add(twrite);
                // update live in and out of new statement block (for piggybacking)
                DataIdentifier diVar = sb.variablesRead().getVariable(reblock.getName());
                if (diVar != null) {
                    // var read should always exist because persistent read
                    sb1.liveOut().addVariable(reblock.getName(), new DataIdentifier(diVar));
                    sb.liveIn().addVariable(reblock.getName(), new DataIdentifier(diVar));
                }
            }
            sb1.setHops(sb1hops);
            sb1.updateRecompilationFlag();
            // statement block with csv reblocks
            ret.add(sb1);
            // statement block with remaining hops
            ret.add(sb);
            // avoid later merge by other rewrites
            sb.setSplitDag(true);
        } catch (Exception ex) {
            throw new HopsException("Failed to split hops dag for csv read with unknown size.", ex);
        }
        LOG.debug("Applied splitDagUnknownCSVRead.");
    } else // keep original hop dag
    {
        ret.add(sb);
    }
    return ret;
}