Examples with StatementBlock org.apache.sysml.parser.StatementBlock used on opensource projects

Example 21 with StatementBlock

use of org.apache.sysml.parser.StatementBlock in project incubator-systemml by apache.

the class RewriteForLoopVectorization method vectorizeElementwiseBinary.

private static StatementBlock vectorizeElementwiseBinary(StatementBlock sb, StatementBlock csb, Hop from, Hop to, Hop increment, String itervar) {
    StatementBlock ret = sb;
    // check supported increment values
    if (!(increment instanceof LiteralOp && ((LiteralOp) increment).getDoubleValue() == 1.0)) {
        return ret;
    }
    // check for applicability
    boolean apply = false;
    // row or col
    boolean rowIx = false;
    if (csb.getHops() != null && csb.getHops().size() == 1) {
        Hop root = csb.getHops().get(0);
        if (root.getDataType() == DataType.MATRIX && root.getInput().get(0) instanceof LeftIndexingOp) {
            LeftIndexingOp lix = (LeftIndexingOp) root.getInput().get(0);
            Hop lixlhs = lix.getInput().get(0);
            Hop lixrhs = lix.getInput().get(1);
            if (lixlhs instanceof DataOp && lixrhs instanceof BinaryOp && lixrhs.getInput().get(0) instanceof IndexingOp && lixrhs.getInput().get(1) instanceof IndexingOp && lixrhs.getInput().get(0).getInput().get(0) instanceof DataOp && lixrhs.getInput().get(1).getInput().get(0) instanceof DataOp) {
                IndexingOp rix0 = (IndexingOp) lixrhs.getInput().get(0);
                IndexingOp rix1 = (IndexingOp) lixrhs.getInput().get(1);
                // check for rowwise
                if (lix.isRowLowerEqualsUpper() && rix0.isRowLowerEqualsUpper() && rix1.isRowLowerEqualsUpper() && lix.getInput().get(2).getName().equals(itervar) && rix0.getInput().get(1).getName().equals(itervar) && rix1.getInput().get(1).getName().equals(itervar)) {
                    apply = true;
                    rowIx = true;
                }
                // check for colwise
                if (lix.isColLowerEqualsUpper() && rix0.isColLowerEqualsUpper() && rix1.isColLowerEqualsUpper() && lix.getInput().get(4).getName().equals(itervar) && rix0.getInput().get(3).getName().equals(itervar) && rix1.getInput().get(3).getName().equals(itervar)) {
                    apply = true;
                    rowIx = false;
                }
            }
        }
    }
    // apply rewrite if possible
    if (apply) {
        Hop root = csb.getHops().get(0);
        LeftIndexingOp lix = (LeftIndexingOp) root.getInput().get(0);
        BinaryOp bop = (BinaryOp) lix.getInput().get(1);
        IndexingOp rix0 = (IndexingOp) bop.getInput().get(0);
        IndexingOp rix1 = (IndexingOp) bop.getInput().get(1);
        int index1 = rowIx ? 2 : 4;
        int index2 = rowIx ? 3 : 5;
        // modify left indexing bounds
        HopRewriteUtils.replaceChildReference(lix, lix.getInput().get(index1), from, index1);
        HopRewriteUtils.replaceChildReference(lix, lix.getInput().get(index2), to, index2);
        // modify both right indexing
        HopRewriteUtils.replaceChildReference(rix0, rix0.getInput().get(index1 - 1), from, index1 - 1);
        HopRewriteUtils.replaceChildReference(rix0, rix0.getInput().get(index2 - 1), to, index2 - 1);
        HopRewriteUtils.replaceChildReference(rix1, rix1.getInput().get(index1 - 1), from, index1 - 1);
        HopRewriteUtils.replaceChildReference(rix1, rix1.getInput().get(index2 - 1), to, index2 - 1);
        updateLeftAndRightIndexingSizes(rowIx, lix, rix0, rix1);
        bop.refreshSizeInformation();
        // after bop update
        lix.refreshSizeInformation();
        ret = csb;
        // ret.liveIn().removeVariable(itervar);
        LOG.debug("Applied vectorizeElementwiseBinaryForLoop.");
    }
    return ret;
}

Example 22 with StatementBlock

use of org.apache.sysml.parser.StatementBlock in project incubator-systemml by apache.

the class RewriteInjectSparkLoopCheckpointing method rewriteStatementBlock.

@Override
public List<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus status) {
    if (!OptimizerUtils.isSparkExecutionMode()) {
        // nothing to do here, return original statement block
        return Arrays.asList(sb);
    }
    // 1) We currently add checkpoint operations without information about the global program structure,
    // this assumes that redundant checkpointing is prevented at runtime level (instruction-level)
    // 2) Also, we do not take size information into account right now. This means that all candidates
    // are checkpointed even if they are only used by CP operations.
    ArrayList<StatementBlock> ret = new ArrayList<>();
    // block size set by reblock rewrite
    int blocksize = status.getBlocksize();
    // optimization because otherwise we would prevent remote parfor)
    if (// incl parfor
    (sb instanceof WhileStatementBlock || sb instanceof ForStatementBlock) && (_checkCtx ? !status.isInParforContext() : true)) {
        // step 1: determine checkpointing candidates
        ArrayList<String> candidates = new ArrayList<>();
        VariableSet read = sb.variablesRead();
        VariableSet updated = sb.variablesUpdated();
        for (String rvar : read.getVariableNames()) if (!updated.containsVariable(rvar) && read.getVariable(rvar).getDataType() == DataType.MATRIX)
            candidates.add(rvar);
        // step 2: insert statement block with checkpointing operations
        if (// existing candidates
        !candidates.isEmpty()) {
            StatementBlock sb0 = new StatementBlock();
            sb0.setDMLProg(sb.getDMLProg());
            sb0.setParseInfo(sb);
            ArrayList<Hop> hops = new ArrayList<>();
            VariableSet livein = new VariableSet();
            VariableSet liveout = new VariableSet();
            for (String var : candidates) {
                DataIdentifier dat = read.getVariable(var);
                long dim1 = (dat instanceof IndexedIdentifier) ? ((IndexedIdentifier) dat).getOrigDim1() : dat.getDim1();
                long dim2 = (dat instanceof IndexedIdentifier) ? ((IndexedIdentifier) dat).getOrigDim2() : dat.getDim2();
                DataOp tread = new DataOp(var, DataType.MATRIX, ValueType.DOUBLE, DataOpTypes.TRANSIENTREAD, dat.getFilename(), dim1, dim2, dat.getNnz(), blocksize, blocksize);
                tread.setRequiresCheckpoint(true);
                DataOp twrite = new DataOp(var, DataType.MATRIX, ValueType.DOUBLE, tread, DataOpTypes.TRANSIENTWRITE, null);
                HopRewriteUtils.setOutputParameters(twrite, dim1, dim2, blocksize, blocksize, dat.getNnz());
                hops.add(twrite);
                livein.addVariable(var, read.getVariable(var));
                liveout.addVariable(var, read.getVariable(var));
            }
            sb0.setHops(hops);
            sb0.setLiveIn(livein);
            sb0.setLiveOut(liveout);
            sb0.setSplitDag(true);
            ret.add(sb0);
            // maintain rewrite status
            status.setInjectedCheckpoints();
        }
    }
    // add original statement block to end
    ret.add(sb);
    return ret;
}

Example 23 with StatementBlock

use of org.apache.sysml.parser.StatementBlock in project incubator-systemml by apache.

the class RewriteSplitDagUnknownCSVRead method rewriteStatementBlock.

@Override
public List<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus state) {
    // DAG splits not required for forced single node
    if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE || !HopRewriteUtils.isLastLevelStatementBlock(sb))
        return Arrays.asList(sb);
    ArrayList<StatementBlock> ret = new ArrayList<>();
    // collect all unknown csv reads hops
    ArrayList<Hop> cand = new ArrayList<>();
    collectCSVReadHopsUnknownSize(sb.getHops(), cand);
    // split hop dag on demand
    if (!cand.isEmpty()) {
        try {
            // duplicate sb incl live variable sets
            StatementBlock sb1 = new StatementBlock();
            sb1.setDMLProg(sb.getDMLProg());
            sb1.setParseInfo(sb);
            sb1.setLiveIn(new VariableSet());
            sb1.setLiveOut(new VariableSet());
            // move csv reads incl reblock to new statement block
            // (and replace original persistent read with transient read)
            ArrayList<Hop> sb1hops = new ArrayList<>();
            for (Hop reblock : cand) {
                long rlen = reblock.getDim1();
                long clen = reblock.getDim2();
                long nnz = reblock.getNnz();
                UpdateType update = reblock.getUpdateType();
                int brlen = reblock.getRowsInBlock();
                int bclen = reblock.getColsInBlock();
                // (otherwise, for instance, literal ops are shared across dags)
                for (int i = 0; i < reblock.getInput().size(); i++) if (reblock.getInput().get(i) instanceof LiteralOp)
                    HopRewriteUtils.replaceChildReference(reblock, reblock.getInput().get(i), new LiteralOp((LiteralOp) reblock.getInput().get(i)));
                // create new transient read
                DataOp tread = new DataOp(reblock.getName(), reblock.getDataType(), reblock.getValueType(), DataOpTypes.TRANSIENTREAD, null, rlen, clen, nnz, update, brlen, bclen);
                HopRewriteUtils.copyLineNumbers(reblock, tread);
                // replace reblock with transient read
                ArrayList<Hop> parents = new ArrayList<>(reblock.getParent());
                for (int i = 0; i < parents.size(); i++) {
                    Hop parent = parents.get(i);
                    HopRewriteUtils.replaceChildReference(parent, reblock, tread);
                }
                // add reblock sub dag to first statement block
                DataOp twrite = new DataOp(reblock.getName(), reblock.getDataType(), reblock.getValueType(), reblock, DataOpTypes.TRANSIENTWRITE, null);
                twrite.setOutputParams(rlen, clen, nnz, update, brlen, bclen);
                HopRewriteUtils.copyLineNumbers(reblock, twrite);
                sb1hops.add(twrite);
                // update live in and out of new statement block (for piggybacking)
                DataIdentifier diVar = sb.variablesRead().getVariable(reblock.getName());
                if (diVar != null) {
                    // var read should always exist because persistent read
                    sb1.liveOut().addVariable(reblock.getName(), new DataIdentifier(diVar));
                    sb.liveIn().addVariable(reblock.getName(), new DataIdentifier(diVar));
                }
            }
            sb1.setHops(sb1hops);
            sb1.updateRecompilationFlag();
            // statement block with csv reblocks
            ret.add(sb1);
            // statement block with remaining hops
            ret.add(sb);
            // avoid later merge by other rewrites
            sb.setSplitDag(true);
        } catch (Exception ex) {
            throw new HopsException("Failed to split hops dag for csv read with unknown size.", ex);
        }
        LOG.debug("Applied splitDagUnknownCSVRead.");
    } else // keep original hop dag
    {
        ret.add(sb);
    }
    return ret;
}

Example 24 with StatementBlock

use of org.apache.sysml.parser.StatementBlock in project incubator-systemml by apache.

the class Explain method getHopDAG.

public static String getHopDAG(DMLProgram prog, ArrayList<Integer> lines, boolean withSubgraph) {
    StringBuilder sb = new StringBuilder();
    StringBuilder nodes = new StringBuilder();
    // create header
    sb.append("digraph {");
    // Explain functions (if exists)
    if (prog.hasFunctionStatementBlocks()) {
        // show individual functions
        for (String namespace : prog.getNamespaces().keySet()) {
            for (String fname : prog.getFunctionStatementBlocks(namespace).keySet()) {
                FunctionStatementBlock fsb = prog.getFunctionStatementBlock(namespace, fname);
                FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
                String fkey = DMLProgram.constructFunctionKey(namespace, fname);
                if (!(fstmt instanceof ExternalFunctionStatement)) {
                    addSubGraphHeader(sb, withSubgraph);
                    for (StatementBlock current : fstmt.getBody()) sb.append(getHopDAG(current, nodes, lines, withSubgraph));
                    String label = "FUNCTION " + fkey + " recompile=" + fsb.isRecompileOnce() + "\n";
                    addSubGraphFooter(sb, withSubgraph, label);
                }
            }
        }
    }
    // Explain main program
    for (StatementBlock sblk : prog.getStatementBlocks()) sb.append(getHopDAG(sblk, nodes, lines, withSubgraph));
    sb.append(nodes);
    sb.append("rankdir = \"BT\"\n");
    sb.append("}\n");
    return sb.toString();
}

Example 25 with StatementBlock

use of org.apache.sysml.parser.StatementBlock in project incubator-systemml by apache.

the class Explain method explainProgramBlock.

// ////////////
// internal explain RUNTIME
private static String explainProgramBlock(ProgramBlock pb, int level) {
    StringBuilder sb = new StringBuilder();
    String offset = createOffset(level);
    if (pb instanceof FunctionProgramBlock) {
        FunctionProgramBlock fpb = (FunctionProgramBlock) pb;
        for (ProgramBlock pbc : fpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
    } else if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        StatementBlock wsb = pb.getStatementBlock();
        sb.append(offset);
        if (wsb != null && !wsb.getUpdateInPlaceVars().isEmpty())
            sb.append("WHILE (lines " + wpb.getBeginLine() + "-" + wpb.getEndLine() + ") [in-place=" + wsb.getUpdateInPlaceVars().toString() + "]\n");
        else
            sb.append("WHILE (lines " + wpb.getBeginLine() + "-" + wpb.getEndLine() + ")\n");
        sb.append(explainInstructions(wpb.getPredicate(), level + 1));
        for (ProgramBlock pbc : wpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        sb.append(offset);
        sb.append("IF (lines " + ipb.getBeginLine() + "-" + ipb.getEndLine() + ")\n");
        sb.append(explainInstructions(ipb.getPredicate(), level + 1));
        for (ProgramBlock pbc : ipb.getChildBlocksIfBody()) sb.append(explainProgramBlock(pbc, level + 1));
        if (!ipb.getChildBlocksElseBody().isEmpty()) {
            sb.append(offset);
            sb.append("ELSE\n");
            for (ProgramBlock pbc : ipb.getChildBlocksElseBody()) sb.append(explainProgramBlock(pbc, level + 1));
        }
    } else if (// incl parfor
    pb instanceof ForProgramBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        StatementBlock fsb = pb.getStatementBlock();
        sb.append(offset);
        if (pb instanceof ParForProgramBlock)
            sb.append("PARFOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ")\n");
        else {
            if (fsb != null && !fsb.getUpdateInPlaceVars().isEmpty())
                sb.append("FOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ") [in-place=" + fsb.getUpdateInPlaceVars().toString() + "]\n");
            else
                sb.append("FOR (lines " + fpb.getBeginLine() + "-" + fpb.getEndLine() + ")\n");
        }
        sb.append(explainInstructions(fpb.getFromInstructions(), level + 1));
        sb.append(explainInstructions(fpb.getToInstructions(), level + 1));
        sb.append(explainInstructions(fpb.getIncrementInstructions(), level + 1));
        for (ProgramBlock pbc : fpb.getChildBlocks()) sb.append(explainProgramBlock(pbc, level + 1));
    } else {
        sb.append(offset);
        if (pb.getStatementBlock() != null)
            sb.append("GENERIC (lines " + pb.getBeginLine() + "-" + pb.getEndLine() + ") [recompile=" + pb.getStatementBlock().requiresRecompilation() + "]\n");
        else
            sb.append("GENERIC (lines " + pb.getBeginLine() + "-" + pb.getEndLine() + ") \n");
        sb.append(explainInstructions(pb.getInstructions(), level + 1));
    }
    return sb.toString();
}