Examples with ParForStatement org.apache.sysml.parser.ParForStatement used on opensource projects

Example 1 with ParForStatement

use of org.apache.sysml.parser.ParForStatement in project incubator-systemml by apache.

the class PydmlSyntacticValidator method exitParForStatement.

@Override
public void exitParForStatement(ParForStatementContext ctx) {
    ParForStatement parForStmt = new ParForStatement();
    int line = ctx.start.getLine();
    int col = ctx.start.getCharPositionInLine();
    DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
    HashMap<String, String> parForParamValues = new HashMap<String, String>();
    if (ctx.parForParams != null && ctx.parForParams.size() > 0) {
        for (StrictParameterizedExpressionContext parForParamCtx : ctx.parForParams) {
            String paramVal = parForParamCtx.paramVal.getText();
            if (argVals.containsKey(paramVal))
                paramVal = argVals.get(paramVal);
            parForParamValues.put(parForParamCtx.paramName.getText(), paramVal);
        }
    }
    //1/-1
    Expression incrementExpr = null;
    if (ctx.iterPred.info.increment != null) {
        incrementExpr = ctx.iterPred.info.increment;
    }
    IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, currentFile, line, col, line, col);
    parForStmt.setPredicate(predicate);
    if (ctx.body.size() > 0) {
        for (StatementContext stmtCtx : ctx.body) {
            parForStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
        }
        parForStmt.mergeStatementBlocks();
    }
    ctx.info.stmt = parForStmt;
    setFileLineColumn(ctx.info.stmt, ctx);
}

Example 2 with ParForStatement

use of org.apache.sysml.parser.ParForStatement in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteInjectSparkLoopCheckpointing.

///////
//REWRITE inject spark loop checkpointing
///
protected void rewriteInjectSparkLoopCheckpointing(OptNode n) throws DMLRuntimeException {
    //get program blocks of root parfor
    Object[] progobj = OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID());
    ParForStatementBlock pfsb = (ParForStatementBlock) progobj[0];
    ParForStatement fs = (ParForStatement) pfsb.getStatement(0);
    ParForProgramBlock pfpb = (ParForProgramBlock) progobj[1];
    boolean applied = false;
    try {
        //apply hop rewrite inject spark checkpoints (but without context awareness)
        RewriteInjectSparkLoopCheckpointing rewrite = new RewriteInjectSparkLoopCheckpointing(false);
        ProgramRewriter rewriter = new ProgramRewriter(rewrite);
        ProgramRewriteStatus state = new ProgramRewriteStatus();
        rewriter.rewriteStatementBlockHopDAGs(pfsb, state);
        fs.setBody(rewriter.rewriteStatementBlocks(fs.getBody(), state));
        //recompile if additional checkpoints introduced
        if (state.getInjectedCheckpoints()) {
            pfpb.setChildBlocks(ProgramRecompiler.generatePartitialRuntimeProgram(pfpb.getProgram(), fs.getBody()));
            applied = true;
        }
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    LOG.debug(getOptMode() + " OPT: rewrite 'inject spark loop checkpointing' - result=" + applied);
}

Example 3 with ParForStatement

use of org.apache.sysml.parser.ParForStatement in project incubator-systemml by apache.

the class OptTreeConverter method rCreateAbstractOptNode.

public static OptNode rCreateAbstractOptNode(StatementBlock sb, ProgramBlock pb, LocalVariableMap vars, boolean topLevel, Set<String> memo) throws DMLRuntimeException, HopsException {
    OptNode node = null;
    if (pb instanceof IfProgramBlock && sb instanceof IfStatementBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement is = (IfStatement) isb.getStatement(0);
        node = new OptNode(NodeType.IF);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(isb.getBeginLine(), isb.getEndLine());
        //handle predicate
        isb.getPredicateHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(isb.getPredicateHops(), vars, memo));
        //process if branch
        OptNode ifn = new OptNode(NodeType.GENERIC);
        _hlMap.putProgMapping(sb, pb, ifn);
        ifn.setExecType(ExecType.CP);
        node.addChild(ifn);
        int len = is.getIfBody().size();
        for (int i = 0; i < ipb.getChildBlocksIfBody().size() && i < len; i++) {
            ProgramBlock lpb = ipb.getChildBlocksIfBody().get(i);
            StatementBlock lsb = is.getIfBody().get(i);
            ifn.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
        //process else branch
        if (ipb.getChildBlocksElseBody() != null) {
            OptNode efn = new OptNode(NodeType.GENERIC);
            _hlMap.putProgMapping(sb, pb, efn);
            efn.setExecType(ExecType.CP);
            node.addChild(efn);
            int len2 = is.getElseBody().size();
            for (int i = 0; i < ipb.getChildBlocksElseBody().size() && i < len2; i++) {
                ProgramBlock lpb = ipb.getChildBlocksElseBody().get(i);
                StatementBlock lsb = is.getElseBody().get(i);
                efn.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
            }
        }
    } else if (pb instanceof WhileProgramBlock && sb instanceof WhileStatementBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement ws = (WhileStatement) wsb.getStatement(0);
        node = new OptNode(NodeType.WHILE);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(wsb.getBeginLine(), wsb.getEndLine());
        //handle predicate
        wsb.getPredicateHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(wsb.getPredicateHops(), vars, memo));
        //process body
        int len = ws.getBody().size();
        for (int i = 0; i < wpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = wpb.getChildBlocks().get(i);
            StatementBlock lsb = ws.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
    } else if (pb instanceof ForProgramBlock && sb instanceof ForStatementBlock && !(pb instanceof ParForProgramBlock)) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock fsb = (ForStatementBlock) sb;
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        node = new OptNode(NodeType.FOR);
        _hlMap.putProgMapping(sb, pb, node);
        node.setExecType(ExecType.CP);
        node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
        node.addParam(ParamType.NUM_ITERATIONS, String.valueOf(CostEstimator.FACTOR_NUM_ITERATIONS));
        //handle predicate
        fsb.getFromHops().resetVisitStatus();
        fsb.getToHops().resetVisitStatus();
        if (fsb.getIncrementHops() != null)
            fsb.getIncrementHops().resetVisitStatus();
        node.addChilds(rCreateAbstractOptNodes(fsb.getFromHops(), vars, memo));
        node.addChilds(rCreateAbstractOptNodes(fsb.getToHops(), vars, memo));
        if (fsb.getIncrementHops() != null)
            node.addChilds(rCreateAbstractOptNodes(fsb.getIncrementHops(), vars, memo));
        //process body
        int len = fs.getBody().size();
        for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = fpb.getChildBlocks().get(i);
            StatementBlock lsb = fs.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
    } else if (pb instanceof ParForProgramBlock && sb instanceof ParForStatementBlock) {
        ParForProgramBlock fpb = (ParForProgramBlock) pb;
        ParForStatementBlock fsb = (ParForStatementBlock) sb;
        ParForStatement fs = (ParForStatement) fsb.getStatement(0);
        node = new OptNode(NodeType.PARFOR);
        node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
        _hlMap.putProgMapping(sb, pb, node);
        node.setK(fpb.getDegreeOfParallelism());
        long N = fpb.getNumIterations();
        node.addParam(ParamType.NUM_ITERATIONS, (N != -1) ? String.valueOf(N) : String.valueOf(CostEstimator.FACTOR_NUM_ITERATIONS));
        switch(fpb.getExecMode()) {
            case LOCAL:
                node.setExecType(ExecType.CP);
                break;
            case REMOTE_MR:
            case REMOTE_MR_DP:
                node.setExecType(ExecType.MR);
                break;
            case REMOTE_SPARK:
            case REMOTE_SPARK_DP:
                node.setExecType(ExecType.SPARK);
                break;
            case UNSPECIFIED:
                node.setExecType(null);
        }
        if (!topLevel) {
            fsb.getFromHops().resetVisitStatus();
            fsb.getToHops().resetVisitStatus();
            if (fsb.getIncrementHops() != null)
                fsb.getIncrementHops().resetVisitStatus();
            node.addChilds(rCreateAbstractOptNodes(fsb.getFromHops(), vars, memo));
            node.addChilds(rCreateAbstractOptNodes(fsb.getToHops(), vars, memo));
            if (fsb.getIncrementHops() != null)
                node.addChilds(rCreateAbstractOptNodes(fsb.getIncrementHops(), vars, memo));
        }
        //process body
        int len = fs.getBody().size();
        for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
            ProgramBlock lpb = fpb.getChildBlocks().get(i);
            StatementBlock lsb = fs.getBody().get(i);
            node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
        }
        //parameters, add required parameters
        Map<String, String> lparams = fpb.getParForParams();
        node.addParam(ParamType.DATA_PARTITIONER, lparams.get(ParForStatementBlock.DATA_PARTITIONER));
        node.addParam(ParamType.TASK_PARTITIONER, lparams.get(ParForStatementBlock.TASK_PARTITIONER));
        node.addParam(ParamType.RESULT_MERGE, lparams.get(ParForStatementBlock.RESULT_MERGE));
    //TODO task size
    } else //last level program block
    {
        sb = pb.getStatementBlock();
        //process all hops
        node = new OptNode(NodeType.GENERIC);
        _hlMap.putProgMapping(sb, pb, node);
        node.addChilds(createAbstractOptNodes(sb.get_hops(), vars, memo));
        node.setExecType(ExecType.CP);
        node.setLineNumbers(sb.getBeginLine(), sb.getEndLine());
        //TODO remove this workaround once this information can be obtained from hops/lops compiler
        if (node.isCPOnly()) {
            if (containsMRJobInstruction(pb, false, false))
                node.setExecType(ExecType.MR);
            else if (containsMRJobInstruction(pb, false, true))
                node.setExecType(ExecType.SPARK);
        }
    }
    //final cleanup
    //NOTE: required because this function is also used to create subtrees
    node.checkAndCleanupLeafNodes();
    return node;
}

Example 4 with ParForStatement

use of org.apache.sysml.parser.ParForStatement in project incubator-systemml by apache.

the class DmlSyntacticValidator method exitParForStatement.

@Override
public void exitParForStatement(ParForStatementContext ctx) {
    ParForStatement parForStmt = new ParForStatement();
    int line = ctx.start.getLine();
    int col = ctx.start.getCharPositionInLine();
    DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
    HashMap<String, String> parForParamValues = new HashMap<String, String>();
    if (ctx.parForParams != null && ctx.parForParams.size() > 0) {
        for (StrictParameterizedExpressionContext parForParamCtx : ctx.parForParams) {
            String paramVal = parForParamCtx.paramVal.getText();
            if (argVals.containsKey(paramVal))
                paramVal = argVals.get(paramVal);
            parForParamValues.put(parForParamCtx.paramName.getText(), paramVal);
        }
    }
    //1/-1
    Expression incrementExpr = null;
    if (ctx.iterPred.info.increment != null) {
        incrementExpr = ctx.iterPred.info.increment;
    }
    IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, currentFile, line, col, line, col);
    parForStmt.setPredicate(predicate);
    if (ctx.body.size() > 0) {
        for (StatementContext stmtCtx : ctx.body) {
            parForStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
        }
        parForStmt.mergeStatementBlocks();
    }
    ctx.info.stmt = parForStmt;
    setFileLineColumn(ctx.info.stmt, ctx);
}