Examples with HopsException org.apache.sysml.hops.HopsException used on opensource projects

Example 21 with HopsException

use of org.apache.sysml.hops.HopsException in project incubator-systemml by apache.

the class RewriteSplitDagDataDependentOperators method rewriteStatementBlock.

@Override
public List<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus state) {
    // DAG splits not required for forced single node
    if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE || !HopRewriteUtils.isLastLevelStatementBlock(sb))
        return Arrays.asList(sb);
    ArrayList<StatementBlock> ret = new ArrayList<>();
    // collect all unknown csv reads hops
    ArrayList<Hop> cand = new ArrayList<>();
    collectDataDependentOperators(sb.getHops(), cand);
    Hop.resetVisitStatus(sb.getHops());
    // split hop dag on demand
    if (!cand.isEmpty()) {
        // collect child operators of candidates (to prevent rewrite anomalies)
        HashSet<Hop> candChilds = new HashSet<>();
        collectCandidateChildOperators(cand, candChilds);
        try {
            // duplicate sb incl live variable sets
            StatementBlock sb1 = new StatementBlock();
            sb1.setDMLProg(sb.getDMLProg());
            sb1.setParseInfo(sb);
            sb1.setLiveIn(new VariableSet());
            sb1.setLiveOut(new VariableSet());
            // move data-dependent ops incl transient writes to new statement block
            // (and replace original persistent read with transient read)
            ArrayList<Hop> sb1hops = new ArrayList<>();
            for (Hop c : cand) {
                // if there are already transient writes use them and don't introduce artificial variables;
                // unless there are transient reads w/ the same variable name in the current dag which can
                // lead to invalid reordering if variable consumers are not feeding into the candidate op.
                boolean hasTWrites = hasTransientWriteParents(c);
                boolean moveTWrite = hasTWrites ? HopRewriteUtils.rHasSimpleReadChain(c, getFirstTransientWriteParent(c).getName()) : false;
                String varname = null;
                long rlen = c.getDim1();
                long clen = c.getDim2();
                long nnz = c.getNnz();
                UpdateType update = c.getUpdateType();
                int brlen = c.getRowsInBlock();
                int bclen = c.getColsInBlock();
                if (// reuse existing transient_write
                hasTWrites && moveTWrite) {
                    Hop twrite = getFirstTransientWriteParent(c);
                    varname = twrite.getName();
                    // create new transient read
                    DataOp tread = new DataOp(varname, c.getDataType(), c.getValueType(), DataOpTypes.TRANSIENTREAD, null, rlen, clen, nnz, update, brlen, bclen);
                    tread.setVisited();
                    HopRewriteUtils.copyLineNumbers(c, tread);
                    // replace data-dependent operator with transient read
                    ArrayList<Hop> parents = new ArrayList<>(c.getParent());
                    for (int i = 0; i < parents.size(); i++) {
                        // prevent concurrent modification by index access
                        Hop parent = parents.get(i);
                        if (!candChilds.contains(parent)) {
                            // anomaly filter
                            if (parent != twrite)
                                HopRewriteUtils.replaceChildReference(parent, c, tread);
                            else
                                sb.getHops().remove(parent);
                        }
                    }
                    // add data-dependent operator sub dag to first statement block
                    sb1hops.add(twrite);
                } else // create transient write to artificial variables
                {
                    varname = createCutVarName(false);
                    // create new transient read
                    DataOp tread = new DataOp(varname, c.getDataType(), c.getValueType(), DataOpTypes.TRANSIENTREAD, null, rlen, clen, nnz, update, brlen, bclen);
                    tread.setVisited();
                    HopRewriteUtils.copyLineNumbers(c, tread);
                    // replace data-dependent operator with transient read
                    ArrayList<Hop> parents = new ArrayList<>(c.getParent());
                    for (int i = 0; i < parents.size(); i++) {
                        // prevent concurrent modification by index access
                        Hop parent = parents.get(i);
                        if (// anomaly filter
                        !candChilds.contains(parent))
                            HopRewriteUtils.replaceChildReference(parent, c, tread);
                    }
                    // add data-dependent operator sub dag to first statement block
                    DataOp twrite = new DataOp(varname, c.getDataType(), c.getValueType(), c, DataOpTypes.TRANSIENTWRITE, null);
                    twrite.setVisited();
                    twrite.setOutputParams(rlen, clen, nnz, update, brlen, bclen);
                    HopRewriteUtils.copyLineNumbers(c, twrite);
                    sb1hops.add(twrite);
                }
                // update live in and out of new statement block (for piggybacking)
                DataIdentifier diVar = new DataIdentifier(varname);
                diVar.setDimensions(rlen, clen);
                diVar.setBlockDimensions(brlen, bclen);
                diVar.setDataType(c.getDataType());
                diVar.setValueType(c.getValueType());
                sb1.liveOut().addVariable(varname, new DataIdentifier(diVar));
                sb.liveIn().addVariable(varname, new DataIdentifier(diVar));
            }
            // ensure disjoint operators across DAGs (prevent replicated operations)
            handleReplicatedOperators(sb1hops, sb.getHops(), sb1.liveOut(), sb.liveIn());
            // deep copy new dag (in order to prevent any dangling references)
            sb1.setHops(Recompiler.deepCopyHopsDag(sb1hops));
            sb1.updateRecompilationFlag();
            // avoid later merge by other rewrites
            sb1.setSplitDag(true);
            // recursive application of rewrite rule (in case of multiple data dependent operators
            // with data dependencies in between each other)
            List<StatementBlock> tmp = rewriteStatementBlock(sb1, state);
            // add new statement blocks to output
            // statement block with data dependent hops
            ret.addAll(tmp);
            // statement block with remaining hops
            ret.add(sb);
            // avoid later merge by other rewrites
            sb.setSplitDag(true);
        } catch (Exception ex) {
            throw new HopsException("Failed to split hops dag for data dependent operators with unknown size.", ex);
        }
        LOG.debug("Applied splitDagDataDependentOperators (lines " + sb.getBeginLine() + "-" + sb.getEndLine() + ").");
    } else // keep original hop dag
    {
        ret.add(sb);
    }
    return ret;
}

Example 22 with HopsException

use of org.apache.sysml.hops.HopsException in project incubator-systemml by apache.

the class InterProceduralAnalysis method extractFunctionCallReturnStatistics.

/**
 * Extract return variable statistics from this function into the
 * calling program.
 *
 * @param fstmt  The function statement.
 * @param fop  The function op.
 * @param tmpVars  Function's map of variables eligible for
 *                    extraction.
 * @param callVars  Calling program's map of variables.
 * @param overwrite  Whether or not to overwrite variables in the
 *                      calling program's variable map.
 */
private static void extractFunctionCallReturnStatistics(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap tmpVars, LocalVariableMap callVars, boolean overwrite) {
    ArrayList<DataIdentifier> foutputOps = fstmt.getOutputParams();
    String[] outputVars = fop.getOutputVariableNames();
    String fkey = fop.getFunctionKey();
    try {
        for (int i = 0; i < foutputOps.size(); i++) {
            DataIdentifier di = foutputOps.get(i);
            // name in function signature
            String fvarname = di.getName();
            // name in calling program
            String pvarname = outputVars[i];
            // output, remove that variable from the calling program's variable map.
            if (callVars.keySet().contains(pvarname)) {
                DataType fdataType = di.getDataType();
                DataType pdataType = callVars.get(pvarname).getDataType();
                if (fdataType != pdataType) {
                    // datatype has changed, and the calling program is reassigning the
                    // the variable, so remove it from the calling variable map
                    callVars.remove(pvarname);
                }
            }
            // Update or add to the calling program's variable map.
            if (di.getDataType() == DataType.MATRIX && tmpVars.keySet().contains(fvarname)) {
                MatrixObject moIn = (MatrixObject) tmpVars.get(fvarname);
                if (// not existing so far
                !callVars.keySet().contains(pvarname) || overwrite) {
                    MatrixObject moOut = createOutputMatrix(moIn.getNumRows(), moIn.getNumColumns(), moIn.getNnz());
                    callVars.put(pvarname, moOut);
                } else // already existing: take largest
                {
                    Data dat = callVars.get(pvarname);
                    if (dat instanceof MatrixObject) {
                        MatrixObject moOut = (MatrixObject) dat;
                        MatrixCharacteristics mc = moOut.getMatrixCharacteristics();
                        if (OptimizerUtils.estimateSizeExactSparsity(mc.getRows(), mc.getCols(), (mc.getNonZeros() > 0) ? OptimizerUtils.getSparsity(mc) : 1.0) < OptimizerUtils.estimateSize(moIn.getNumRows(), moIn.getNumColumns())) {
                            // update statistics if necessary
                            mc.setDimension(moIn.getNumRows(), moIn.getNumColumns());
                            mc.setNonZeros(moIn.getNnz());
                        }
                    }
                }
            }
        }
    } catch (Exception ex) {
        throw new HopsException("Failed to extract output statistics of function " + fkey + ".", ex);
    }
}

Example 23 with HopsException

use of org.apache.sysml.hops.HopsException in project incubator-systemml by apache.

the class InterProceduralAnalysis method extractFunctionCallEquivalentReturnStatistics.

private static void extractFunctionCallEquivalentReturnStatistics(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callVars) {
    try {
        Hop input = fop.getInput().get(0);
        MatrixObject moOut = createOutputMatrix(input.getDim1(), input.getDim2(), -1);
        callVars.put(fop.getOutputVariableNames()[0], moOut);
    } catch (Exception ex) {
        throw new HopsException("Failed to extract output statistics " + "for unary function " + fop.getFunctionKey() + ".", ex);
    }
}

Example 24 with HopsException

use of org.apache.sysml.hops.HopsException in project incubator-systemml by apache.

the class LiteralReplacement method getIntValueDataLiteral.

private static long getIntValueDataLiteral(Hop hop, LocalVariableMap vars) {
    long value = -1;
    try {
        if (hop instanceof LiteralOp) {
            value = HopRewriteUtils.getIntValue((LiteralOp) hop);
        } else if (hop instanceof UnaryOp && ((UnaryOp) hop).getOp() == OpOp1.NROW) {
            // get the dimension information from the matrix object because the hop
            // dimensions might not have been updated during recompile
            MatrixObject mo = (MatrixObject) vars.get(hop.getInput().get(0).getName());
            value = mo.getNumRows();
        } else if (hop instanceof UnaryOp && ((UnaryOp) hop).getOp() == OpOp1.NCOL) {
            // get the dimension information from the matrix object because the hop
            // dimensions might not have been updated during recompile
            MatrixObject mo = (MatrixObject) vars.get(hop.getInput().get(0).getName());
            value = mo.getNumColumns();
        } else {
            ScalarObject sdat = (ScalarObject) vars.get(hop.getName());
            value = sdat.getLongValue();
        }
    } catch (HopsException ex) {
        throw new DMLRuntimeException("Failed to get int value for literal replacement", ex);
    }
    return value;
}

Example 25 with HopsException

use of org.apache.sysml.hops.HopsException in project incubator-systemml by apache.

the class OptTreeConverter method createAbstractOptTree.

public static OptTree createAbstractOptTree(int ck, double cm, ParForStatementBlock pfsb, ParForProgramBlock pfpb, Set<String> memo, ExecutionContext ec) {
    OptTree tree = null;
    OptNode root = null;
    try {
        root = rCreateAbstractOptNode(pfsb, pfpb, ec.getVariables(), true, memo);
        tree = new OptTree(ck, cm, root);
    } catch (HopsException he) {
        throw new DMLRuntimeException(he);
    }
    return tree;
}