Examples with Hop org.apache.sysml.hops.Hop used on opensource projects

Example 51 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class IPAPassRemoveConstantBinaryOps method rRemoveConstantBinaryOp.

private static void rRemoveConstantBinaryOp(Hop hop, HashMap<String, Hop> mOnes) {
    if (hop.isVisited())
        return;
    if (hop instanceof BinaryOp && ((BinaryOp) hop).getOp() == OpOp2.MULT && !((BinaryOp) hop).isOuterVectorOperator() && hop.getInput().get(0).getDataType() == DataType.MATRIX && hop.getInput().get(1) instanceof DataOp && mOnes.containsKey(hop.getInput().get(1).getName())) {
        // replace matrix of ones with literal 1 (later on removed by
        // algebraic simplification rewrites; otherwise more complex
        // recursive processing of childs and rewiring required)
        HopRewriteUtils.removeChildReferenceByPos(hop, hop.getInput().get(1), 1);
        HopRewriteUtils.addChildReference(hop, new LiteralOp(1), 1);
    }
    // recursively process child nodes
    for (Hop c : hop.getInput()) rRemoveConstantBinaryOp(c, mOnes);
    hop.setVisited();
}

Example 52 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class IPAPassRemoveUnnecessaryCheckpoints method removeCheckpointBeforeUpdate.

private static void removeCheckpointBeforeUpdate(DMLProgram dmlp) {
    // approach: scan over top-level program (guaranteed to be unconditional),
    // collect checkpoints; determine if used before update; remove first checkpoint
    // on second checkpoint if update in between and not used before update
    HashMap<String, Hop> chkpointCand = new HashMap<>();
    for (StatementBlock sb : dmlp.getStatementBlocks()) {
        // prune candidates (used before updated)
        Set<String> cands = new HashSet<>(chkpointCand.keySet());
        for (String cand : cands) if (sb.variablesRead().containsVariable(cand) && !sb.variablesUpdated().containsVariable(cand)) {
            // note: variableRead might include false positives due to meta
            // data operations like nrow(X) or operations removed by rewrites
            // double check hops on basic blocks; otherwise worst-case
            boolean skipRemove = false;
            if (sb.getHops() != null) {
                Hop.resetVisitStatus(sb.getHops());
                skipRemove = true;
                for (Hop root : sb.getHops()) skipRemove &= !HopRewriteUtils.rContainsRead(root, cand, false);
            }
            if (!skipRemove)
                chkpointCand.remove(cand);
        }
        // prune candidates (updated in conditional control flow)
        Set<String> cands2 = new HashSet<>(chkpointCand.keySet());
        if (sb instanceof IfStatementBlock || sb instanceof WhileStatementBlock || sb instanceof ForStatementBlock) {
            for (String cand : cands2) if (sb.variablesUpdated().containsVariable(cand)) {
                chkpointCand.remove(cand);
            }
        } else // prune candidates (updated w/ multiple reads)
        {
            for (String cand : cands2) if (sb.variablesUpdated().containsVariable(cand) && sb.getHops() != null) {
                Hop.resetVisitStatus(sb.getHops());
                for (Hop root : sb.getHops()) if (root.getName().equals(cand) && !HopRewriteUtils.rHasSimpleReadChain(root, cand)) {
                    chkpointCand.remove(cand);
                }
            }
        }
        // collect checkpoints and remove unnecessary checkpoints
        if (HopRewriteUtils.isLastLevelStatementBlock(sb)) {
            ArrayList<Hop> tmp = collectCheckpoints(sb.getHops());
            for (Hop chkpoint : tmp) {
                if (chkpointCand.containsKey(chkpoint.getName())) {
                    chkpointCand.get(chkpoint.getName()).setRequiresCheckpoint(false);
                }
                chkpointCand.put(chkpoint.getName(), chkpoint);
            }
        }
    }
}

Example 53 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class InterProceduralAnalysis method propagateStatisticsIntoFunctions.

/**
 * Propagate statistics from the calling program into a function
 * block.
 *
 * @param prog  The DML program.
 * @param hop HOP to propagate statistics into.
 * @param callVars  Calling program's map of variables eligible for propagation.
 * @param fcallSizes function call summary
 * @param fnStack  Function stack to determine current scope.
 */
private void propagateStatisticsIntoFunctions(DMLProgram prog, Hop hop, LocalVariableMap callVars, FunctionCallSizeInfo fcallSizes, Set<String> fnStack) {
    if (hop.isVisited())
        return;
    for (Hop c : hop.getInput()) propagateStatisticsIntoFunctions(prog, c, callVars, fcallSizes, fnStack);
    if (hop instanceof FunctionOp) {
        // maintain counters and investigate functions if not seen so far
        FunctionOp fop = (FunctionOp) hop;
        String fkey = fop.getFunctionKey();
        if (fop.getFunctionType() == FunctionType.DML) {
            FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
            FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
            if (fcallSizes.isValidFunction(fkey) && // prevent recursion
            !fnStack.contains(fkey)) {
                // maintain function call stack
                fnStack.add(fkey);
                // create mapping and populate symbol table for refresh
                LocalVariableMap tmpVars = new LocalVariableMap();
                populateLocalVariableMapForFunctionCall(fstmt, fop, callVars, tmpVars, fcallSizes);
                // recursively propagate statistics
                propagateStatisticsAcrossBlock(fsb, tmpVars, fcallSizes, fnStack);
                // extract vars from symbol table, re-map and refresh main program
                extractFunctionCallReturnStatistics(fstmt, fop, tmpVars, callVars, true);
                // maintain function call stack
                fnStack.remove(fkey);
            } else if (fcallSizes.isDimsPreservingFunction(fkey)) {
                extractFunctionCallEquivalentReturnStatistics(fstmt, fop, callVars);
            } else {
                extractFunctionCallUnknownReturnStatistics(fstmt, fop, callVars);
            }
        } else if (fop.getFunctionType() == FunctionType.EXTERNAL_FILE || fop.getFunctionType() == FunctionType.EXTERNAL_MEM) {
            // infer output size for known external functions
            FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
            ExternalFunctionStatement fstmt = (ExternalFunctionStatement) fsb.getStatement(0);
            if (PROPAGATE_KNOWN_UDF_STATISTICS)
                extractExternalFunctionCallReturnStatistics(fstmt, fop, callVars);
            else
                extractFunctionCallUnknownReturnStatistics(fstmt, fop, callVars);
        }
    }
    hop.setVisited();
}

Example 54 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class InterProceduralAnalysis method extractExternalFunctionCallReturnStatistics.

private static void extractExternalFunctionCallReturnStatistics(ExternalFunctionStatement fstmt, FunctionOp fop, LocalVariableMap callVars) {
    String className = fstmt.getOtherParams().get(ExternalFunctionStatement.CLASS_NAME);
    if (className.equals(OrderWrapper.class.getName())) {
        Hop input = fop.getInput().get(0);
        long lnnz = className.equals(OrderWrapper.class.getName()) ? input.getNnz() : -1;
        MatrixObject moOut = createOutputMatrix(input.getDim1(), input.getDim2(), lnnz);
        callVars.put(fop.getOutputVariableNames()[0], moOut);
    } else if (className.equals(DynamicReadMatrixCP.class.getName()) || className.equals(DynamicReadMatrixRcCP.class.getName())) {
        // rows
        Hop input1 = fop.getInput().get(1);
        // cols
        Hop input2 = fop.getInput().get(2);
        if (input1 instanceof LiteralOp && input2 instanceof LiteralOp)
            callVars.put(fop.getOutputVariableNames()[0], createOutputMatrix(((LiteralOp) input1).getLongValue(), ((LiteralOp) input2).getLongValue(), -1));
    } else {
        extractFunctionCallUnknownReturnStatistics(fstmt, fop, callVars);
    }
}

Example 55 with Hop

use of org.apache.sysml.hops.Hop in project incubator-systemml by apache.

the class LiteralReplacement method replaceLiteralFullUnaryAggregate.

private static LiteralOp replaceLiteralFullUnaryAggregate(Hop c, LocalVariableMap vars) {
    LiteralOp ret = null;
    // full unary aggregate w/ matrix less than 10^6 cells
    if (c instanceof AggUnaryOp && isReplaceableUnaryAggregate((AggUnaryOp) c) && c.getInput().get(0) instanceof DataOp && vars.keySet().contains(c.getInput().get(0).getName())) {
        Hop data = c.getInput().get(0);
        MatrixObject mo = (MatrixObject) vars.get(data.getName());
        // dimensions might not have been updated during recompile
        if (mo.getNumRows() * mo.getNumColumns() < REPLACE_LITERALS_MAX_MATRIX_SIZE) {
            MatrixBlock mBlock = mo.acquireRead();
            double value = replaceUnaryAggregate((AggUnaryOp) c, mBlock);
            mo.release();
            // literal substitution (always double)
            ret = new LiteralOp(value);
        }
    }
    return ret;
}