Examples with DataOp org.apache.sysml.hops.DataOp used on opensource projects

Example 66 with DataOp

use of org.apache.sysml.hops.DataOp in project systemml by apache.

the class DMLTranslator method processMultipleReturnBuiltinFunctionExpression.

/**
 * Construct HOps from parse tree: process BuiltinFunction Expressions in
 * MultiAssignment Statements. For all other builtin function expressions,
 * <code>processBuiltinFunctionExpression()</code> is used.
 *
 * @param source built-in function expression
 * @param targetList list of data identifiers
 * @param hops map of high-level operators
 * @return high-level operator
 */
private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpression source, ArrayList<DataIdentifier> targetList, HashMap<String, Hop> hops) {
    // Construct Hops for all inputs
    ArrayList<Hop> inputs = new ArrayList<>();
    inputs.add(processExpression(source.getFirstExpr(), null, hops));
    if (source.getSecondExpr() != null)
        inputs.add(processExpression(source.getSecondExpr(), null, hops));
    if (source.getThirdExpr() != null)
        inputs.add(processExpression(source.getThirdExpr(), null, hops));
    FunctionType ftype = FunctionType.MULTIRETURN_BUILTIN;
    String nameSpace = DMLProgram.INTERNAL_NAMESPACE;
    // Create an array list to hold the outputs of this lop.
    // Exact list of outputs are added based on opcode.
    ArrayList<Hop> outputs = new ArrayList<>();
    // Construct Hop for current builtin function expression based on its type
    Hop currBuiltinOp = null;
    switch(source.getOpCode()) {
        case QR:
        case LU:
        case EIGEN:
        case SVD:
            // Number of outputs = size of targetList = #of identifiers in source.getOutputs
            String[] outputNames = new String[targetList.size()];
            for (int i = 0; i < targetList.size(); i++) {
                outputNames[i] = ((DataIdentifier) targetList.get(i)).getName();
                Hop output = new DataOp(outputNames[i], DataType.MATRIX, ValueType.DOUBLE, inputs.get(0), DataOpTypes.FUNCTIONOUTPUT, outputNames[i]);
                outputs.add(output);
            }
            // Create the hop for current function call
            FunctionOp fcall = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), inputs, outputNames, outputs);
            currBuiltinOp = fcall;
            break;
        default:
            throw new ParseException("Invaid Opcode in DMLTranslator:processMultipleReturnBuiltinFunctionExpression(): " + source.getOpCode());
    }
    // set properties for created hops based on outputs of source expression
    for (int i = 0; i < source.getOutputs().length; i++) {
        setIdentifierParams(outputs.get(i), source.getOutputs()[i]);
        outputs.get(i).setParseInfo(source);
    }
    currBuiltinOp.setParseInfo(source);
    return currBuiltinOp;
}

Example 67 with DataOp

use of org.apache.sysml.hops.DataOp in project systemml by apache.

the class DMLTranslator method processExpression.

/**
 * Construct Hops from parse tree : Process Expression in an assignment
 * statement
 *
 * @param source source expression
 * @param target data identifier
 * @param hops map of high-level operators
 * @return high-level operator
 */
private Hop processExpression(Expression source, DataIdentifier target, HashMap<String, Hop> hops) {
    try {
        if (source instanceof BinaryExpression)
            return processBinaryExpression((BinaryExpression) source, target, hops);
        else if (source instanceof RelationalExpression)
            return processRelationalExpression((RelationalExpression) source, target, hops);
        else if (source instanceof BooleanExpression)
            return processBooleanExpression((BooleanExpression) source, target, hops);
        else if (source instanceof BuiltinFunctionExpression)
            return processBuiltinFunctionExpression((BuiltinFunctionExpression) source, target, hops);
        else if (source instanceof ParameterizedBuiltinFunctionExpression)
            return processParameterizedBuiltinFunctionExpression((ParameterizedBuiltinFunctionExpression) source, target, hops);
        else if (source instanceof DataExpression) {
            Hop ae = (Hop) processDataExpression((DataExpression) source, target, hops);
            if (ae instanceof DataOp) {
                String formatName = ((DataExpression) source).getVarParam(DataExpression.FORMAT_TYPE).toString();
                ((DataOp) ae).setInputFormatType(Expression.convertFormatType(formatName));
            }
            return ae;
        } else if (source instanceof IndexedIdentifier)
            return processIndexingExpression((IndexedIdentifier) source, target, hops);
        else if (source instanceof IntIdentifier) {
            IntIdentifier sourceInt = (IntIdentifier) source;
            LiteralOp litop = new LiteralOp(sourceInt.getValue());
            litop.setParseInfo(sourceInt);
            setIdentifierParams(litop, sourceInt);
            return litop;
        } else if (source instanceof DoubleIdentifier) {
            DoubleIdentifier sourceDouble = (DoubleIdentifier) source;
            LiteralOp litop = new LiteralOp(sourceDouble.getValue());
            litop.setParseInfo(sourceDouble);
            setIdentifierParams(litop, sourceDouble);
            return litop;
        } else if (source instanceof BooleanIdentifier) {
            BooleanIdentifier sourceBoolean = (BooleanIdentifier) source;
            LiteralOp litop = new LiteralOp(sourceBoolean.getValue());
            litop.setParseInfo(sourceBoolean);
            setIdentifierParams(litop, sourceBoolean);
            return litop;
        } else if (source instanceof StringIdentifier) {
            StringIdentifier sourceString = (StringIdentifier) source;
            LiteralOp litop = new LiteralOp(sourceString.getValue());
            litop.setParseInfo(sourceString);
            setIdentifierParams(litop, sourceString);
            return litop;
        } else if (source instanceof DataIdentifier)
            return hops.get(((DataIdentifier) source).getName());
    } catch (Exception e) {
        throw new ParseException(e.getMessage());
    }
    return null;
}

Example 68 with DataOp

use of org.apache.sysml.hops.DataOp in project systemml by apache.

the class TemplateCell method rConstructCplan.

protected void rConstructCplan(Hop hop, CPlanMemoTable memo, HashMap<Long, CNode> tmp, HashSet<Hop> inHops, boolean compileLiterals) {
    // memoization for common subexpression elimination and to avoid redundant work
    if (tmp.containsKey(hop.getHopID()))
        return;
    MemoTableEntry me = memo.getBest(hop.getHopID(), TemplateType.CELL);
    // recursively process required childs
    if (me != null && me.type.isIn(TemplateType.ROW, TemplateType.OUTER)) {
        CNodeData cdata = TemplateUtils.createCNodeData(hop, compileLiterals);
        tmp.put(hop.getHopID(), cdata);
        inHops.add(hop);
        return;
    }
    for (int i = 0; i < hop.getInput().size(); i++) {
        Hop c = hop.getInput().get(i);
        if (me != null && me.isPlanRef(i) && !(c instanceof DataOp) && (me.type != TemplateType.MAGG || memo.contains(c.getHopID(), TemplateType.CELL)))
            rConstructCplan(c, memo, tmp, inHops, compileLiterals);
        else if (me != null && (me.type == TemplateType.MAGG || me.type == TemplateType.CELL) && HopRewriteUtils.isMatrixMultiply(hop) && // skip transpose
        i == 0)
            rConstructCplan(c.getInput().get(0), memo, tmp, inHops, compileLiterals);
        else {
            CNodeData cdata = TemplateUtils.createCNodeData(c, compileLiterals);
            tmp.put(c.getHopID(), cdata);
            inHops.add(c);
        }
    }
    // construct cnode for current hop
    CNode out = null;
    if (hop instanceof UnaryOp) {
        CNode cdata1 = tmp.get(hop.getInput().get(0).getHopID());
        cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
        String primitiveOpName = ((UnaryOp) hop).getOp().name();
        out = new CNodeUnary(cdata1, UnaryType.valueOf(primitiveOpName));
    } else if (hop instanceof BinaryOp) {
        BinaryOp bop = (BinaryOp) hop;
        CNode cdata1 = tmp.get(hop.getInput().get(0).getHopID());
        CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID());
        String primitiveOpName = bop.getOp().name();
        // add lookups if required
        cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
        cdata2 = TemplateUtils.wrapLookupIfNecessary(cdata2, hop.getInput().get(1));
        // construct binary cnode
        out = new CNodeBinary(cdata1, cdata2, BinType.valueOf(primitiveOpName));
    } else if (hop instanceof TernaryOp) {
        TernaryOp top = (TernaryOp) hop;
        CNode cdata1 = tmp.get(hop.getInput().get(0).getHopID());
        CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID());
        CNode cdata3 = tmp.get(hop.getInput().get(2).getHopID());
        // add lookups if required
        cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
        cdata2 = TemplateUtils.wrapLookupIfNecessary(cdata2, hop.getInput().get(1));
        cdata3 = TemplateUtils.wrapLookupIfNecessary(cdata3, hop.getInput().get(2));
        // construct ternary cnode, primitive operation derived from OpOp3
        out = new CNodeTernary(cdata1, cdata2, cdata3, TernaryType.valueOf(top.getOp().name()));
    } else if (hop instanceof ParameterizedBuiltinOp) {
        CNode cdata1 = tmp.get(((ParameterizedBuiltinOp) hop).getTargetHop().getHopID());
        cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
        CNode cdata2 = tmp.get(((ParameterizedBuiltinOp) hop).getParameterHop("pattern").getHopID());
        CNode cdata3 = tmp.get(((ParameterizedBuiltinOp) hop).getParameterHop("replacement").getHopID());
        TernaryType ttype = (cdata2.isLiteral() && cdata2.getVarname().equals("Double.NaN")) ? TernaryType.REPLACE_NAN : TernaryType.REPLACE;
        out = new CNodeTernary(cdata1, cdata2, cdata3, ttype);
    } else if (hop instanceof IndexingOp) {
        CNode cdata1 = tmp.get(hop.getInput().get(0).getHopID());
        out = new CNodeTernary(cdata1, TemplateUtils.createCNodeData(new LiteralOp(hop.getInput().get(0).getDim2()), true), TemplateUtils.createCNodeData(hop.getInput().get(4), true), TernaryType.LOOKUP_RC1);
    } else if (HopRewriteUtils.isTransposeOperation(hop)) {
        out = TemplateUtils.skipTranspose(tmp.get(hop.getHopID()), hop, tmp, compileLiterals);
        // correct indexing types of existing lookups
        if (!HopRewriteUtils.containsOp(hop.getParent(), AggBinaryOp.class))
            TemplateUtils.rFlipVectorLookups(out);
        // maintain input hops
        if (out instanceof CNodeData && !inHops.contains(hop.getInput().get(0)))
            inHops.add(hop.getInput().get(0));
    } else if (hop instanceof AggUnaryOp) {
        // aggregation handled in template implementation (note: we do not compile
        // ^2 of SUM_SQ into the operator to simplify the detection of single operators)
        out = tmp.get(hop.getInput().get(0).getHopID());
    } else if (hop instanceof AggBinaryOp) {
        // (1) t(X)%*%X -> sum(X^2) and t(X) %*% Y -> sum(X*Y)
        if (HopRewriteUtils.isTransposeOfItself(hop.getInput().get(0), hop.getInput().get(1))) {
            CNode cdata1 = tmp.get(hop.getInput().get(1).getHopID());
            if (TemplateUtils.isColVector(cdata1))
                cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
            out = new CNodeUnary(cdata1, UnaryType.POW2);
        } else {
            CNode cdata1 = TemplateUtils.skipTranspose(tmp.get(hop.getInput().get(0).getHopID()), hop.getInput().get(0), tmp, compileLiterals);
            if (cdata1 instanceof CNodeData && !inHops.contains(hop.getInput().get(0).getInput().get(0)))
                inHops.add(hop.getInput().get(0).getInput().get(0));
            if (TemplateUtils.isColVector(cdata1))
                cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
            CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID());
            if (TemplateUtils.isColVector(cdata2))
                cdata2 = new CNodeUnary(cdata2, UnaryType.LOOKUP_R);
            out = new CNodeBinary(cdata1, cdata2, BinType.MULT);
        }
    }
    tmp.put(hop.getHopID(), out);
}

Example 69 with DataOp

use of org.apache.sysml.hops.DataOp in project systemml by apache.

the class RewriteBlockSizeAndReblock method rule_BlockSizeAndReblock.

private void rule_BlockSizeAndReblock(Hop hop, final int blocksize) {
    // Go to the source(s) of the DAG
    for (Hop hi : hop.getInput()) {
        if (!hi.isVisited())
            rule_BlockSizeAndReblock(hi, blocksize);
    }
    boolean canReblock = isReblockValid();
    if (hop instanceof DataOp) {
        DataOp dop = (DataOp) hop;
        // if block size does not match
        if (canReblock && ((dop.getDataType() == DataType.MATRIX && (dop.getRowsInBlock() != blocksize || dop.getColsInBlock() != blocksize)) || (dop.getDataType() == DataType.FRAME && OptimizerUtils.isSparkExecutionMode() && (dop.getInputFormatType() == FileFormatTypes.TEXT || dop.getInputFormatType() == FileFormatTypes.CSV)))) {
            if (dop.getDataOpType() == DataOp.DataOpTypes.PERSISTENTREAD) {
                // insert reblock after the hop
                dop.setRequiresReblock(true);
                dop.setOutputBlocksizes(blocksize, blocksize);
            } else if (dop.getDataOpType() == DataOp.DataOpTypes.PERSISTENTWRITE) {
                if (dop.getRowsInBlock() == -1 && dop.getColsInBlock() == -1) {
                // if this dataop is for cell output, then no reblock is needed
                // as (A) all jobtypes can produce block2cell and cell2cell and
                // (B) we don't generate an explicit instruction for it (the info
                // is conveyed through OutputInfo.
                } else if (dop.getInput().get(0).requiresReblock() && dop.getInput().get(0).getParent().size() == 1) {
                    // if a reblock is feeding into this, then use it if this is
                    // the only parent, otherwise new Reblock
                    dop.getInput().get(0).setOutputBlocksizes(dop.getRowsInBlock(), dop.getColsInBlock());
                } else {
                    // insert reblock after the hop
                    dop.setRequiresReblock(true);
                    dop.setOutputBlocksizes(blocksize, blocksize);
                }
            } else if (dop.getDataOpType() == DataOp.DataOpTypes.TRANSIENTWRITE || dop.getDataOpType() == DataOp.DataOpTypes.TRANSIENTREAD) {
                if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE) {
                    // simply copy the values from its input
                    dop.setRowsInBlock(hop.getInput().get(0).getRowsInBlock());
                    dop.setColsInBlock(hop.getInput().get(0).getColsInBlock());
                } else {
                    // by default, all transient reads and writes are in blocked format
                    dop.setRowsInBlock(blocksize);
                    dop.setColsInBlock(blocksize);
                }
            } else {
                throw new HopsException(hop.printErrorLocation() + "unexpected non-scalar Data HOP in reblock.\n");
            }
        }
    } else // NO DATAOP
    {
        if (hop.requiresReblock()) {
            hop.setRowsInBlock(blocksize);
            hop.setColsInBlock(blocksize);
        } else // Constraint C2:
        if (hop.getDataType() == DataType.SCALAR) {
            hop.setRowsInBlock(-1);
            hop.setColsInBlock(-1);
        } else // Constraint C3:
        {
            if (!canReblock) {
                hop.setRowsInBlock(-1);
                hop.setColsInBlock(-1);
            } else {
                hop.setRowsInBlock(blocksize);
                hop.setColsInBlock(blocksize);
                // Reblock properties need to be set for each output.
                if (hop instanceof FunctionOp) {
                    FunctionOp fop = (FunctionOp) hop;
                    if (fop.getOutputs() != null) {
                        for (Hop out : fop.getOutputs()) {
                            out.setRowsInBlock(blocksize);
                            out.setColsInBlock(blocksize);
                        }
                    }
                }
            }
            // if any input is not blocked then the output of current Hop should not be blocked
            for (Hop h : hop.getInput()) {
                if (h.getDataType() == DataType.MATRIX && h.getRowsInBlock() == -1 && h.getColsInBlock() == -1) {
                    hop.setRowsInBlock(-1);
                    hop.setColsInBlock(-1);
                    break;
                }
            }
        }
    }
    hop.setVisited();
}

Example 70 with DataOp

use of org.apache.sysml.hops.DataOp in project systemml by apache.

the class RewriteCommonSubexpressionElimination method rule_CommonSubexpressionElimination_MergeLeafs.

private int rule_CommonSubexpressionElimination_MergeLeafs(Hop hop, HashMap<String, Hop> dataops, HashMap<String, Hop> literalops) {
    int ret = 0;
    if (hop.isVisited())
        return ret;
    if (// LEAF NODE
    hop.getInput().isEmpty()) {
        if (hop instanceof LiteralOp) {
            String key = hop.getValueType() + "_" + hop.getName();
            if (!literalops.containsKey(key))
                literalops.put(key, hop);
        } else if (hop instanceof DataOp && ((DataOp) hop).isRead()) {
            if (!dataops.containsKey(hop.getName()))
                dataops.put(hop.getName(), hop);
        }
    } else // INNER NODE
    {
        // merge leaf nodes (data, literal)
        for (int i = 0; i < hop.getInput().size(); i++) {
            Hop hi = hop.getInput().get(i);
            String litKey = hi.getValueType() + "_" + hi.getName();
            if (hi instanceof DataOp && ((DataOp) hi).isRead() && dataops.containsKey(hi.getName())) {
                // replace child node ref
                Hop tmp = dataops.get(hi.getName());
                if (tmp != hi) {
                    // if required
                    tmp.getParent().add(hop);
                    tmp.setVisited();
                    hop.getInput().set(i, tmp);
                    ret++;
                }
            } else if (hi instanceof LiteralOp && literalops.containsKey(litKey)) {
                Hop tmp = literalops.get(litKey);
                // replace child node ref
                if (tmp != hi) {
                    // if required
                    tmp.getParent().add(hop);
                    tmp.setVisited();
                    hop.getInput().set(i, tmp);
                    ret++;
                }
            }
            // recursive invocation (direct return on merged nodes)
            ret += rule_CommonSubexpressionElimination_MergeLeafs(hi, dataops, literalops);
        }
    }
    hop.setVisited();
    return ret;
}