Examples with DataOp org.apache.sysml.hops.DataOp used on opensource projects

Example 61 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class HopRewriteUtils method hasOnlyWriteParents.

public static boolean hasOnlyWriteParents(Hop hop, boolean inclTransient, boolean inclPersistent) {
    boolean ret = true;
    ArrayList<Hop> parents = hop.getParent();
    for (Hop p : parents) {
        if (inclTransient && inclPersistent)
            ret &= (p instanceof DataOp && (((DataOp) p).getDataOpType() == DataOpTypes.TRANSIENTWRITE || ((DataOp) p).getDataOpType() == DataOpTypes.PERSISTENTWRITE));
        else if (inclTransient)
            ret &= (p instanceof DataOp && ((DataOp) p).getDataOpType() == DataOpTypes.TRANSIENTWRITE);
        else if (inclPersistent)
            ret &= (p instanceof DataOp && ((DataOp) p).getDataOpType() == DataOpTypes.PERSISTENTWRITE);
    }
    return ret;
}

Example 62 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class OptTreeConverter method rCreateAbstractOptNodes.

public static ArrayList<OptNode> rCreateAbstractOptNodes(Hop hop, LocalVariableMap vars, Set<String> memo) {
    ArrayList<OptNode> ret = new ArrayList<>();
    ArrayList<Hop> in = hop.getInput();
    if (hop.isVisited())
        return ret;
    // general case
    if (!(hop instanceof DataOp || hop instanceof LiteralOp || hop instanceof FunctionOp)) {
        OptNode node = new OptNode(NodeType.HOP);
        String opstr = hop.getOpString();
        node.addParam(ParamType.OPSTRING, opstr);
        // handle execution type
        LopProperties.ExecType et = (hop.getExecType() != null) ? hop.getExecType() : LopProperties.ExecType.CP;
        switch(et) {
            case CP:
            case GPU:
                node.setExecType(ExecType.CP);
                break;
            case SPARK:
                node.setExecType(ExecType.SPARK);
                break;
            case MR:
                node.setExecType(ExecType.MR);
                break;
            default:
                throw new DMLRuntimeException("Unsupported optnode exec type: " + et);
        }
        // handle degree of parallelism
        if (et == LopProperties.ExecType.CP && hop instanceof MultiThreadedHop) {
            MultiThreadedHop mtop = (MultiThreadedHop) hop;
            node.setK(OptimizerUtils.getConstrainedNumThreads(mtop.getMaxNumThreads()));
        }
        // assign node to return
        _hlMap.putHopMapping(hop, node);
        ret.add(node);
    } else // process function calls
    if (hop instanceof FunctionOp && INCLUDE_FUNCTIONS) {
        FunctionOp fhop = (FunctionOp) hop;
        String fname = fhop.getFunctionName();
        String fnspace = fhop.getFunctionNamespace();
        String fKey = fhop.getFunctionKey();
        Object[] prog = _hlMap.getRootProgram();
        OptNode node = new OptNode(NodeType.FUNCCALL);
        _hlMap.putHopMapping(fhop, node);
        node.setExecType(ExecType.CP);
        node.addParam(ParamType.OPSTRING, fKey);
        if (!fnspace.equals(DMLProgram.INTERNAL_NAMESPACE)) {
            FunctionProgramBlock fpb = ((Program) prog[1]).getFunctionProgramBlock(fnspace, fname);
            FunctionStatementBlock fsb = ((DMLProgram) prog[0]).getFunctionStatementBlock(fnspace, fname);
            FunctionStatement fs = (FunctionStatement) fsb.getStatement(0);
            // process body; NOTE: memo prevents inclusion of functions multiple times
            if (!memo.contains(fKey)) {
                memo.add(fKey);
                int len = fs.getBody().size();
                for (int i = 0; i < fpb.getChildBlocks().size() && i < len; i++) {
                    ProgramBlock lpb = fpb.getChildBlocks().get(i);
                    StatementBlock lsb = fs.getBody().get(i);
                    node.addChild(rCreateAbstractOptNode(lsb, lpb, vars, false, memo));
                }
                memo.remove(fKey);
            } else
                node.addParam(ParamType.RECURSIVE_CALL, "true");
        }
        ret.add(node);
    }
    if (in != null)
        for (Hop hin : in) if (// no need for opt nodes
        !(hin instanceof DataOp || hin instanceof LiteralOp))
            ret.addAll(rCreateAbstractOptNodes(hin, vars, memo));
    hop.setVisited();
    return ret;
}

Example 63 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class ResourceOptimizer method collectReadVariables.

private static void collectReadVariables(Hop hop, LocalVariableMap vars) {
    if (hop == null)
        return;
    // process childs
    for (Hop hi : hop.getInput()) collectReadVariables(hi, vars);
    // investigate hop exec type and known dimensions
    if (hop instanceof DataOp && hop.getDataType() == DataType.MATRIX && (((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTREAD || ((DataOp) hop).getDataOpType() == DataOpTypes.PERSISTENTREAD)) {
        String varname = hop.getName();
        MatrixCharacteristics mc = new MatrixCharacteristics(hop.getDim1(), hop.getDim2(), (int) hop.getRowsInBlock(), (int) hop.getColsInBlock(), hop.getNnz());
        MatrixObject mo = new MatrixObject(ValueType.DOUBLE, "/tmp", new MetaData(mc));
        vars.put(varname, mo);
    }
    hop.setVisited();
}

Example 64 with DataOp

use of org.apache.sysml.hops.DataOp in project systemml by apache.

the class ResourceOptimizer method collectReadVariables.

private static void collectReadVariables(Hop hop, LocalVariableMap vars) {
    if (hop == null)
        return;
    // process childs
    for (Hop hi : hop.getInput()) collectReadVariables(hi, vars);
    // investigate hop exec type and known dimensions
    if (hop instanceof DataOp && hop.getDataType() == DataType.MATRIX && (((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTREAD || ((DataOp) hop).getDataOpType() == DataOpTypes.PERSISTENTREAD)) {
        String varname = hop.getName();
        MatrixCharacteristics mc = new MatrixCharacteristics(hop.getDim1(), hop.getDim2(), (int) hop.getRowsInBlock(), (int) hop.getColsInBlock(), hop.getNnz());
        MatrixObject mo = new MatrixObject(ValueType.DOUBLE, "/tmp", new MetaData(mc));
        vars.put(varname, mo);
    }
    hop.setVisited();
}

Example 65 with DataOp

use of org.apache.sysml.hops.DataOp in project systemml by apache.

the class DMLTranslator method constructHopsForConditionalPredicate.

public void constructHopsForConditionalPredicate(StatementBlock passedSB) {
    HashMap<String, Hop> _ids = new HashMap<>();
    // set conditional predicate
    ConditionalPredicate cp = null;
    if (passedSB instanceof WhileStatementBlock) {
        WhileStatement ws = (WhileStatement) ((WhileStatementBlock) passedSB).getStatement(0);
        cp = ws.getConditionalPredicate();
    } else if (passedSB instanceof IfStatementBlock) {
        IfStatement ws = (IfStatement) ((IfStatementBlock) passedSB).getStatement(0);
        cp = ws.getConditionalPredicate();
    } else {
        throw new ParseException("ConditionalPredicate expected only for while or if statements.");
    }
    VariableSet varsRead = cp.variablesRead();
    for (String varName : varsRead.getVariables().keySet()) {
        // creating transient read for live in variables
        DataIdentifier var = passedSB.liveIn().getVariables().get(varName);
        DataOp read = null;
        if (var == null) {
            LOG.error("variable " + varName + " not live variable for conditional predicate");
            throw new ParseException("variable " + varName + " not live variable for conditional predicate");
        } else {
            long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim1() : var.getDim1();
            long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim2() : var.getDim2();
            read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
            read.setParseInfo(var);
        }
        _ids.put(varName, read);
    }
    DataIdentifier target = new DataIdentifier(Expression.getTempName());
    target.setDataType(DataType.SCALAR);
    target.setValueType(ValueType.BOOLEAN);
    target.setParseInfo(passedSB);
    Hop predicateHops = null;
    Expression predicate = cp.getPredicate();
    if (predicate instanceof RelationalExpression) {
        predicateHops = processRelationalExpression((RelationalExpression) cp.getPredicate(), target, _ids);
    } else if (predicate instanceof BooleanExpression) {
        predicateHops = processBooleanExpression((BooleanExpression) cp.getPredicate(), target, _ids);
    } else if (predicate instanceof DataIdentifier) {
        // handle data identifier predicate
        predicateHops = processExpression(cp.getPredicate(), null, _ids);
    } else if (predicate instanceof ConstIdentifier) {
        // b) disallow string values
        if ((predicate instanceof IntIdentifier && ((IntIdentifier) predicate).getValue() == 0) || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier) predicate).getValue() == 0.0)) {
            cp.setPredicate(new BooleanIdentifier(false, predicate));
        } else if ((predicate instanceof IntIdentifier && ((IntIdentifier) predicate).getValue() == 1) || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier) predicate).getValue() == 1.0)) {
            cp.setPredicate(new BooleanIdentifier(true, predicate));
        } else if (predicate instanceof IntIdentifier || predicate instanceof DoubleIdentifier) {
            cp.setPredicate(new BooleanIdentifier(true, predicate));
            LOG.warn(predicate.printWarningLocation() + "Numerical value '" + predicate.toString() + "' (!= 0/1) is converted to boolean TRUE by DML");
        } else if (predicate instanceof StringIdentifier) {
            LOG.error(predicate.printErrorLocation() + "String value '" + predicate.toString() + "' is not allowed for iterable predicate");
            throw new ParseException(predicate.printErrorLocation() + "String value '" + predicate.toString() + "' is not allowed for iterable predicate");
        }
        predicateHops = processExpression(cp.getPredicate(), null, _ids);
    }
    // create transient write to internal variable name on top of expression
    // in order to ensure proper instruction generation
    predicateHops = HopRewriteUtils.createDataOp(ProgramBlock.PRED_VAR, predicateHops, DataOpTypes.TRANSIENTWRITE);
    if (passedSB instanceof WhileStatementBlock)
        ((WhileStatementBlock) passedSB).setPredicateHops(predicateHops);
    else if (passedSB instanceof IfStatementBlock)
        ((IfStatementBlock) passedSB).setPredicateHops(predicateHops);
}