Examples with DataOp org.apache.sysml.hops.DataOp used on opensource projects

Example 6 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class LiteralReplacement method replaceLiteralValueTypeCastScalarRead.

private static LiteralOp replaceLiteralValueTypeCastScalarRead(Hop c, LocalVariableMap vars) {
    LiteralOp ret = null;
    // as.double/as.integer/as.boolean over scalar read - literal replacement
    if (c instanceof UnaryOp && (((UnaryOp) c).getOp() == OpOp1.CAST_AS_DOUBLE || ((UnaryOp) c).getOp() == OpOp1.CAST_AS_INT || ((UnaryOp) c).getOp() == OpOp1.CAST_AS_BOOLEAN) && c.getInput().get(0) instanceof DataOp && c.getDataType() == DataType.SCALAR) {
        Data dat = vars.get(c.getInput().get(0).getName());
        if (// required for selective constant propagation
        dat != null) {
            ScalarObject sdat = (ScalarObject) dat;
            UnaryOp cast = (UnaryOp) c;
            switch(cast.getOp()) {
                case CAST_AS_INT:
                    ret = new LiteralOp(sdat.getLongValue());
                    break;
                case CAST_AS_DOUBLE:
                    ret = new LiteralOp(sdat.getDoubleValue());
                    break;
                case CAST_AS_BOOLEAN:
                    ret = new LiteralOp(sdat.getBooleanValue());
                    break;
                default:
            }
        }
    }
    return ret;
}

Example 7 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class LiteralReplacement method replaceLiteralFullUnaryAggregate.

private static LiteralOp replaceLiteralFullUnaryAggregate(Hop c, LocalVariableMap vars) {
    LiteralOp ret = null;
    // full unary aggregate w/ matrix less than 10^6 cells
    if (c instanceof AggUnaryOp && isReplaceableUnaryAggregate((AggUnaryOp) c) && c.getInput().get(0) instanceof DataOp && vars.keySet().contains(c.getInput().get(0).getName())) {
        Hop data = c.getInput().get(0);
        MatrixObject mo = (MatrixObject) vars.get(data.getName());
        // dimensions might not have been updated during recompile
        if (mo.getNumRows() * mo.getNumColumns() < REPLACE_LITERALS_MAX_MATRIX_SIZE) {
            MatrixBlock mBlock = mo.acquireRead();
            double value = replaceUnaryAggregate((AggUnaryOp) c, mBlock);
            mo.release();
            // literal substitution (always double)
            ret = new LiteralOp(value);
        }
    }
    return ret;
}

Example 8 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class LiteralReplacement method replaceLiteralDataTypeCastMatrixRead.

private static LiteralOp replaceLiteralDataTypeCastMatrixRead(Hop c, LocalVariableMap vars) {
    LiteralOp ret = null;
    // as.scalar/matrix read - literal replacement
    if (c instanceof UnaryOp && ((UnaryOp) c).getOp() == OpOp1.CAST_AS_SCALAR && c.getInput().get(0) instanceof DataOp && c.getInput().get(0).getDataType() == DataType.MATRIX) {
        Data dat = vars.get(c.getInput().get(0).getName());
        if (// required for selective constant propagation
        dat != null) {
            // cast as scalar (see VariableCPInstruction)
            MatrixObject mo = (MatrixObject) dat;
            MatrixBlock mBlock = mo.acquireRead();
            if (mBlock.getNumRows() != 1 || mBlock.getNumColumns() != 1)
                throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix of dimension (" + mBlock.getNumRows() + " x " + mBlock.getNumColumns() + ") to scalar.");
            double value = mBlock.getValue(0, 0);
            mo.release();
            // literal substitution (always double)
            ret = new LiteralOp(value);
        }
    }
    return ret;
}

Example 9 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class HopRewriteUtils method rContainsRead.

public static boolean rContainsRead(Hop root, String var, boolean includeMetaOp) {
    if (root.isVisited())
        return false;
    boolean ret = false;
    // handle leaf node for variable
    if (root instanceof DataOp && ((DataOp) root).isRead() && root.getName().equals(var)) {
        boolean onlyMetaOp = true;
        if (!includeMetaOp) {
            for (Hop p : root.getParent()) {
                onlyMetaOp &= (p instanceof UnaryOp && (((UnaryOp) p).getOp() == OpOp1.NROW || ((UnaryOp) p).getOp() == OpOp1.NCOL));
            }
            ret = !onlyMetaOp;
        } else
            ret = true;
    }
    // recursively process childs
    for (Hop c : root.getInput()) ret |= rContainsRead(c, var, includeMetaOp);
    root.setVisited();
    return ret;
}

Example 10 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class DMLTranslator method constructHops.

public void constructHops(StatementBlock sb) {
    if (sb instanceof WhileStatementBlock) {
        constructHopsForWhileControlBlock((WhileStatementBlock) sb);
        return;
    }
    if (sb instanceof IfStatementBlock) {
        constructHopsForIfControlBlock((IfStatementBlock) sb);
        return;
    }
    if (sb instanceof ForStatementBlock) {
        // incl ParForStatementBlock
        constructHopsForForControlBlock((ForStatementBlock) sb);
        return;
    }
    if (sb instanceof FunctionStatementBlock) {
        constructHopsForFunctionControlBlock((FunctionStatementBlock) sb);
        return;
    }
    HashMap<String, Hop> ids = new HashMap<>();
    ArrayList<Hop> output = new ArrayList<>();
    VariableSet liveIn = sb.liveIn();
    VariableSet liveOut = sb.liveOut();
    VariableSet updated = sb._updated;
    VariableSet gen = sb._gen;
    VariableSet updatedLiveOut = new VariableSet();
    // handle liveout variables that are updated --> target identifiers for Assignment
    HashMap<String, Integer> liveOutToTemp = new HashMap<>();
    for (int i = 0; i < sb.getNumStatements(); i++) {
        Statement current = sb.getStatement(i);
        if (current instanceof AssignmentStatement) {
            AssignmentStatement as = (AssignmentStatement) current;
            DataIdentifier target = as.getTarget();
            if (target != null) {
                if (liveOut.containsVariable(target.getName())) {
                    liveOutToTemp.put(target.getName(), Integer.valueOf(i));
                }
            }
        }
        if (current instanceof MultiAssignmentStatement) {
            MultiAssignmentStatement mas = (MultiAssignmentStatement) current;
            for (DataIdentifier target : mas.getTargetList()) {
                if (liveOut.containsVariable(target.getName())) {
                    liveOutToTemp.put(target.getName(), Integer.valueOf(i));
                }
            }
        }
    }
    // (i.e., from LV analysis, updated and gen sets)
    if (!liveIn.getVariables().values().isEmpty()) {
        for (String varName : liveIn.getVariables().keySet()) {
            if (updated.containsVariable(varName) || gen.containsVariable(varName)) {
                DataIdentifier var = liveIn.getVariables().get(varName);
                long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim1() : var.getDim1();
                long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim2() : var.getDim2();
                DataOp read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
                read.setParseInfo(var);
                ids.put(varName, read);
            }
        }
    }
    for (int i = 0; i < sb.getNumStatements(); i++) {
        Statement current = sb.getStatement(i);
        if (current instanceof OutputStatement) {
            OutputStatement os = (OutputStatement) current;
            DataExpression source = os.getSource();
            DataIdentifier target = os.getIdentifier();
            // error handling unsupported indexing expression in write statement
            if (target instanceof IndexedIdentifier) {
                throw new LanguageException(source.printErrorLocation() + ": Unsupported indexing expression in write statement. " + "Please, assign the right indexing result to a variable and write this variable.");
            }
            DataOp ae = (DataOp) processExpression(source, target, ids);
            String formatName = os.getExprParam(DataExpression.FORMAT_TYPE).toString();
            ae.setInputFormatType(Expression.convertFormatType(formatName));
            if (ae.getDataType() == DataType.SCALAR) {
                ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), -1, -1);
            } else {
                switch(ae.getInputFormatType()) {
                    case TEXT:
                    case MM:
                    case CSV:
                        // write output in textcell format
                        ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), -1, -1);
                        break;
                    case BINARY:
                        // write output in binary block format
                        ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
                        break;
                    default:
                        throw new LanguageException("Unrecognized file format: " + ae.getInputFormatType());
                }
            }
            output.add(ae);
        }
        if (current instanceof PrintStatement) {
            DataIdentifier target = createTarget();
            target.setDataType(DataType.SCALAR);
            target.setValueType(ValueType.STRING);
            target.setParseInfo(current);
            PrintStatement ps = (PrintStatement) current;
            PRINTTYPE ptype = ps.getType();
            try {
                if (ptype == PRINTTYPE.PRINT) {
                    Hop.OpOp1 op = Hop.OpOp1.PRINT;
                    Expression source = ps.getExpressions().get(0);
                    Hop ae = processExpression(source, target, ids);
                    Hop printHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae);
                    printHop.setParseInfo(current);
                    output.add(printHop);
                } else if (ptype == PRINTTYPE.ASSERT) {
                    Hop.OpOp1 op = Hop.OpOp1.ASSERT;
                    Expression source = ps.getExpressions().get(0);
                    Hop ae = processExpression(source, target, ids);
                    Hop printHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae);
                    printHop.setParseInfo(current);
                    output.add(printHop);
                } else if (ptype == PRINTTYPE.STOP) {
                    Hop.OpOp1 op = Hop.OpOp1.STOP;
                    Expression source = ps.getExpressions().get(0);
                    Hop ae = processExpression(source, target, ids);
                    Hop stopHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae);
                    stopHop.setParseInfo(current);
                    output.add(stopHop);
                    // avoid merge
                    sb.setSplitDag(true);
                } else if (ptype == PRINTTYPE.PRINTF) {
                    List<Expression> expressions = ps.getExpressions();
                    Hop[] inHops = new Hop[expressions.size()];
                    // Hop (ie, MultipleOp) as input Hops
                    for (int j = 0; j < expressions.size(); j++) {
                        Hop inHop = processExpression(expressions.get(j), target, ids);
                        inHops[j] = inHop;
                    }
                    target.setValueType(ValueType.STRING);
                    Hop printfHop = new NaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOpN.PRINTF, inHops);
                    output.add(printfHop);
                }
            } catch (HopsException e) {
                throw new LanguageException(e);
            }
        }
        if (current instanceof AssignmentStatement) {
            AssignmentStatement as = (AssignmentStatement) current;
            DataIdentifier target = as.getTarget();
            Expression source = as.getSource();
            // CASE: regular assignment statement -- source is DML expression that is NOT user-defined or external function
            if (!(source instanceof FunctionCallIdentifier)) {
                // CASE: target is regular data identifier
                if (!(target instanceof IndexedIdentifier)) {
                    // process right hand side and accumulation
                    Hop ae = processExpression(source, target, ids);
                    if (((AssignmentStatement) current).isAccumulator()) {
                        DataIdentifier accum = liveIn.getVariable(target.getName());
                        if (accum == null)
                            throw new LanguageException("Invalid accumulator assignment " + "to non-existing variable " + target.getName() + ".");
                        ae = HopRewriteUtils.createBinary(ids.get(target.getName()), ae, OpOp2.PLUS);
                        target.setProperties(accum.getOutput());
                    } else
                        target.setProperties(source.getOutput());
                    ids.put(target.getName(), ae);
                    // add transient write if needed
                    Integer statementId = liveOutToTemp.get(target.getName());
                    if ((statementId != null) && (statementId.intValue() == i)) {
                        DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, DataOpTypes.TRANSIENTWRITE, null);
                        transientwrite.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), ae.getRowsInBlock(), ae.getColsInBlock());
                        transientwrite.setParseInfo(target);
                        updatedLiveOut.addVariable(target.getName(), target);
                        output.add(transientwrite);
                    }
                } else // CASE: target is indexed identifier (left-hand side indexed expression)
                {
                    Hop ae = processLeftIndexedExpression(source, (IndexedIdentifier) target, ids);
                    ids.put(target.getName(), ae);
                    // obtain origDim values BEFORE they are potentially updated during setProperties call
                    // (this is incorrect for LHS Indexing)
                    long origDim1 = ((IndexedIdentifier) target).getOrigDim1();
                    long origDim2 = ((IndexedIdentifier) target).getOrigDim2();
                    target.setProperties(source.getOutput());
                    ((IndexedIdentifier) target).setOriginalDimensions(origDim1, origDim2);
                    // (required for scalar input to left indexing)
                    if (target.getDataType() != DataType.MATRIX) {
                        target.setDataType(DataType.MATRIX);
                        target.setValueType(ValueType.DOUBLE);
                        target.setBlockDimensions(ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
                    }
                    Integer statementId = liveOutToTemp.get(target.getName());
                    if ((statementId != null) && (statementId.intValue() == i)) {
                        DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, DataOpTypes.TRANSIENTWRITE, null);
                        transientwrite.setOutputParams(origDim1, origDim2, ae.getNnz(), ae.getUpdateType(), ae.getRowsInBlock(), ae.getColsInBlock());
                        transientwrite.setParseInfo(target);
                        updatedLiveOut.addVariable(target.getName(), target);
                        output.add(transientwrite);
                    }
                }
            } else {
                // assignment, function call
                FunctionCallIdentifier fci = (FunctionCallIdentifier) source;
                FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(), fci.getName());
                // error handling missing function
                if (fsb == null) {
                    String error = source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace();
                    LOG.error(error);
                    throw new LanguageException(error);
                }
                // error handling unsupported function call in indexing expression
                if (target instanceof IndexedIdentifier) {
                    String fkey = DMLProgram.constructFunctionKey(fci.getNamespace(), fci.getName());
                    throw new LanguageException("Unsupported function call to '" + fkey + "' in left indexing expression. " + "Please, assign the function output to a variable.");
                }
                ArrayList<Hop> finputs = new ArrayList<>();
                for (ParameterExpression paramName : fci.getParamExprs()) {
                    Hop in = processExpression(paramName.getExpr(), null, ids);
                    finputs.add(in);
                }
                // create function op
                FunctionType ftype = fsb.getFunctionOpType();
                FunctionOp fcall = (target == null) ? new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new String[] {}, false) : new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new String[] { target.getName() }, false);
                output.add(fcall);
            // TODO function output dataops (phase 3)
            // DataOp trFoutput = new DataOp(target.getName(), target.getDataType(), target.getValueType(), fcall, DataOpTypes.FUNCTIONOUTPUT, null);
            // DataOp twFoutput = new DataOp(target.getName(), target.getDataType(), target.getValueType(), trFoutput, DataOpTypes.TRANSIENTWRITE, null);
            }
        } else if (current instanceof MultiAssignmentStatement) {
            // multi-assignment, by definition a function call
            MultiAssignmentStatement mas = (MultiAssignmentStatement) current;
            Expression source = mas.getSource();
            if (source instanceof FunctionCallIdentifier) {
                FunctionCallIdentifier fci = (FunctionCallIdentifier) source;
                FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(), fci.getName());
                FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
                if (fstmt == null) {
                    LOG.error(source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace());
                    throw new LanguageException(source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace());
                }
                ArrayList<Hop> finputs = new ArrayList<>();
                for (ParameterExpression paramName : fci.getParamExprs()) {
                    Hop in = processExpression(paramName.getExpr(), null, ids);
                    finputs.add(in);
                }
                // create function op
                String[] foutputs = mas.getTargetList().stream().map(d -> d.getName()).toArray(String[]::new);
                FunctionType ftype = fsb.getFunctionOpType();
                FunctionOp fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, foutputs, false);
                output.add(fcall);
            // TODO function output dataops (phase 3)
            /*for ( DataIdentifier paramName : mas.getTargetList() ){
						DataOp twFoutput = new DataOp(paramName.getName(), paramName.getDataType(), paramName.getValueType(), fcall, DataOpTypes.TRANSIENTWRITE, null);
						output.add(twFoutput);
					}*/
            } else if (source instanceof BuiltinFunctionExpression && ((BuiltinFunctionExpression) source).multipleReturns()) {
                // construct input hops
                Hop fcall = processMultipleReturnBuiltinFunctionExpression((BuiltinFunctionExpression) source, mas.getTargetList(), ids);
                output.add(fcall);
            } else if (source instanceof ParameterizedBuiltinFunctionExpression && ((ParameterizedBuiltinFunctionExpression) source).multipleReturns()) {
                // construct input hops
                Hop fcall = processMultipleReturnParameterizedBuiltinFunctionExpression((ParameterizedBuiltinFunctionExpression) source, mas.getTargetList(), ids);
                output.add(fcall);
            } else
                throw new LanguageException("Class \"" + source.getClass() + "\" is not supported in Multiple Assignment statements");
        }
    }
    sb.updateLiveVariablesOut(updatedLiveOut);
    sb.setHops(output);
}