Examples with DataOp org.apache.sysml.hops.DataOp used on opensource projects

Example 11 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class DMLTranslator method constructHopsForIterablePredicate.

/**
 * Constructs all predicate Hops (for FROM, TO, INCREMENT) of an iterable predicate
 * and assigns these Hops to the passed statement block.
 *
 * Method used for both ForStatementBlock and ParForStatementBlock.
 *
 * @param fsb for statement block
 */
public void constructHopsForIterablePredicate(ForStatementBlock fsb) {
    HashMap<String, Hop> _ids = new HashMap<>();
    // set iterable predicate
    ForStatement fs = (ForStatement) fsb.getStatement(0);
    IterablePredicate ip = fs.getIterablePredicate();
    for (int i = 0; i < 3; i++) {
        Expression expr = (i == 0) ? ip.getFromExpr() : (i == 1) ? ip.getToExpr() : (ip.getIncrementExpr() != null) ? ip.getIncrementExpr() : null;
        VariableSet varsRead = (expr != null) ? expr.variablesRead() : null;
        if (varsRead != null) {
            for (String varName : varsRead.getVariables().keySet()) {
                DataIdentifier var = fsb.liveIn().getVariable(varName);
                DataOp read = null;
                if (var == null) {
                    LOG.error("variable '" + varName + "' is not available for iterable predicate");
                    throw new ParseException("variable '" + varName + "' is not available for iterable predicate");
                } else {
                    long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim1() : var.getDim1();
                    long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim2() : var.getDim2();
                    read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
                    read.setParseInfo(var);
                }
                _ids.put(varName, read);
            }
        }
        // create transient write to internal variable name on top of expression
        // in order to ensure proper instruction generation
        Hop predicateHops = processTempIntExpression(expr, _ids);
        if (predicateHops != null)
            predicateHops = HopRewriteUtils.createDataOp(ProgramBlock.PRED_VAR, predicateHops, DataOpTypes.TRANSIENTWRITE);
        // construct hops for from, to, and increment expressions
        if (i == 0)
            fsb.setFromHops(predicateHops);
        else if (i == 1)
            fsb.setToHops(predicateHops);
        else if (ip.getIncrementExpr() != null)
            fsb.setIncrementHops(predicateHops);
    }
}

Example 12 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class DMLTranslator method constructHopsForConditionalPredicate.

public void constructHopsForConditionalPredicate(StatementBlock passedSB) {
    HashMap<String, Hop> _ids = new HashMap<>();
    // set conditional predicate
    ConditionalPredicate cp = null;
    if (passedSB instanceof WhileStatementBlock) {
        WhileStatement ws = (WhileStatement) ((WhileStatementBlock) passedSB).getStatement(0);
        cp = ws.getConditionalPredicate();
    } else if (passedSB instanceof IfStatementBlock) {
        IfStatement ws = (IfStatement) ((IfStatementBlock) passedSB).getStatement(0);
        cp = ws.getConditionalPredicate();
    } else {
        throw new ParseException("ConditionalPredicate expected only for while or if statements.");
    }
    VariableSet varsRead = cp.variablesRead();
    for (String varName : varsRead.getVariables().keySet()) {
        // creating transient read for live in variables
        DataIdentifier var = passedSB.liveIn().getVariables().get(varName);
        DataOp read = null;
        if (var == null) {
            LOG.error("variable " + varName + " not live variable for conditional predicate");
            throw new ParseException("variable " + varName + " not live variable for conditional predicate");
        } else {
            long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim1() : var.getDim1();
            long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier) var).getOrigDim2() : var.getDim2();
            read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
            read.setParseInfo(var);
        }
        _ids.put(varName, read);
    }
    DataIdentifier target = new DataIdentifier(Expression.getTempName());
    target.setDataType(DataType.SCALAR);
    target.setValueType(ValueType.BOOLEAN);
    target.setParseInfo(passedSB);
    Hop predicateHops = null;
    Expression predicate = cp.getPredicate();
    if (predicate instanceof RelationalExpression) {
        predicateHops = processRelationalExpression((RelationalExpression) cp.getPredicate(), target, _ids);
    } else if (predicate instanceof BooleanExpression) {
        predicateHops = processBooleanExpression((BooleanExpression) cp.getPredicate(), target, _ids);
    } else if (predicate instanceof DataIdentifier) {
        // handle data identifier predicate
        predicateHops = processExpression(cp.getPredicate(), null, _ids);
    } else if (predicate instanceof ConstIdentifier) {
        // b) disallow string values
        if ((predicate instanceof IntIdentifier && ((IntIdentifier) predicate).getValue() == 0) || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier) predicate).getValue() == 0.0)) {
            cp.setPredicate(new BooleanIdentifier(false, predicate));
        } else if ((predicate instanceof IntIdentifier && ((IntIdentifier) predicate).getValue() == 1) || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier) predicate).getValue() == 1.0)) {
            cp.setPredicate(new BooleanIdentifier(true, predicate));
        } else if (predicate instanceof IntIdentifier || predicate instanceof DoubleIdentifier) {
            cp.setPredicate(new BooleanIdentifier(true, predicate));
            LOG.warn(predicate.printWarningLocation() + "Numerical value '" + predicate.toString() + "' (!= 0/1) is converted to boolean TRUE by DML");
        } else if (predicate instanceof StringIdentifier) {
            LOG.error(predicate.printErrorLocation() + "String value '" + predicate.toString() + "' is not allowed for iterable predicate");
            throw new ParseException(predicate.printErrorLocation() + "String value '" + predicate.toString() + "' is not allowed for iterable predicate");
        }
        predicateHops = processExpression(cp.getPredicate(), null, _ids);
    }
    // create transient write to internal variable name on top of expression
    // in order to ensure proper instruction generation
    predicateHops = HopRewriteUtils.createDataOp(ProgramBlock.PRED_VAR, predicateHops, DataOpTypes.TRANSIENTWRITE);
    if (passedSB instanceof WhileStatementBlock)
        ((WhileStatementBlock) passedSB).setPredicateHops(predicateHops);
    else if (passedSB instanceof IfStatementBlock)
        ((IfStatementBlock) passedSB).setPredicateHops(predicateHops);
}

Example 13 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class DMLTranslator method processExpression.

/**
 * Construct Hops from parse tree : Process Expression in an assignment
 * statement
 *
 * @param source source expression
 * @param target data identifier
 * @param hops map of high-level operators
 * @return high-level operator
 */
private Hop processExpression(Expression source, DataIdentifier target, HashMap<String, Hop> hops) {
    try {
        if (source instanceof BinaryExpression)
            return processBinaryExpression((BinaryExpression) source, target, hops);
        else if (source instanceof RelationalExpression)
            return processRelationalExpression((RelationalExpression) source, target, hops);
        else if (source instanceof BooleanExpression)
            return processBooleanExpression((BooleanExpression) source, target, hops);
        else if (source instanceof BuiltinFunctionExpression)
            return processBuiltinFunctionExpression((BuiltinFunctionExpression) source, target, hops);
        else if (source instanceof ParameterizedBuiltinFunctionExpression)
            return processParameterizedBuiltinFunctionExpression((ParameterizedBuiltinFunctionExpression) source, target, hops);
        else if (source instanceof DataExpression) {
            Hop ae = (Hop) processDataExpression((DataExpression) source, target, hops);
            if (ae instanceof DataOp) {
                String formatName = ((DataExpression) source).getVarParam(DataExpression.FORMAT_TYPE).toString();
                ((DataOp) ae).setInputFormatType(Expression.convertFormatType(formatName));
            }
            return ae;
        } else if (source instanceof IndexedIdentifier)
            return processIndexingExpression((IndexedIdentifier) source, target, hops);
        else if (source instanceof IntIdentifier) {
            IntIdentifier sourceInt = (IntIdentifier) source;
            LiteralOp litop = new LiteralOp(sourceInt.getValue());
            litop.setParseInfo(sourceInt);
            setIdentifierParams(litop, sourceInt);
            return litop;
        } else if (source instanceof DoubleIdentifier) {
            DoubleIdentifier sourceDouble = (DoubleIdentifier) source;
            LiteralOp litop = new LiteralOp(sourceDouble.getValue());
            litop.setParseInfo(sourceDouble);
            setIdentifierParams(litop, sourceDouble);
            return litop;
        } else if (source instanceof BooleanIdentifier) {
            BooleanIdentifier sourceBoolean = (BooleanIdentifier) source;
            LiteralOp litop = new LiteralOp(sourceBoolean.getValue());
            litop.setParseInfo(sourceBoolean);
            setIdentifierParams(litop, sourceBoolean);
            return litop;
        } else if (source instanceof StringIdentifier) {
            StringIdentifier sourceString = (StringIdentifier) source;
            LiteralOp litop = new LiteralOp(sourceString.getValue());
            litop.setParseInfo(sourceString);
            setIdentifierParams(litop, sourceString);
            return litop;
        } else if (source instanceof DataIdentifier)
            return hops.get(((DataIdentifier) source).getName());
    } catch (Exception e) {
        throw new ParseException(e.getMessage());
    }
    return null;
}

Example 14 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class DMLTranslator method processMultipleReturnParameterizedBuiltinFunctionExpression.

private Hop processMultipleReturnParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFunctionExpression source, ArrayList<DataIdentifier> targetList, HashMap<String, Hop> hops) {
    FunctionType ftype = FunctionType.MULTIRETURN_BUILTIN;
    String nameSpace = DMLProgram.INTERNAL_NAMESPACE;
    // Create an array list to hold the outputs of this lop.
    // Exact list of outputs are added based on opcode.
    ArrayList<Hop> outputs = new ArrayList<>();
    // Construct Hop for current builtin function expression based on its type
    Hop currBuiltinOp = null;
    switch(source.getOpCode()) {
        case TRANSFORMENCODE:
            ArrayList<Hop> inputs = new ArrayList<>();
            inputs.add(processExpression(source.getVarParam("target"), null, hops));
            inputs.add(processExpression(source.getVarParam("spec"), null, hops));
            String[] outputNames = new String[targetList.size()];
            outputNames[0] = ((DataIdentifier) targetList.get(0)).getName();
            outputNames[1] = ((DataIdentifier) targetList.get(1)).getName();
            outputs.add(new DataOp(outputNames[0], DataType.MATRIX, ValueType.DOUBLE, inputs.get(0), DataOpTypes.FUNCTIONOUTPUT, outputNames[0]));
            outputs.add(new DataOp(outputNames[1], DataType.FRAME, ValueType.STRING, inputs.get(0), DataOpTypes.FUNCTIONOUTPUT, outputNames[1]));
            currBuiltinOp = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), inputs, outputNames, outputs);
            break;
        default:
            throw new ParseException("Invaid Opcode in DMLTranslator:processMultipleReturnParameterizedBuiltinFunctionExpression(): " + source.getOpCode());
    }
    // set properties for created hops based on outputs of source expression
    for (int i = 0; i < source.getOutputs().length; i++) {
        setIdentifierParams(outputs.get(i), source.getOutputs()[i]);
        outputs.get(i).setParseInfo(source);
    }
    currBuiltinOp.setParseInfo(source);
    return currBuiltinOp;
}

Example 15 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class RewriteBlockSizeAndReblock method rule_BlockSizeAndReblock.

private void rule_BlockSizeAndReblock(Hop hop, final int blocksize) {
    // Go to the source(s) of the DAG
    for (Hop hi : hop.getInput()) {
        if (!hi.isVisited())
            rule_BlockSizeAndReblock(hi, blocksize);
    }
    boolean canReblock = isReblockValid();
    if (hop instanceof DataOp) {
        DataOp dop = (DataOp) hop;
        // if block size does not match
        if (canReblock && ((dop.getDataType() == DataType.MATRIX && (dop.getRowsInBlock() != blocksize || dop.getColsInBlock() != blocksize)) || (dop.getDataType() == DataType.FRAME && OptimizerUtils.isSparkExecutionMode() && (dop.getInputFormatType() == FileFormatTypes.TEXT || dop.getInputFormatType() == FileFormatTypes.CSV)))) {
            if (dop.getDataOpType() == DataOp.DataOpTypes.PERSISTENTREAD) {
                // insert reblock after the hop
                dop.setRequiresReblock(true);
                dop.setOutputBlocksizes(blocksize, blocksize);
            } else if (dop.getDataOpType() == DataOp.DataOpTypes.PERSISTENTWRITE) {
                if (dop.getRowsInBlock() == -1 && dop.getColsInBlock() == -1) {
                // if this dataop is for cell output, then no reblock is needed
                // as (A) all jobtypes can produce block2cell and cell2cell and
                // (B) we don't generate an explicit instruction for it (the info
                // is conveyed through OutputInfo.
                } else if (dop.getInput().get(0).requiresReblock() && dop.getInput().get(0).getParent().size() == 1) {
                    // if a reblock is feeding into this, then use it if this is
                    // the only parent, otherwise new Reblock
                    dop.getInput().get(0).setOutputBlocksizes(dop.getRowsInBlock(), dop.getColsInBlock());
                } else {
                    // insert reblock after the hop
                    dop.setRequiresReblock(true);
                    dop.setOutputBlocksizes(blocksize, blocksize);
                }
            } else if (dop.getDataOpType() == DataOp.DataOpTypes.TRANSIENTWRITE || dop.getDataOpType() == DataOp.DataOpTypes.TRANSIENTREAD) {
                if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE) {
                    // simply copy the values from its input
                    dop.setRowsInBlock(hop.getInput().get(0).getRowsInBlock());
                    dop.setColsInBlock(hop.getInput().get(0).getColsInBlock());
                } else {
                    // by default, all transient reads and writes are in blocked format
                    dop.setRowsInBlock(blocksize);
                    dop.setColsInBlock(blocksize);
                }
            } else {
                throw new HopsException(hop.printErrorLocation() + "unexpected non-scalar Data HOP in reblock.\n");
            }
        }
    } else // NO DATAOP
    {
        if (hop.requiresReblock()) {
            hop.setRowsInBlock(blocksize);
            hop.setColsInBlock(blocksize);
        } else // Constraint C2:
        if (hop.getDataType() == DataType.SCALAR) {
            hop.setRowsInBlock(-1);
            hop.setColsInBlock(-1);
        } else // Constraint C3:
        {
            if (!canReblock) {
                hop.setRowsInBlock(-1);
                hop.setColsInBlock(-1);
            } else {
                hop.setRowsInBlock(blocksize);
                hop.setColsInBlock(blocksize);
                // Reblock properties need to be set for each output.
                if (hop instanceof FunctionOp) {
                    FunctionOp fop = (FunctionOp) hop;
                    if (fop.getOutputs() != null) {
                        for (Hop out : fop.getOutputs()) {
                            out.setRowsInBlock(blocksize);
                            out.setColsInBlock(blocksize);
                        }
                    }
                }
            }
            // if any input is not blocked then the output of current Hop should not be blocked
            for (Hop h : hop.getInput()) {
                if (h.getDataType() == DataType.MATRIX && h.getRowsInBlock() == -1 && h.getColsInBlock() == -1) {
                    hop.setRowsInBlock(-1);
                    hop.setColsInBlock(-1);
                    break;
                }
            }
        }
    }
    hop.setVisited();
}