Examples with DataOp org.apache.sysml.hops.DataOp used on opensource projects

Example 41 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class GDFEnumOptimizer method costRuntimePlan.

private static double costRuntimePlan(Plan p) throws DMLRuntimeException {
    Program prog = p.getNode().getProgram();
    if (prog == null)
        throw new DMLRuntimeException("Program not available for runtime plan costing.");
    //put data flow configuration into program
    rSetRuntimePlanConfig(p, new HashMap<Long, Plan>());
    double costs = -1;
    if (COST_FULL_PROGRAMS || (p.getNode().getHop() == null || p.getNode().getProgramBlock() == null)) {
        //recompile entire runtime program
        Recompiler.recompileProgramBlockHierarchy(prog.getProgramBlocks(), new LocalVariableMap(), 0, false);
        _compiledPlans++;
        //cost entire runtime program
        ExecutionContext ec = ExecutionContextFactory.createContext(prog);
        costs = CostEstimationWrapper.getTimeEstimate(prog, ec);
    } else {
        Hop currentHop = p.getNode().getHop();
        ProgramBlock pb = p.getNode().getProgramBlock();
        try {
            //keep the old dag roots
            ArrayList<Hop> oldRoots = pb.getStatementBlock().get_hops();
            Hop tmpHop = null;
            if (!(currentHop instanceof DataOp && ((DataOp) currentHop).isWrite())) {
                ArrayList<Hop> newRoots = new ArrayList<Hop>();
                tmpHop = new DataOp("_tmp", currentHop.getDataType(), currentHop.getValueType(), currentHop, DataOpTypes.TRANSIENTWRITE, "tmp");
                //ensure recursive visitstatus reset on recompile
                tmpHop.setVisited();
                newRoots.add(tmpHop);
                pb.getStatementBlock().set_hops(newRoots);
            }
            //recompile modified runtime program
            Recompiler.recompileProgramBlockHierarchy(prog.getProgramBlocks(), new LocalVariableMap(), 0, false);
            _compiledPlans++;
            //cost partial runtime program up to current hop
            ExecutionContext ec = ExecutionContextFactory.createContext(prog);
            costs = CostEstimationWrapper.getTimeEstimate(prog, ec);
            //restore original hop dag
            if (tmpHop != null)
                HopRewriteUtils.removeChildReference(tmpHop, currentHop);
            pb.getStatementBlock().set_hops(oldRoots);
        } catch (HopsException ex) {
            throw new DMLRuntimeException(ex);
        }
    }
    //release forced data flow configuration from program
    rResetRuntimePlanConfig(p, new HashMap<Long, Plan>());
    _costedPlans++;
    return costs;
}

Example 42 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class GDFEnumOptimizer method rSetRuntimePlanConfig.

private static void rSetRuntimePlanConfig(Plan p, HashMap<Long, Plan> memo) {
    ExecType CLUSTER = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    //basic memoization including containment check 
    if (memo.containsKey(p.getNode().getID())) {
        Plan pmemo = memo.get(p.getNode().getID());
        if (!p.getInterestingProperties().equals(pmemo.getInterestingProperties())) {
            //TODO this would require additional cleanup in special cases
            if (_resolve.resolveMismatch(pmemo.getRewriteConfig(), p.getRewriteConfig()))
                memo.put(p.getNode().getID(), p);
            //logging of encounter plan mismatch
            LOG.warn("Configuration mismatch on shared node (" + p.getNode().getHop().getHopID() + "). Falling back to heuristic '" + _resolve.getName() + "'.");
            LOG.warn(p.getInterestingProperties().toString());
            LOG.warn(memo.get(p.getNode().getID()).getInterestingProperties());
            _planMismatches++;
            return;
        }
    }
    //set plan configuration
    Hop hop = p.getNode().getHop();
    if (hop != null) {
        RewriteConfig rc = p.getRewriteConfig();
        //set exec type
        hop.setForcedExecType(rc.getExecType());
        //set blocksizes and reblock
        hop.setRowsInBlock(rc.getBlockSize());
        hop.setColsInBlock(rc.getBlockSize());
        if (//after blocksize update
        rc.getExecType() == CLUSTER) {
            //TODO double check dataop condition - side effect from plan validity
            boolean reblock = HopRewriteUtils.alwaysRequiresReblock(hop) || (hop.hasMatrixInputWithDifferentBlocksizes() && !(hop instanceof DataOp));
            hop.setRequiresReblock(reblock);
        } else
            hop.setRequiresReblock(false);
    }
    //process childs
    if (p.getChilds() != null)
        for (Plan c : p.getChilds()) rSetRuntimePlanConfig(c, memo);
    //memoization (mark as processed)
    memo.put(p.getNode().getID(), p);
}

Example 43 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class DMLTranslator method processDataExpression.

/**
 * Construct Hops from parse tree : Process ParameterizedExpression in a
 * read/write/rand statement
 *
 * @param source data expression
 * @param target data identifier
 * @param hops map of high-level operators
 * @return high-level operator
 */
private Hop processDataExpression(DataExpression source, DataIdentifier target, HashMap<String, Hop> hops) {
    // this expression has multiple "named" parameters
    HashMap<String, Hop> paramHops = new HashMap<>();
    // -- construct hops for all input parameters
    // -- store them in hashmap so that their "name"s are maintained
    Hop pHop = null;
    for (String paramName : source.getVarParams().keySet()) {
        pHop = processExpression(source.getVarParam(paramName), null, hops);
        paramHops.put(paramName, pHop);
    }
    Hop currBuiltinOp = null;
    if (target == null) {
        target = createTarget(source);
    }
    // construct hop based on opcode
    switch(source.getOpCode()) {
        case READ:
            currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), DataOpTypes.PERSISTENTREAD, paramHops);
            ((DataOp) currBuiltinOp).setFileName(((StringIdentifier) source.getVarParam(DataExpression.IO_FILENAME)).getValue());
            break;
        case WRITE:
            currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), DataOpTypes.PERSISTENTWRITE, hops.get(target.getName()), paramHops);
            break;
        case RAND:
            // We limit RAND_MIN, RAND_MAX, RAND_SPARSITY, RAND_SEED, and RAND_PDF to be constants
            DataGenMethod method = (paramHops.get(DataExpression.RAND_MIN).getValueType() == ValueType.STRING) ? DataGenMethod.SINIT : DataGenMethod.RAND;
            currBuiltinOp = new DataGenOp(method, target, paramHops);
            break;
        case MATRIX:
            ArrayList<Hop> tmp = new ArrayList<>();
            tmp.add(0, paramHops.get(DataExpression.RAND_DATA));
            tmp.add(1, paramHops.get(DataExpression.RAND_ROWS));
            tmp.add(2, paramHops.get(DataExpression.RAND_COLS));
            tmp.add(3, paramHops.get(DataExpression.RAND_BY_ROW));
            currBuiltinOp = new ReorgOp(target.getName(), target.getDataType(), target.getValueType(), ReOrgOp.RESHAPE, tmp);
            break;
        default:
            LOG.error(source.printErrorLocation() + "processDataExpression():: Unknown operation:  " + source.getOpCode());
            throw new ParseException(source.printErrorLocation() + "processDataExpression():: Unknown operation:  " + source.getOpCode());
    }
    // set identifier meta data (incl dimensions and blocksizes)
    setIdentifierParams(currBuiltinOp, source.getOutput());
    if (source.getOpCode() == DataExpression.DataOp.READ)
        ((DataOp) currBuiltinOp).setInputBlockSizes(target.getRowsInBlock(), target.getColumnsInBlock());
    currBuiltinOp.setParseInfo(source);
    return currBuiltinOp;
}

Example 44 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class DMLTranslator method processMultipleReturnBuiltinFunctionExpression.

/**
 * Construct HOps from parse tree: process BuiltinFunction Expressions in
 * MultiAssignment Statements. For all other builtin function expressions,
 * <code>processBuiltinFunctionExpression()</code> is used.
 *
 * @param source built-in function expression
 * @param targetList list of data identifiers
 * @param hops map of high-level operators
 * @return high-level operator
 */
private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpression source, ArrayList<DataIdentifier> targetList, HashMap<String, Hop> hops) {
    // Construct Hops for all inputs
    ArrayList<Hop> inputs = new ArrayList<>();
    inputs.add(processExpression(source.getFirstExpr(), null, hops));
    if (source.getSecondExpr() != null)
        inputs.add(processExpression(source.getSecondExpr(), null, hops));
    if (source.getThirdExpr() != null)
        inputs.add(processExpression(source.getThirdExpr(), null, hops));
    FunctionType ftype = FunctionType.MULTIRETURN_BUILTIN;
    String nameSpace = DMLProgram.INTERNAL_NAMESPACE;
    // Create an array list to hold the outputs of this lop.
    // Exact list of outputs are added based on opcode.
    ArrayList<Hop> outputs = new ArrayList<>();
    // Construct Hop for current builtin function expression based on its type
    Hop currBuiltinOp = null;
    switch(source.getOpCode()) {
        case QR:
        case LU:
        case EIGEN:
        case SVD:
            // Number of outputs = size of targetList = #of identifiers in source.getOutputs
            String[] outputNames = new String[targetList.size()];
            for (int i = 0; i < targetList.size(); i++) {
                outputNames[i] = ((DataIdentifier) targetList.get(i)).getName();
                Hop output = new DataOp(outputNames[i], DataType.MATRIX, ValueType.DOUBLE, inputs.get(0), DataOpTypes.FUNCTIONOUTPUT, outputNames[i]);
                outputs.add(output);
            }
            // Create the hop for current function call
            FunctionOp fcall = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), inputs, outputNames, outputs);
            currBuiltinOp = fcall;
            break;
        default:
            throw new ParseException("Invaid Opcode in DMLTranslator:processMultipleReturnBuiltinFunctionExpression(): " + source.getOpCode());
    }
    // set properties for created hops based on outputs of source expression
    for (int i = 0; i < source.getOutputs().length; i++) {
        setIdentifierParams(outputs.get(i), source.getOutputs()[i]);
        outputs.get(i).setParseInfo(source);
    }
    currBuiltinOp.setParseInfo(source);
    return currBuiltinOp;
}

Example 45 with DataOp

use of org.apache.sysml.hops.DataOp in project incubator-systemml by apache.

the class RewriteConstantFolding method evalScalarOperation.

/**
 * In order to (1) prevent unexpected side effects from constant folding and
 * (2) for simplicity with regard to arbitrary value type combinations,
 * we use the same compilation and runtime for constant folding as we would
 * use for actual instruction execution.
 *
 * @param bop high-level operator
 * @return literal op
 */
private LiteralOp evalScalarOperation(Hop bop) {
    // Timing time = new Timing( true );
    DataOp tmpWrite = new DataOp(TMP_VARNAME, bop.getDataType(), bop.getValueType(), bop, DataOpTypes.TRANSIENTWRITE, TMP_VARNAME);
    // generate runtime instruction
    Dag<Lop> dag = new Dag<>();
    // prevent lops reuse
    Recompiler.rClearLops(tmpWrite);
    // reconstruct lops
    Lop lops = tmpWrite.constructLops();
    lops.addToDag(dag);
    ArrayList<Instruction> inst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
    // execute instructions
    ExecutionContext ec = getExecutionContext();
    ProgramBlock pb = getProgramBlock();
    pb.setInstructions(inst);
    pb.execute(ec);
    // get scalar result (check before invocation) and create literal according
    // to observed scalar output type (not hop type) for runtime consistency
    ScalarObject so = (ScalarObject) ec.getVariable(TMP_VARNAME);
    LiteralOp literal = null;
    switch(so.getValueType()) {
        case DOUBLE:
            literal = new LiteralOp(so.getDoubleValue());
            break;
        case INT:
            literal = new LiteralOp(so.getLongValue());
            break;
        case BOOLEAN:
            literal = new LiteralOp(so.getBooleanValue());
            break;
        case STRING:
            literal = new LiteralOp(so.getStringValue());
            break;
        default:
            throw new HopsException("Unsupported literal value type: " + bop.getValueType());
    }
    // cleanup
    tmpWrite.getInput().clear();
    bop.getParent().remove(tmpWrite);
    pb.setInstructions(null);
    ec.getVariables().removeAll();
    // set literal properties (scalar)
    HopRewriteUtils.setOutputParametersForScalar(literal);
    return literal;
}