Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 31 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class Recompiler method rUpdateStatistics.

public static void rUpdateStatistics(Hop hop, LocalVariableMap vars) {
    if (hop.isVisited())
        return;
    // recursively process children
    if (hop.getInput() != null)
        for (Hop c : hop.getInput()) rUpdateStatistics(c, vars);
    boolean updatedSizeExpr = false;
    // (with awareness not to override persistent reads to an existing name)
    if (hop instanceof DataOp && ((DataOp) hop).getDataOpType() != DataOpTypes.PERSISTENTREAD) {
        DataOp d = (DataOp) hop;
        String varName = d.getName();
        if (vars.keySet().contains(varName)) {
            Data dat = vars.get(varName);
            if (dat instanceof MatrixObject) {
                MatrixObject mo = (MatrixObject) dat;
                d.setDim1(mo.getNumRows());
                d.setDim2(mo.getNumColumns());
                d.setNnz(mo.getNnz());
            } else if (dat instanceof FrameObject) {
                FrameObject fo = (FrameObject) dat;
                d.setDim1(fo.getNumRows());
                d.setDim2(fo.getNumColumns());
            }
        }
    } else // special case for persistent reads with unknown size (read-after-write)
    if (hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.PERSISTENTREAD && !hop.dimsKnown() && ((DataOp) hop).getInputFormatType() != FileFormatTypes.CSV && !ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_READ_WRITE_METADATA)) {
        // update hop with read meta data
        DataOp dop = (DataOp) hop;
        tryReadMetaDataFileMatrixCharacteristics(dop);
    } else // update size expression for rand/seq according to symbol table entries
    if (hop instanceof DataGenOp) {
        DataGenOp d = (DataGenOp) hop;
        HashMap<String, Integer> params = d.getParamIndexMap();
        if (d.getOp() == DataGenMethod.RAND || d.getOp() == DataGenMethod.SINIT || d.getOp() == DataGenMethod.SAMPLE) {
            boolean initUnknown = !d.dimsKnown();
            int ix1 = params.get(DataExpression.RAND_ROWS);
            int ix2 = params.get(DataExpression.RAND_COLS);
            // update rows/cols by evaluating simple expression of literals, nrow, ncol, scalars, binaryops
            HashMap<Long, Long> memo = new HashMap<>();
            d.refreshRowsParameterInformation(d.getInput().get(ix1), vars, memo);
            d.refreshColsParameterInformation(d.getInput().get(ix2), vars, memo);
            updatedSizeExpr = initUnknown & d.dimsKnown();
        } else if (d.getOp() == DataGenMethod.SEQ) {
            boolean initUnknown = !d.dimsKnown();
            int ix1 = params.get(Statement.SEQ_FROM);
            int ix2 = params.get(Statement.SEQ_TO);
            int ix3 = params.get(Statement.SEQ_INCR);
            HashMap<Long, Double> memo = new HashMap<>();
            double from = d.computeBoundsInformation(d.getInput().get(ix1), vars, memo);
            double to = d.computeBoundsInformation(d.getInput().get(ix2), vars, memo);
            double incr = d.computeBoundsInformation(d.getInput().get(ix3), vars, memo);
            // special case increment
            if (from != Double.MAX_VALUE && to != Double.MAX_VALUE) {
                incr *= ((from > to && incr > 0) || (from < to && incr < 0)) ? -1.0 : 1.0;
            }
            if (from != Double.MAX_VALUE && to != Double.MAX_VALUE && incr != Double.MAX_VALUE) {
                d.setDim1(UtilFunctions.getSeqLength(from, to, incr));
                d.setDim2(1);
                d.setIncrementValue(incr);
            }
            updatedSizeExpr = initUnknown & d.dimsKnown();
        } else {
            throw new DMLRuntimeException("Unexpected data generation method: " + d.getOp());
        }
    } else // update size expression for reshape according to symbol table entries
    if (hop instanceof ReorgOp && ((ReorgOp) (hop)).getOp() == Hop.ReOrgOp.RESHAPE) {
        ReorgOp d = (ReorgOp) hop;
        boolean initUnknown = !d.dimsKnown();
        HashMap<Long, Long> memo = new HashMap<>();
        d.refreshRowsParameterInformation(d.getInput().get(1), vars, memo);
        d.refreshColsParameterInformation(d.getInput().get(2), vars, memo);
        updatedSizeExpr = initUnknown & d.dimsKnown();
    } else // update size expression for indexing according to symbol table entries
    if (hop instanceof IndexingOp) {
        IndexingOp iop = (IndexingOp) hop;
        // inpRowL
        Hop input2 = iop.getInput().get(1);
        // inpRowU
        Hop input3 = iop.getInput().get(2);
        // inpColL
        Hop input4 = iop.getInput().get(3);
        // inpColU
        Hop input5 = iop.getInput().get(4);
        boolean initUnknown = !iop.dimsKnown();
        HashMap<Long, Double> memo = new HashMap<>();
        double rl = iop.computeBoundsInformation(input2, vars, memo);
        double ru = iop.computeBoundsInformation(input3, vars, memo);
        double cl = iop.computeBoundsInformation(input4, vars, memo);
        double cu = iop.computeBoundsInformation(input5, vars, memo);
        if (rl != Double.MAX_VALUE && ru != Double.MAX_VALUE)
            iop.setDim1((long) (ru - rl + 1));
        if (cl != Double.MAX_VALUE && cu != Double.MAX_VALUE)
            iop.setDim2((long) (cu - cl + 1));
        updatedSizeExpr = initUnknown & iop.dimsKnown();
    }
    // without overwriting inferred size expressions
    if (!updatedSizeExpr) {
        hop.refreshSizeInformation();
    }
    hop.setVisited();
}

Example 32 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ProgramConverter method createDeepCopyExecutionContext.

// //////////////////////////////
// CREATION of DEEP COPIES
// //////////////////////////////
/**
 * Creates a deep copy of the given execution context.
 * For rt_platform=Hadoop, execution context has a symbol table.
 *
 * @param ec execution context
 * @return execution context
 * @throws CloneNotSupportedException if CloneNotSupportedException occurs
 */
public static ExecutionContext createDeepCopyExecutionContext(ExecutionContext ec) throws CloneNotSupportedException {
    ExecutionContext cpec = ExecutionContextFactory.createContext(false, ec.getProgram());
    cpec.setVariables((LocalVariableMap) ec.getVariables().clone());
    // (each worker requires its own copy of the empty matrix object)
    for (String var : cpec.getVariables().keySet()) {
        Data dat = cpec.getVariables().get(var);
        if (dat instanceof MatrixObject && ((MatrixObject) dat).getUpdateType().isInPlace()) {
            MatrixObject mo = (MatrixObject) dat;
            MatrixObject moNew = new MatrixObject(mo);
            if (mo.getNnz() != 0) {
                // If output matrix is not empty (NNZ != 0), then local copy is created so that
                // update in place operation can be applied.
                MatrixBlock mbVar = mo.acquireRead();
                moNew.acquireModify(new MatrixBlock(mbVar));
                mo.release();
            } else {
                // create empty matrix block w/ dense representation (preferred for update in-place)
                // Creating a dense matrix block is valid because empty block not allocated and transfer
                // to sparse representation happens in left indexing in place operation.
                moNew.acquireModify(new MatrixBlock((int) mo.getNumRows(), (int) mo.getNumColumns(), false));
            }
            moNew.release();
            cpec.setVariable(var, moNew);
        }
    }
    return cpec;
}

Example 33 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ProgramConverter method parseDataObject.

/**
 * NOTE: MRJobConfiguration cannot be used for the general case because program blocks and
 * related symbol tables can be hierarchically structured.
 *
 * @param in data object as string
 * @return array of objects
 */
public static Object[] parseDataObject(String in) {
    Object[] ret = new Object[2];
    StringTokenizer st = new StringTokenizer(in, DATA_FIELD_DELIM);
    String name = st.nextToken();
    DataType datatype = DataType.valueOf(st.nextToken());
    ValueType valuetype = ValueType.valueOf(st.nextToken());
    String valString = st.hasMoreTokens() ? st.nextToken() : "";
    Data dat = null;
    switch(datatype) {
        case SCALAR:
            {
                switch(valuetype) {
                    case INT:
                        dat = new IntObject(Long.parseLong(valString));
                        break;
                    case DOUBLE:
                        dat = new DoubleObject(Double.parseDouble(valString));
                        break;
                    case BOOLEAN:
                        dat = new BooleanObject(Boolean.parseBoolean(valString));
                        break;
                    case STRING:
                        dat = new StringObject(valString);
                        break;
                    default:
                        throw new DMLRuntimeException("Unable to parse valuetype " + valuetype);
                }
                break;
            }
        case MATRIX:
            {
                MatrixObject mo = new MatrixObject(valuetype, valString);
                long rows = Long.parseLong(st.nextToken());
                long cols = Long.parseLong(st.nextToken());
                int brows = Integer.parseInt(st.nextToken());
                int bcols = Integer.parseInt(st.nextToken());
                long nnz = Long.parseLong(st.nextToken());
                InputInfo iin = InputInfo.stringToInputInfo(st.nextToken());
                OutputInfo oin = OutputInfo.stringToOutputInfo(st.nextToken());
                PartitionFormat partFormat = PartitionFormat.valueOf(st.nextToken());
                UpdateType inplace = UpdateType.valueOf(st.nextToken());
                MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, brows, bcols, nnz);
                MetaDataFormat md = new MetaDataFormat(mc, oin, iin);
                mo.setMetaData(md);
                if (partFormat._dpf != PDataPartitionFormat.NONE)
                    mo.setPartitioned(partFormat._dpf, partFormat._N);
                mo.setUpdateType(inplace);
                dat = mo;
                break;
            }
        default:
            throw new DMLRuntimeException("Unable to parse datatype " + datatype);
    }
    ret[0] = name;
    ret[1] = dat;
    return ret;
}

Example 34 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class RemoteParForUtils method exportResultVariables.

/**
 * For remote Spark parfor workers. This is a simplified version compared to MR.
 *
 * @param workerID worker id
 * @param vars local variable map
 * @param resultVars list of result variables
 * @return list of result variables
 * @throws IOException if IOException occurs
 */
public static ArrayList<String> exportResultVariables(long workerID, LocalVariableMap vars, ArrayList<ResultVar> resultVars) throws IOException {
    ArrayList<String> ret = new ArrayList<>();
    // foreach result variables probe if export necessary
    for (ResultVar rvar : resultVars) {
        Data dat = vars.get(rvar._name);
        // export output variable to HDFS (see RunMRJobs)
        if (dat != null && dat.getDataType() == DataType.MATRIX) {
            MatrixObject mo = (MatrixObject) dat;
            if (mo.isDirty()) {
                // export result var (iff actually modified in parfor)
                mo.exportData();
                // pass output vars (scalars by value, matrix by ref) to result
                // (only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
                ret.add(ProgramConverter.serializeDataObject(rvar._name, mo));
            }
        }
    }
    return ret;
}

Example 35 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class RemoteParForUtils method exportResultVariables.

/**
 * For remote MR parfor workers.
 *
 * @param workerID worker id
 * @param vars local variable map
 * @param resultVars list of result variables
 * @param rvarFnames ?
 * @param out output collectors
 * @throws IOException if IOException occurs
 */
public static void exportResultVariables(long workerID, LocalVariableMap vars, ArrayList<ResultVar> resultVars, HashMap<String, String> rvarFnames, OutputCollector<Writable, Writable> out) throws IOException {
    // create key and value for reuse
    LongWritable okey = new LongWritable(workerID);
    Text ovalue = new Text();
    // foreach result variables probe if export necessary
    for (ResultVar rvar : resultVars) {
        Data dat = vars.get(rvar._name);
        // export output variable to HDFS (see RunMRJobs)
        if (dat != null && dat.getDataType() == DataType.MATRIX) {
            MatrixObject mo = (MatrixObject) dat;
            if (mo.isDirty()) {
                if (ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER && rvarFnames != null) {
                    String fname = rvarFnames.get(rvar._name);
                    if (fname != null)
                        mo.setFileName(fname);
                    // export result var (iff actually modified in parfor)
                    // note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
                    mo.exportData();
                    rvarFnames.put(rvar._name, mo.getFileName());
                } else {
                    // export result var (iff actually modified in parfor)
                    // note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
                    mo.exportData();
                }
                // pass output vars (scalars by value, matrix by ref) to result
                // (only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
                String datStr = ProgramConverter.serializeDataObject(rvar._name, mo);
                ovalue.set(datStr);
                out.collect(okey, ovalue);
            }
        }
    }
}