Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 76 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project systemml by apache.

the class RemoteParForUtils method exportResultVariables.

/**
 * For remote MR parfor workers.
 *
 * @param workerID worker id
 * @param vars local variable map
 * @param resultVars list of result variables
 * @param rvarFnames ?
 * @param out output collectors
 * @throws IOException if IOException occurs
 */
public static void exportResultVariables(long workerID, LocalVariableMap vars, ArrayList<ResultVar> resultVars, HashMap<String, String> rvarFnames, OutputCollector<Writable, Writable> out) throws IOException {
    // create key and value for reuse
    LongWritable okey = new LongWritable(workerID);
    Text ovalue = new Text();
    // foreach result variables probe if export necessary
    for (ResultVar rvar : resultVars) {
        Data dat = vars.get(rvar._name);
        // export output variable to HDFS (see RunMRJobs)
        if (dat != null && dat.getDataType() == DataType.MATRIX) {
            MatrixObject mo = (MatrixObject) dat;
            if (mo.isDirty()) {
                if (ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER && rvarFnames != null) {
                    String fname = rvarFnames.get(rvar._name);
                    if (fname != null)
                        mo.setFileName(fname);
                    // export result var (iff actually modified in parfor)
                    // note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
                    mo.exportData();
                    rvarFnames.put(rvar._name, mo.getFileName());
                } else {
                    // export result var (iff actually modified in parfor)
                    // note: this is equivalent to doing it in close (currently not required because 1 Task=1Map tasks, hence only one map invocation)
                    mo.exportData();
                }
                // pass output vars (scalars by value, matrix by ref) to result
                // (only if actually exported, hence in check for dirty, otherwise potential problems in result merge)
                String datStr = ProgramConverter.serializeDataObject(rvar._name, mo);
                ovalue.set(datStr);
                out.collect(okey, ovalue);
            }
        }
    }
}

Example 77 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project systemml by apache.

the class Script method in.

/**
 * Register an input (parameter ($) or variable) with optional matrix
 * metadata.
 *
 * @param name
 *            name of the input
 * @param value
 *            value of the input
 * @param metadata
 *            optional matrix/frame metadata
 * @return {@code this} Script object to allow chaining of methods
 */
public Script in(String name, Object value, Metadata metadata) {
    if ((value != null) && (value instanceof Long)) {
        // convert Long to Integer since Long not a supported value type
        Long lng = (Long) value;
        value = lng.intValue();
    } else if ((value != null) && (value instanceof Float)) {
        // convert Float to Double since Float not a supported value type
        Float flt = (Float) value;
        value = flt.doubleValue();
    }
    MLContextUtil.checkInputValueType(name, value);
    if (inputs == null) {
        inputs = new LinkedHashMap<>();
    }
    inputs.put(name, value);
    if (name.startsWith("$")) {
        MLContextUtil.checkInputParameterType(name, value);
        if (inputParameters == null) {
            inputParameters = new LinkedHashMap<>();
        }
        inputParameters.put(name, value);
    } else {
        Data data = MLContextUtil.convertInputType(name, value, metadata);
        if (data != null) {
            // store input variable name and data
            symbolTable.put(name, data);
            inputVariables.add(name);
            // store matrix/frame meta data and disable variable cleanup
            if (data instanceof CacheableData) {
                if (metadata != null)
                    inputMetadata.put(name, metadata);
                ((CacheableData<?>) data).enableCleanup(false);
            }
        }
    }
    return this;
}

Example 78 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project systemml by apache.

the class InterProceduralAnalysis method populateLocalVariableMapForFunctionCall.

private static void populateLocalVariableMapForFunctionCall(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callvars, LocalVariableMap vars, FunctionCallSizeInfo fcallSizes) {
    ArrayList<DataIdentifier> inputVars = fstmt.getInputParams();
    ArrayList<Hop> inputOps = fop.getInput();
    String fkey = fop.getFunctionKey();
    for (int i = 0; i < inputVars.size(); i++) {
        // create mapping between input hops and vars
        DataIdentifier dat = inputVars.get(i);
        Hop input = inputOps.get(i);
        if (input.getDataType() == DataType.MATRIX) {
            // propagate matrix characteristics
            MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
            MatrixCharacteristics mc = new MatrixCharacteristics(input.getDim1(), input.getDim2(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), fcallSizes.isSafeNnz(fkey, i) ? input.getNnz() : -1);
            MetaDataFormat meta = new MetaDataFormat(mc, null, null);
            mo.setMetaData(meta);
            vars.put(dat.getName(), mo);
        } else if (input.getDataType() == DataType.SCALAR) {
            // (for multiple calls, literal equivalence already checked)
            if (input instanceof LiteralOp) {
                vars.put(dat.getName(), ScalarObjectFactory.createScalarObject(input.getValueType(), (LiteralOp) input));
            } else // and input scalar is existing variable in symbol table
            if (PROPAGATE_SCALAR_VARS_INTO_FUN && fcallSizes.getFunctionCallCount(fkey) == 1 && input instanceof DataOp) {
                Data scalar = callvars.get(input.getName());
                if (scalar != null && scalar instanceof ScalarObject) {
                    vars.put(dat.getName(), scalar);
                }
            }
        }
    }
}

Example 79 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project systemml by apache.

the class LiteralReplacement method replaceLiteralDataTypeCastMatrixRead.

private static LiteralOp replaceLiteralDataTypeCastMatrixRead(Hop c, LocalVariableMap vars) {
    LiteralOp ret = null;
    // as.scalar/matrix read - literal replacement
    if (c instanceof UnaryOp && ((UnaryOp) c).getOp() == OpOp1.CAST_AS_SCALAR && c.getInput().get(0) instanceof DataOp && c.getInput().get(0).getDataType() == DataType.MATRIX) {
        Data dat = vars.get(c.getInput().get(0).getName());
        if (// required for selective constant propagation
        dat != null) {
            // cast as scalar (see VariableCPInstruction)
            MatrixObject mo = (MatrixObject) dat;
            MatrixBlock mBlock = mo.acquireRead();
            if (mBlock.getNumRows() != 1 || mBlock.getNumColumns() != 1)
                throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix of dimension (" + mBlock.getNumRows() + " x " + mBlock.getNumColumns() + ") to scalar.");
            double value = mBlock.getValue(0, 0);
            mo.release();
            // literal substitution (always double)
            ret = new LiteralOp(value);
        }
    }
    return ret;
}

Example 80 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project systemml by apache.

the class Recompiler method reconcileUpdatedCallVarsLoops.

public static boolean reconcileUpdatedCallVarsLoops(LocalVariableMap oldCallVars, LocalVariableMap callVars, StatementBlock sb) {
    boolean requiresRecompile = false;
    // handle matrices
    for (String varname : sb.variablesUpdated().getVariableNames()) {
        Data dat1 = oldCallVars.get(varname);
        Data dat2 = callVars.get(varname);
        if (dat1 != null && dat1 instanceof MatrixObject && dat2 != null && dat2 instanceof MatrixObject) {
            MatrixObject moOld = (MatrixObject) dat1;
            MatrixObject mo = (MatrixObject) dat2;
            MatrixCharacteristics mcOld = moOld.getMatrixCharacteristics();
            MatrixCharacteristics mc = mo.getMatrixCharacteristics();
            if (mcOld.getRows() != mc.getRows() || mcOld.getCols() != mc.getCols() || mcOld.getNonZeros() != mc.getNonZeros()) {
                long ldim1 = mc.getRows(), ldim2 = mc.getCols(), lnnz = mc.getNonZeros();
                // handle row dimension change in body
                if (mcOld.getRows() != mc.getRows()) {
                    // unknown
                    ldim1 = -1;
                    requiresRecompile = true;
                }
                // handle column dimension change in body
                if (mcOld.getCols() != mc.getCols()) {
                    // unknown
                    ldim2 = -1;
                    requiresRecompile = true;
                }
                // handle sparsity change
                if (mcOld.getNonZeros() != mc.getNonZeros()) {
                    // unknown
                    lnnz = -1;
                    requiresRecompile = true;
                }
                MatrixObject moNew = createOutputMatrix(ldim1, ldim2, lnnz);
                callVars.put(varname, moNew);
            }
        }
    }
    return requiresRecompile;
}