Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 26 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class Script method in.

/**
 * Register an input (parameter ($) or variable) with optional matrix
 * metadata.
 *
 * @param name
 *            name of the input
 * @param value
 *            value of the input
 * @param metadata
 *            optional matrix/frame metadata
 * @return {@code this} Script object to allow chaining of methods
 */
public Script in(String name, Object value, Metadata metadata) {
    if ((value != null) && (value instanceof Long)) {
        // convert Long to Integer since Long not a supported value type
        Long lng = (Long) value;
        value = lng.intValue();
    } else if ((value != null) && (value instanceof Float)) {
        // convert Float to Double since Float not a supported value type
        Float flt = (Float) value;
        value = flt.doubleValue();
    }
    MLContextUtil.checkInputValueType(name, value);
    if (inputs == null) {
        inputs = new LinkedHashMap<>();
    }
    inputs.put(name, value);
    if (name.startsWith("$")) {
        MLContextUtil.checkInputParameterType(name, value);
        if (inputParameters == null) {
            inputParameters = new LinkedHashMap<>();
        }
        inputParameters.put(name, value);
    } else {
        Data data = MLContextUtil.convertInputType(name, value, metadata);
        if (data != null) {
            // store input variable name and data
            symbolTable.put(name, data);
            inputVariables.add(name);
            // store matrix/frame meta data and disable variable cleanup
            if (data instanceof CacheableData) {
                if (metadata != null)
                    inputMetadata.put(name, metadata);
                ((CacheableData<?>) data).enableCleanup(false);
            }
        }
    }
    return this;
}

Example 27 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class InterProceduralAnalysis method extractFunctionCallReturnStatistics.

/**
 * Extract return variable statistics from this function into the
 * calling program.
 *
 * @param fstmt  The function statement.
 * @param fop  The function op.
 * @param tmpVars  Function's map of variables eligible for
 *                    extraction.
 * @param callVars  Calling program's map of variables.
 * @param overwrite  Whether or not to overwrite variables in the
 *                      calling program's variable map.
 */
private static void extractFunctionCallReturnStatistics(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap tmpVars, LocalVariableMap callVars, boolean overwrite) {
    ArrayList<DataIdentifier> foutputOps = fstmt.getOutputParams();
    String[] outputVars = fop.getOutputVariableNames();
    String fkey = fop.getFunctionKey();
    try {
        for (int i = 0; i < foutputOps.size(); i++) {
            DataIdentifier di = foutputOps.get(i);
            // name in function signature
            String fvarname = di.getName();
            // name in calling program
            String pvarname = outputVars[i];
            // output, remove that variable from the calling program's variable map.
            if (callVars.keySet().contains(pvarname)) {
                DataType fdataType = di.getDataType();
                DataType pdataType = callVars.get(pvarname).getDataType();
                if (fdataType != pdataType) {
                    // datatype has changed, and the calling program is reassigning the
                    // the variable, so remove it from the calling variable map
                    callVars.remove(pvarname);
                }
            }
            // Update or add to the calling program's variable map.
            if (di.getDataType() == DataType.MATRIX && tmpVars.keySet().contains(fvarname)) {
                MatrixObject moIn = (MatrixObject) tmpVars.get(fvarname);
                if (// not existing so far
                !callVars.keySet().contains(pvarname) || overwrite) {
                    MatrixObject moOut = createOutputMatrix(moIn.getNumRows(), moIn.getNumColumns(), moIn.getNnz());
                    callVars.put(pvarname, moOut);
                } else // already existing: take largest
                {
                    Data dat = callVars.get(pvarname);
                    if (dat instanceof MatrixObject) {
                        MatrixObject moOut = (MatrixObject) dat;
                        MatrixCharacteristics mc = moOut.getMatrixCharacteristics();
                        if (OptimizerUtils.estimateSizeExactSparsity(mc.getRows(), mc.getCols(), (mc.getNonZeros() > 0) ? OptimizerUtils.getSparsity(mc) : 1.0) < OptimizerUtils.estimateSize(moIn.getNumRows(), moIn.getNumColumns())) {
                            // update statistics if necessary
                            mc.setDimension(moIn.getNumRows(), moIn.getNumColumns());
                            mc.setNonZeros(moIn.getNnz());
                        }
                    }
                }
            }
        }
    } catch (Exception ex) {
        throw new HopsException("Failed to extract output statistics of function " + fkey + ".", ex);
    }
}

Example 28 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class InterProceduralAnalysis method populateLocalVariableMapForFunctionCall.

private static void populateLocalVariableMapForFunctionCall(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callvars, LocalVariableMap vars, FunctionCallSizeInfo fcallSizes) {
    ArrayList<DataIdentifier> inputVars = fstmt.getInputParams();
    ArrayList<Hop> inputOps = fop.getInput();
    String fkey = fop.getFunctionKey();
    for (int i = 0; i < inputVars.size(); i++) {
        // create mapping between input hops and vars
        DataIdentifier dat = inputVars.get(i);
        Hop input = inputOps.get(i);
        if (input.getDataType() == DataType.MATRIX) {
            // propagate matrix characteristics
            MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
            MatrixCharacteristics mc = new MatrixCharacteristics(input.getDim1(), input.getDim2(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), fcallSizes.isSafeNnz(fkey, i) ? input.getNnz() : -1);
            MetaDataFormat meta = new MetaDataFormat(mc, null, null);
            mo.setMetaData(meta);
            vars.put(dat.getName(), mo);
        } else if (input.getDataType() == DataType.SCALAR) {
            // (for multiple calls, literal equivalence already checked)
            if (input instanceof LiteralOp) {
                vars.put(dat.getName(), ScalarObjectFactory.createScalarObject(input.getValueType(), (LiteralOp) input));
            } else // and input scalar is existing variable in symbol table
            if (PROPAGATE_SCALAR_VARS_INTO_FUN && fcallSizes.getFunctionCallCount(fkey) == 1 && input instanceof DataOp) {
                Data scalar = callvars.get(input.getName());
                if (scalar != null && scalar instanceof ScalarObject) {
                    vars.put(dat.getName(), scalar);
                }
            }
        }
    }
}

Example 29 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class Recompiler method reconcileUpdatedCallVarsLoops.

public static boolean reconcileUpdatedCallVarsLoops(LocalVariableMap oldCallVars, LocalVariableMap callVars, StatementBlock sb) {
    boolean requiresRecompile = false;
    // handle matrices
    for (String varname : sb.variablesUpdated().getVariableNames()) {
        Data dat1 = oldCallVars.get(varname);
        Data dat2 = callVars.get(varname);
        if (dat1 != null && dat1 instanceof MatrixObject && dat2 != null && dat2 instanceof MatrixObject) {
            MatrixObject moOld = (MatrixObject) dat1;
            MatrixObject mo = (MatrixObject) dat2;
            MatrixCharacteristics mcOld = moOld.getMatrixCharacteristics();
            MatrixCharacteristics mc = mo.getMatrixCharacteristics();
            if (mcOld.getRows() != mc.getRows() || mcOld.getCols() != mc.getCols() || mcOld.getNonZeros() != mc.getNonZeros()) {
                long ldim1 = mc.getRows(), ldim2 = mc.getCols(), lnnz = mc.getNonZeros();
                // handle row dimension change in body
                if (mcOld.getRows() != mc.getRows()) {
                    // unknown
                    ldim1 = -1;
                    requiresRecompile = true;
                }
                // handle column dimension change in body
                if (mcOld.getCols() != mc.getCols()) {
                    // unknown
                    ldim2 = -1;
                    requiresRecompile = true;
                }
                // handle sparsity change
                if (mcOld.getNonZeros() != mc.getNonZeros()) {
                    // unknown
                    lnnz = -1;
                    requiresRecompile = true;
                }
                MatrixObject moNew = createOutputMatrix(ldim1, ldim2, lnnz);
                callVars.put(varname, moNew);
            }
        }
    }
    return requiresRecompile;
}

Example 30 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class Recompiler method extractDAGOutputStatistics.

public static void extractDAGOutputStatistics(Hop hop, LocalVariableMap vars, boolean overwrite) {
    if (// for all writes to symbol table
    hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTWRITE) {
        String varName = hop.getName();
        if (// not existing so far
        !vars.keySet().contains(varName) || overwrite) {
            // extract matrix sizes for size propagation
            if (hop.getDataType() == DataType.MATRIX) {
                MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
                MatrixCharacteristics mc = new MatrixCharacteristics(hop.getDim1(), hop.getDim2(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), hop.getNnz());
                MetaDataFormat meta = new MetaDataFormat(mc, null, null);
                mo.setMetaData(meta);
                vars.put(varName, mo);
            } else // extract scalar constants for second constant propagation
            if (hop.getDataType() == DataType.SCALAR) {
                // extract literal assignments
                if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof LiteralOp) {
                    ScalarObject constant = HopRewriteUtils.getScalarObject((LiteralOp) hop.getInput().get(0));
                    if (constant != null)
                        vars.put(varName, constant);
                } else // extract constant variable assignments
                if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof DataOp) {
                    DataOp dop = (DataOp) hop.getInput().get(0);
                    String dopvarname = dop.getName();
                    if (dop.isRead() && vars.keySet().contains(dopvarname)) {
                        ScalarObject constant = (ScalarObject) vars.get(dopvarname);
                        // no clone because constant
                        vars.put(varName, constant);
                    }
                } else // extract ncol/nrow variable assignments
                if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof UnaryOp && (((UnaryOp) hop.getInput().get(0)).getOp() == OpOp1.NROW || ((UnaryOp) hop.getInput().get(0)).getOp() == OpOp1.NCOL)) {
                    UnaryOp uop = (UnaryOp) hop.getInput().get(0);
                    if (uop.getOp() == OpOp1.NROW && uop.getInput().get(0).getDim1() > 0)
                        vars.put(varName, new IntObject(uop.getInput().get(0).getDim1()));
                    else if (uop.getOp() == OpOp1.NCOL && uop.getInput().get(0).getDim2() > 0)
                        vars.put(varName, new IntObject(uop.getInput().get(0).getDim2()));
                } else // remove other updated scalars
                {
                    // we need to remove other updated scalars in order to ensure result
                    // correctness of recompilation w/o being too conservative
                    vars.remove(varName);
                }
            }
        } else // already existing: take largest
        {
            Data dat = vars.get(varName);
            if (dat instanceof MatrixObject) {
                MatrixObject mo = (MatrixObject) dat;
                MatrixCharacteristics mc = mo.getMatrixCharacteristics();
                if (OptimizerUtils.estimateSizeExactSparsity(mc.getRows(), mc.getCols(), (mc.getNonZeros() >= 0) ? ((double) mc.getNonZeros()) / mc.getRows() / mc.getCols() : 1.0) < OptimizerUtils.estimateSize(hop.getDim1(), hop.getDim2())) {
                    // update statistics if necessary
                    mc.setDimension(hop.getDim1(), hop.getDim2());
                    mc.setNonZeros(hop.getNnz());
                }
            } else // scalar (just overwrite)
            {
                if (hop.getInput().size() == 1 && hop.getInput().get(0) instanceof LiteralOp) {
                    ScalarObject constant = HopRewriteUtils.getScalarObject((LiteralOp) hop.getInput().get(0));
                    if (constant != null)
                        vars.put(varName, constant);
                }
            }
        }
    }
}