Examples with CacheableData org.apache.sysml.runtime.controlprogram.caching.CacheableData used on opensource projects

Example 1 with CacheableData

use of org.apache.sysml.runtime.controlprogram.caching.CacheableData in project incubator-systemml by apache.

the class VariableCPInstruction method processMoveInstruction.

/**
 * Handler for mvvar instructions.
 * Example: mvvar <srcvar> <destFile> <format>
 * Move the file pointed by srcvar to destFile.
 * Currently, applicable only when format=binaryblock.
 *
 * @param ec execution context
 */
@SuppressWarnings("rawtypes")
private void processMoveInstruction(ExecutionContext ec) {
    if (getInput3() == null) {
        // example: mvvar tempA A
        // get source variable
        Data srcData = ec.getVariable(getInput1().getName());
        if (srcData == null) {
            throw new DMLRuntimeException("Unexpected error: could not find a data object " + "for variable name:" + getInput1().getName() + ", while processing instruction ");
        }
        if (getInput2().getDataType().isMatrix() || getInput2().getDataType().isFrame()) {
            // remove existing variable bound to target name
            Data tgt = ec.removeVariable(getInput2().getName());
            // cleanup matrix data on fs/hdfs (if necessary)
            if (tgt != null && tgt instanceof CacheableData) {
                ec.cleanupCacheableData((CacheableData<?>) tgt);
            }
        }
        // do the actual move
        ec.setVariable(getInput2().getName(), srcData);
        ec.removeVariable(getInput1().getName());
    } else {
        // example instruction: mvvar <srcVar> <destFile> <format>
        if (ec.getVariable(getInput1().getName()) == null)
            throw new DMLRuntimeException("Unexpected error: could not find a data object for variable name:" + getInput1().getName() + ", while processing instruction " + this.toString());
        Object object = ec.getVariable(getInput1().getName());
        if (getInput3().getName().equalsIgnoreCase("binaryblock")) {
            boolean success = false;
            success = ((CacheableData) object).moveData(getInput2().getName(), getInput3().getName());
            if (!success) {
                throw new DMLRuntimeException("Failed to move var " + getInput1().getName() + " to file " + getInput2().getName() + ".");
            }
        } else if (object instanceof MatrixObject)
            throw new DMLRuntimeException("Unexpected formats while copying: from matrix blocks [" + ((MatrixObject) object).getNumRowsPerBlock() + "," + ((MatrixObject) object).getNumColumnsPerBlock() + "] to " + getInput3().getName());
        else if (object instanceof FrameObject)
            throw new DMLRuntimeException("Unexpected formats while copying: from fram object [" + ((FrameObject) object).getNumColumns() + "," + ((FrameObject) object).getNumColumns() + "] to " + getInput3().getName());
    }
}

Example 2 with CacheableData

use of org.apache.sysml.runtime.controlprogram.caching.CacheableData in project incubator-systemml by apache.

the class VariableCPInstruction method processCopyInstruction.

/**
 * Handler for cpvar instructions.
 * Example: cpvar <srcvar> <destvar>
 *
 * @param ec execution context
 */
private void processCopyInstruction(ExecutionContext ec) {
    // get source variable
    Data dd = ec.getVariable(getInput1().getName());
    if (dd == null)
        throw new DMLRuntimeException("Unexpected error: could not find a data object for variable name:" + getInput1().getName() + ", while processing instruction " + this.toString());
    // remove existing variable bound to target name
    Data input2_data = ec.removeVariable(getInput2().getName());
    // cleanup matrix data on fs/hdfs (if necessary)
    if (input2_data != null && input2_data instanceof CacheableData) {
        ec.cleanupCacheableData((CacheableData<?>) input2_data);
    }
    // do the actual copy!
    ec.setVariable(getInput2().getName(), dd);
}

Example 3 with CacheableData

use of org.apache.sysml.runtime.controlprogram.caching.CacheableData in project systemml by apache.

the class VariableCPInstruction method processCopyInstruction.

/**
 * Handler for cpvar instructions.
 * Example: cpvar <srcvar> <destvar>
 *
 * @param ec execution context
 */
private void processCopyInstruction(ExecutionContext ec) {
    // get source variable
    Data dd = ec.getVariable(getInput1().getName());
    if (dd == null)
        throw new DMLRuntimeException("Unexpected error: could not find a data object for variable name:" + getInput1().getName() + ", while processing instruction " + this.toString());
    // remove existing variable bound to target name
    Data input2_data = ec.removeVariable(getInput2().getName());
    // cleanup matrix data on fs/hdfs (if necessary)
    if (input2_data != null && input2_data instanceof CacheableData) {
        ec.cleanupCacheableData((CacheableData<?>) input2_data);
    }
    // do the actual copy!
    ec.setVariable(getInput2().getName(), dd);
}

Example 4 with CacheableData

use of org.apache.sysml.runtime.controlprogram.caching.CacheableData in project systemml by apache.

the class ParameterizedBuiltinCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    String opcode = getOpcode();
    ScalarObject sores = null;
    if (opcode.equalsIgnoreCase("cdf")) {
        SimpleOperator op = (SimpleOperator) _optr;
        double result = op.fn.execute(params);
        sores = new DoubleObject(result);
        ec.setScalarOutput(output.getName(), sores);
    } else if (opcode.equalsIgnoreCase("invcdf")) {
        SimpleOperator op = (SimpleOperator) _optr;
        double result = op.fn.execute(params);
        sores = new DoubleObject(result);
        ec.setScalarOutput(output.getName(), sores);
    } else if (opcode.equalsIgnoreCase("groupedagg")) {
        // acquire locks
        MatrixBlock target = ec.getMatrixInput(params.get(Statement.GAGG_TARGET), getExtendedOpcode());
        MatrixBlock groups = ec.getMatrixInput(params.get(Statement.GAGG_GROUPS), getExtendedOpcode());
        MatrixBlock weights = null;
        if (params.get(Statement.GAGG_WEIGHTS) != null)
            weights = ec.getMatrixInput(params.get(Statement.GAGG_WEIGHTS), getExtendedOpcode());
        int ngroups = -1;
        if (params.get(Statement.GAGG_NUM_GROUPS) != null) {
            ngroups = (int) Double.parseDouble(params.get(Statement.GAGG_NUM_GROUPS));
        }
        // compute the result
        // num threads
        int k = Integer.parseInt(params.get("k"));
        MatrixBlock soresBlock = groups.groupedAggOperations(target, weights, new MatrixBlock(), ngroups, _optr, k);
        ec.setMatrixOutput(output.getName(), soresBlock, getExtendedOpcode());
        // release locks
        target = groups = weights = null;
        ec.releaseMatrixInput(params.get(Statement.GAGG_TARGET), getExtendedOpcode());
        ec.releaseMatrixInput(params.get(Statement.GAGG_GROUPS), getExtendedOpcode());
        if (params.get(Statement.GAGG_WEIGHTS) != null)
            ec.releaseMatrixInput(params.get(Statement.GAGG_WEIGHTS), getExtendedOpcode());
    } else if (opcode.equalsIgnoreCase("rmempty")) {
        String margin = params.get("margin");
        if (!(margin.equals("rows") || margin.equals("cols")))
            throw new DMLRuntimeException("Unspupported margin identifier '" + margin + "'.");
        // acquire locks
        MatrixBlock target = ec.getMatrixInput(params.get("target"), getExtendedOpcode());
        MatrixBlock select = params.containsKey("select") ? ec.getMatrixInput(params.get("select"), getExtendedOpcode()) : null;
        // compute the result
        boolean emptyReturn = Boolean.parseBoolean(params.get("empty.return").toLowerCase());
        MatrixBlock soresBlock = target.removeEmptyOperations(new MatrixBlock(), margin.equals("rows"), emptyReturn, select);
        // release locks
        ec.setMatrixOutput(output.getName(), soresBlock, getExtendedOpcode());
        ec.releaseMatrixInput(params.get("target"), getExtendedOpcode());
        if (params.containsKey("select"))
            ec.releaseMatrixInput(params.get("select"), getExtendedOpcode());
    } else if (opcode.equalsIgnoreCase("replace")) {
        MatrixBlock target = ec.getMatrixInput(params.get("target"), getExtendedOpcode());
        double pattern = Double.parseDouble(params.get("pattern"));
        double replacement = Double.parseDouble(params.get("replacement"));
        MatrixBlock ret = (MatrixBlock) target.replaceOperations(new MatrixBlock(), pattern, replacement);
        ec.setMatrixOutput(output.getName(), ret, getExtendedOpcode());
        ec.releaseMatrixInput(params.get("target"), getExtendedOpcode());
    } else if (opcode.equals("lowertri") || opcode.equals("uppertri")) {
        MatrixBlock target = ec.getMatrixInput(params.get("target"), getExtendedOpcode());
        boolean lower = opcode.equals("lowertri");
        boolean diag = Boolean.parseBoolean(params.get("diag"));
        boolean values = Boolean.parseBoolean(params.get("values"));
        MatrixBlock ret = (MatrixBlock) target.extractTriangular(new MatrixBlock(), lower, diag, values);
        ec.setMatrixOutput(output.getName(), ret, getExtendedOpcode());
        ec.releaseMatrixInput(params.get("target"), getExtendedOpcode());
    } else if (opcode.equalsIgnoreCase("rexpand")) {
        // acquire locks
        MatrixBlock target = ec.getMatrixInput(params.get("target"), getExtendedOpcode());
        // compute the result
        double maxVal = Double.parseDouble(params.get("max"));
        boolean dirVal = params.get("dir").equals("rows");
        boolean cast = Boolean.parseBoolean(params.get("cast"));
        boolean ignore = Boolean.parseBoolean(params.get("ignore"));
        int numThreads = Integer.parseInt(params.get("k"));
        MatrixBlock ret = (MatrixBlock) target.rexpandOperations(new MatrixBlock(), maxVal, dirVal, cast, ignore, numThreads);
        // release locks
        ec.setMatrixOutput(output.getName(), ret, getExtendedOpcode());
        ec.releaseMatrixInput(params.get("target"), getExtendedOpcode());
    } else if (opcode.equalsIgnoreCase("transformapply")) {
        // acquire locks
        FrameBlock data = ec.getFrameInput(params.get("target"));
        FrameBlock meta = ec.getFrameInput(params.get("meta"));
        String[] colNames = data.getColumnNames();
        // compute transformapply
        Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), colNames, data.getNumColumns(), meta);
        MatrixBlock mbout = encoder.apply(data, new MatrixBlock(data.getNumRows(), data.getNumColumns(), false));
        // release locks
        ec.setMatrixOutput(output.getName(), mbout, getExtendedOpcode());
        ec.releaseFrameInput(params.get("target"));
        ec.releaseFrameInput(params.get("meta"));
    } else if (opcode.equalsIgnoreCase("transformdecode")) {
        // acquire locks
        MatrixBlock data = ec.getMatrixInput(params.get("target"), getExtendedOpcode());
        FrameBlock meta = ec.getFrameInput(params.get("meta"));
        String[] colnames = meta.getColumnNames();
        // compute transformdecode
        Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta);
        FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()));
        fbout.setColumnNames(Arrays.copyOfRange(colnames, 0, fbout.getNumColumns()));
        // release locks
        ec.setFrameOutput(output.getName(), fbout);
        ec.releaseMatrixInput(params.get("target"), getExtendedOpcode());
        ec.releaseFrameInput(params.get("meta"));
    } else if (opcode.equalsIgnoreCase("transformcolmap")) {
        // acquire locks
        FrameBlock meta = ec.getFrameInput(params.get("target"));
        String[] colNames = meta.getColumnNames();
        // compute transformapply
        Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), colNames, meta.getNumColumns(), null);
        MatrixBlock mbout = encoder.getColMapping(meta, new MatrixBlock(meta.getNumColumns(), 3, false));
        // release locks
        ec.setMatrixOutput(output.getName(), mbout, getExtendedOpcode());
        ec.releaseFrameInput(params.get("target"));
    } else if (opcode.equalsIgnoreCase("transformmeta")) {
        // get input spec and path
        String spec = getParameterMap().get("spec");
        String path = getParameterMap().get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_MTD);
        String delim = getParameterMap().containsKey("sep") ? getParameterMap().get("sep") : TfUtils.TXMTD_SEP;
        // execute transform meta data read
        FrameBlock meta = null;
        try {
            meta = TfMetaUtils.readTransformMetaDataFromFile(spec, path, delim);
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
        // release locks
        ec.setFrameOutput(output.getName(), meta);
    } else if (opcode.equalsIgnoreCase("toString")) {
        // handle input parameters
        int rows = (getParam("rows") != null) ? Integer.parseInt(getParam("rows")) : TOSTRING_MAXROWS;
        int cols = (getParam("cols") != null) ? Integer.parseInt(getParam("cols")) : TOSTRING_MAXCOLS;
        int decimal = (getParam("decimal") != null) ? Integer.parseInt(getParam("decimal")) : TOSTRING_DECIMAL;
        boolean sparse = (getParam("sparse") != null) ? Boolean.parseBoolean(getParam("sparse")) : TOSTRING_SPARSE;
        String separator = (getParam("sep") != null) ? getParam("sep") : TOSTRING_SEPARATOR;
        String lineseparator = (getParam("linesep") != null) ? getParam("linesep") : TOSTRING_LINESEPARATOR;
        // get input matrix/frame and convert to string
        CacheableData<?> data = ec.getCacheableData(getParam("target"));
        String out = null;
        if (data instanceof MatrixObject) {
            MatrixBlock matrix = (MatrixBlock) data.acquireRead();
            warnOnTrunction(matrix, rows, cols);
            out = DataConverter.toString(matrix, sparse, separator, lineseparator, rows, cols, decimal);
        } else if (data instanceof FrameObject) {
            FrameBlock frame = (FrameBlock) data.acquireRead();
            warnOnTrunction(frame, rows, cols);
            out = DataConverter.toString(frame, sparse, separator, lineseparator, rows, cols, decimal);
        } else {
            throw new DMLRuntimeException("toString only converts matrix or frames to string");
        }
        ec.releaseCacheableData(getParam("target"));
        ec.setScalarOutput(output.getName(), new StringObject(out));
    } else {
        throw new DMLRuntimeException("Unknown opcode : " + opcode);
    }
}

Example 5 with CacheableData

use of org.apache.sysml.runtime.controlprogram.caching.CacheableData in project incubator-systemml by apache.

the class Script method in.

/**
 * Register an input (parameter ($) or variable) with optional matrix
 * metadata.
 *
 * @param name
 *            name of the input
 * @param value
 *            value of the input
 * @param metadata
 *            optional matrix/frame metadata
 * @return {@code this} Script object to allow chaining of methods
 */
public Script in(String name, Object value, Metadata metadata) {
    if ((value != null) && (value instanceof Long)) {
        // convert Long to Integer since Long not a supported value type
        Long lng = (Long) value;
        value = lng.intValue();
    } else if ((value != null) && (value instanceof Float)) {
        // convert Float to Double since Float not a supported value type
        Float flt = (Float) value;
        value = flt.doubleValue();
    }
    MLContextUtil.checkInputValueType(name, value);
    if (inputs == null) {
        inputs = new LinkedHashMap<>();
    }
    inputs.put(name, value);
    if (name.startsWith("$")) {
        MLContextUtil.checkInputParameterType(name, value);
        if (inputParameters == null) {
            inputParameters = new LinkedHashMap<>();
        }
        inputParameters.put(name, value);
    } else {
        Data data = MLContextUtil.convertInputType(name, value, metadata);
        if (data != null) {
            // store input variable name and data
            symbolTable.put(name, data);
            inputVariables.add(name);
            // store matrix/frame meta data and disable variable cleanup
            if (data instanceof CacheableData) {
                if (metadata != null)
                    inputMetadata.put(name, metadata);
                ((CacheableData<?>) data).enableCleanup(false);
            }
        }
    }
    return this;
}