Examples with CacheableData org.apache.sysml.runtime.controlprogram.caching.CacheableData used on opensource projects

Example 6 with CacheableData

use of org.apache.sysml.runtime.controlprogram.caching.CacheableData in project incubator-systemml by apache.

the class FunctionCallCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    if (LOG.isTraceEnabled()) {
        LOG.trace("Executing instruction : " + this.toString());
    }
    // get the function program block (stored in the Program object)
    FunctionProgramBlock fpb = ec.getProgram().getFunctionProgramBlock(_namespace, _functionName);
    // sanity check number of function parameters
    if (_boundInputs.length < fpb.getInputParams().size()) {
        throw new DMLRuntimeException("Number of bound input parameters does not match the function signature " + "(" + _boundInputs.length + ", but " + fpb.getInputParams().size() + " expected)");
    }
    // create bindings to formal parameters for given function call
    // These are the bindings passed to the FunctionProgramBlock for function execution
    LocalVariableMap functionVariables = new LocalVariableMap();
    for (int i = 0; i < fpb.getInputParams().size(); i++) {
        // error handling non-existing variables
        CPOperand input = _boundInputs[i];
        if (!input.isLiteral() && !ec.containsVariable(input.getName())) {
            throw new DMLRuntimeException("Input variable '" + input.getName() + "' not existing on call of " + DMLProgram.constructFunctionKey(_namespace, _functionName) + " (line " + getLineNum() + ").");
        }
        // get input matrix/frame/scalar
        DataIdentifier currFormalParam = fpb.getInputParams().get(i);
        Data value = ec.getVariable(input);
        // graceful value type conversion for scalar inputs with wrong type
        if (value.getDataType() == DataType.SCALAR && value.getValueType() != currFormalParam.getValueType()) {
            value = ScalarObjectFactory.createScalarObject(currFormalParam.getValueType(), (ScalarObject) value);
        }
        // set input parameter
        functionVariables.put(currFormalParam.getName(), value);
    }
    // Pin the input variables so that they do not get deleted
    // from pb's symbol table at the end of execution of function
    boolean[] pinStatus = ec.pinVariables(_boundInputNames);
    // Create a symbol table under a new execution context for the function invocation,
    // and copy the function arguments into the created table.
    ExecutionContext fn_ec = ExecutionContextFactory.createContext(false, ec.getProgram());
    if (DMLScript.USE_ACCELERATOR) {
        fn_ec.setGPUContexts(ec.getGPUContexts());
        fn_ec.getGPUContext(0).initializeThread();
    }
    fn_ec.setVariables(functionVariables);
    // execute the function block
    try {
        fpb._functionName = this._functionName;
        fpb._namespace = this._namespace;
        fpb.execute(fn_ec);
    } catch (DMLScriptException e) {
        throw e;
    } catch (Exception e) {
        String fname = DMLProgram.constructFunctionKey(_namespace, _functionName);
        throw new DMLRuntimeException("error executing function " + fname, e);
    }
    // cleanup all returned variables w/o binding
    HashSet<String> expectRetVars = new HashSet<>();
    for (DataIdentifier di : fpb.getOutputParams()) expectRetVars.add(di.getName());
    LocalVariableMap retVars = fn_ec.getVariables();
    for (Entry<String, Data> var : retVars.entrySet()) {
        if (expectRetVars.contains(var.getKey()))
            continue;
        // cleanup unexpected return values to avoid leaks
        if (var.getValue() instanceof CacheableData)
            fn_ec.cleanupCacheableData((CacheableData<?>) var.getValue());
    }
    // Unpin the pinned variables
    ec.unpinVariables(_boundInputNames, pinStatus);
    // add the updated binding for each return variable to the variables in original symbol table
    for (int i = 0; i < fpb.getOutputParams().size(); i++) {
        String boundVarName = _boundOutputNames.get(i);
        Data boundValue = retVars.get(fpb.getOutputParams().get(i).getName());
        if (boundValue == null)
            throw new DMLRuntimeException(boundVarName + " was not assigned a return value");
        // cleanup existing data bound to output variable name
        Data exdata = ec.removeVariable(boundVarName);
        if (exdata != null && exdata instanceof CacheableData && exdata != boundValue) {
            ec.cleanupCacheableData((CacheableData<?>) exdata);
        }
        // add/replace data in symbol table
        ec.setVariable(boundVarName, boundValue);
    }
}

Example 7 with CacheableData

use of org.apache.sysml.runtime.controlprogram.caching.CacheableData in project systemml by apache.

the class FunctionCallCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    if (LOG.isTraceEnabled()) {
        LOG.trace("Executing instruction : " + this.toString());
    }
    // get the function program block (stored in the Program object)
    FunctionProgramBlock fpb = ec.getProgram().getFunctionProgramBlock(_namespace, _functionName);
    // sanity check number of function parameters
    if (_boundInputs.length < fpb.getInputParams().size()) {
        throw new DMLRuntimeException("Number of bound input parameters does not match the function signature " + "(" + _boundInputs.length + ", but " + fpb.getInputParams().size() + " expected)");
    }
    // create bindings to formal parameters for given function call
    // These are the bindings passed to the FunctionProgramBlock for function execution
    LocalVariableMap functionVariables = new LocalVariableMap();
    for (int i = 0; i < fpb.getInputParams().size(); i++) {
        // error handling non-existing variables
        CPOperand input = _boundInputs[i];
        if (!input.isLiteral() && !ec.containsVariable(input.getName())) {
            throw new DMLRuntimeException("Input variable '" + input.getName() + "' not existing on call of " + DMLProgram.constructFunctionKey(_namespace, _functionName) + " (line " + getLineNum() + ").");
        }
        // get input matrix/frame/scalar
        DataIdentifier currFormalParam = fpb.getInputParams().get(i);
        Data value = ec.getVariable(input);
        // graceful value type conversion for scalar inputs with wrong type
        if (value.getDataType() == DataType.SCALAR && value.getValueType() != currFormalParam.getValueType()) {
            value = ScalarObjectFactory.createScalarObject(currFormalParam.getValueType(), (ScalarObject) value);
        }
        // set input parameter
        functionVariables.put(currFormalParam.getName(), value);
    }
    // Pin the input variables so that they do not get deleted
    // from pb's symbol table at the end of execution of function
    boolean[] pinStatus = ec.pinVariables(_boundInputNames);
    // Create a symbol table under a new execution context for the function invocation,
    // and copy the function arguments into the created table.
    ExecutionContext fn_ec = ExecutionContextFactory.createContext(false, ec.getProgram());
    if (DMLScript.USE_ACCELERATOR) {
        fn_ec.setGPUContexts(ec.getGPUContexts());
        fn_ec.getGPUContext(0).initializeThread();
    }
    fn_ec.setVariables(functionVariables);
    // execute the function block
    try {
        fpb._functionName = this._functionName;
        fpb._namespace = this._namespace;
        fpb.execute(fn_ec);
    } catch (DMLScriptException e) {
        throw e;
    } catch (Exception e) {
        String fname = DMLProgram.constructFunctionKey(_namespace, _functionName);
        throw new DMLRuntimeException("error executing function " + fname, e);
    }
    // cleanup all returned variables w/o binding
    HashSet<String> expectRetVars = new HashSet<>();
    for (DataIdentifier di : fpb.getOutputParams()) expectRetVars.add(di.getName());
    LocalVariableMap retVars = fn_ec.getVariables();
    for (Entry<String, Data> var : retVars.entrySet()) {
        if (expectRetVars.contains(var.getKey()))
            continue;
        // cleanup unexpected return values to avoid leaks
        if (var.getValue() instanceof CacheableData)
            fn_ec.cleanupCacheableData((CacheableData<?>) var.getValue());
    }
    // Unpin the pinned variables
    ec.unpinVariables(_boundInputNames, pinStatus);
    // add the updated binding for each return variable to the variables in original symbol table
    for (int i = 0; i < fpb.getOutputParams().size(); i++) {
        String boundVarName = _boundOutputNames.get(i);
        Data boundValue = retVars.get(fpb.getOutputParams().get(i).getName());
        if (boundValue == null)
            throw new DMLRuntimeException(boundVarName + " was not assigned a return value");
        // cleanup existing data bound to output variable name
        Data exdata = ec.removeVariable(boundVarName);
        if (exdata != null && exdata instanceof CacheableData && exdata != boundValue) {
            ec.cleanupCacheableData((CacheableData<?>) exdata);
        }
        // add/replace data in symbol table
        ec.setVariable(boundVarName, boundValue);
    }
}

Example 8 with CacheableData

use of org.apache.sysml.runtime.controlprogram.caching.CacheableData in project systemml by apache.

the class VariableCPInstruction method processMoveInstruction.

/**
 * Handler for mvvar instructions.
 * Example: mvvar <srcvar> <destFile> <format>
 * Move the file pointed by srcvar to destFile.
 * Currently, applicable only when format=binaryblock.
 *
 * @param ec execution context
 */
@SuppressWarnings("rawtypes")
private void processMoveInstruction(ExecutionContext ec) {
    if (getInput3() == null) {
        // example: mvvar tempA A
        // get source variable
        Data srcData = ec.getVariable(getInput1().getName());
        if (srcData == null) {
            throw new DMLRuntimeException("Unexpected error: could not find a data object " + "for variable name:" + getInput1().getName() + ", while processing instruction ");
        }
        if (getInput2().getDataType().isMatrix() || getInput2().getDataType().isFrame()) {
            // remove existing variable bound to target name
            Data tgt = ec.removeVariable(getInput2().getName());
            // cleanup matrix data on fs/hdfs (if necessary)
            if (tgt != null && tgt instanceof CacheableData) {
                ec.cleanupCacheableData((CacheableData<?>) tgt);
            }
        }
        // do the actual move
        ec.setVariable(getInput2().getName(), srcData);
        ec.removeVariable(getInput1().getName());
    } else {
        // example instruction: mvvar <srcVar> <destFile> <format>
        if (ec.getVariable(getInput1().getName()) == null)
            throw new DMLRuntimeException("Unexpected error: could not find a data object for variable name:" + getInput1().getName() + ", while processing instruction " + this.toString());
        Object object = ec.getVariable(getInput1().getName());
        if (getInput3().getName().equalsIgnoreCase("binaryblock")) {
            boolean success = false;
            success = ((CacheableData) object).moveData(getInput2().getName(), getInput3().getName());
            if (!success) {
                throw new DMLRuntimeException("Failed to move var " + getInput1().getName() + " to file " + getInput2().getName() + ".");
            }
        } else if (object instanceof MatrixObject)
            throw new DMLRuntimeException("Unexpected formats while copying: from matrix blocks [" + ((MatrixObject) object).getNumRowsPerBlock() + "," + ((MatrixObject) object).getNumColumnsPerBlock() + "] to " + getInput3().getName());
        else if (object instanceof FrameObject)
            throw new DMLRuntimeException("Unexpected formats while copying: from fram object [" + ((FrameObject) object).getNumColumns() + "," + ((FrameObject) object).getNumColumns() + "] to " + getInput3().getName());
    }
}

Example 9 with CacheableData

use of org.apache.sysml.runtime.controlprogram.caching.CacheableData in project systemml by apache.

the class Script method in.

/**
 * Register an input (parameter ($) or variable) with optional matrix
 * metadata.
 *
 * @param name
 *            name of the input
 * @param value
 *            value of the input
 * @param metadata
 *            optional matrix/frame metadata
 * @return {@code this} Script object to allow chaining of methods
 */
public Script in(String name, Object value, Metadata metadata) {
    if ((value != null) && (value instanceof Long)) {
        // convert Long to Integer since Long not a supported value type
        Long lng = (Long) value;
        value = lng.intValue();
    } else if ((value != null) && (value instanceof Float)) {
        // convert Float to Double since Float not a supported value type
        Float flt = (Float) value;
        value = flt.doubleValue();
    }
    MLContextUtil.checkInputValueType(name, value);
    if (inputs == null) {
        inputs = new LinkedHashMap<>();
    }
    inputs.put(name, value);
    if (name.startsWith("$")) {
        MLContextUtil.checkInputParameterType(name, value);
        if (inputParameters == null) {
            inputParameters = new LinkedHashMap<>();
        }
        inputParameters.put(name, value);
    } else {
        Data data = MLContextUtil.convertInputType(name, value, metadata);
        if (data != null) {
            // store input variable name and data
            symbolTable.put(name, data);
            inputVariables.add(name);
            // store matrix/frame meta data and disable variable cleanup
            if (data instanceof CacheableData) {
                if (metadata != null)
                    inputMetadata.put(name, metadata);
                ((CacheableData<?>) data).enableCleanup(false);
            }
        }
    }
    return this;
}