Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 71 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project systemml by apache.

the class MRJobInstruction method extractOutputMatrices.

/**
 * Extracts MatrixObject references to output variables, all of which will be
 * of MATRIX data type, and stores them in <code>outputMatrices</code>. Also,
 * populates auxiliary data structures.
 *
 * @param ec execution context
 * @return array of matrix objects
 */
public MatrixObject[] extractOutputMatrices(ExecutionContext ec) {
    outputMatrices = new MatrixObject[getOutputVars().length];
    int ind = 0;
    for (String oo : getOutputVars()) {
        Data d = ec.getVariable(oo);
        if (d.getDataType() == DataType.MATRIX) {
            outputMatrices[ind++] = (MatrixObject) d;
        } else {
            throw new DMLRuntimeException(getJobType() + ": invalid datatype (" + d.getDataType() + ") for output variable " + oo);
        }
    }
    // populate auxiliary data structures
    populateOutputs();
    return outputMatrices;
}

Example 72 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project systemml by apache.

the class MRJobInstruction method extractInputMatrices.

/**
 * Extracts input variables with MATRIX data type, and stores references to
 * corresponding matrix objects in <code>inputMatrices</code>. Also, stores
 * the data types in <code>inputDataTypes</code>.
 *
 * @param ec execution context
 * @return array of matrix objects
 */
public MatrixObject[] extractInputMatrices(ExecutionContext ec) {
    ArrayList<MatrixObject> inputmat = new ArrayList<>();
    inputDataTypes = new DataType[inputVars.length];
    for (int i = 0; i < inputVars.length; i++) {
        Data d = ec.getVariable(inputVars[i]);
        inputDataTypes[i] = d.getDataType();
        if (d.getDataType() == DataType.MATRIX) {
            inputmat.add((MatrixObject) d);
        } else if (d.getDataType() == DataType.FRAME) {
            // FIXME conversion from frame to matrix object (meta data only) to adhere to
            // the given matrix-based mr job submission framework
            FrameObject fo = (FrameObject) d;
            MatrixObject mo = new MatrixObject(fo.getValueType(), fo.getFileName(), fo.getMetaData());
            mo.setFileFormatProperties(fo.getFileFormatProperties());
            inputmat.add(mo);
        }
    }
    inputMatrices = inputmat.toArray(new MatrixObject[inputmat.size()]);
    // populate auxiliary data structures
    populateInputs();
    return inputMatrices;
}

Example 73 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project systemml by apache.

the class ProgramConverter method createDeepCopyExecutionContext.

// //////////////////////////////
// CREATION of DEEP COPIES
// //////////////////////////////
/**
 * Creates a deep copy of the given execution context.
 * For rt_platform=Hadoop, execution context has a symbol table.
 *
 * @param ec execution context
 * @return execution context
 * @throws CloneNotSupportedException if CloneNotSupportedException occurs
 */
public static ExecutionContext createDeepCopyExecutionContext(ExecutionContext ec) throws CloneNotSupportedException {
    ExecutionContext cpec = ExecutionContextFactory.createContext(false, ec.getProgram());
    cpec.setVariables((LocalVariableMap) ec.getVariables().clone());
    // (each worker requires its own copy of the empty matrix object)
    for (String var : cpec.getVariables().keySet()) {
        Data dat = cpec.getVariables().get(var);
        if (dat instanceof MatrixObject && ((MatrixObject) dat).getUpdateType().isInPlace()) {
            MatrixObject mo = (MatrixObject) dat;
            MatrixObject moNew = new MatrixObject(mo);
            if (mo.getNnz() != 0) {
                // If output matrix is not empty (NNZ != 0), then local copy is created so that
                // update in place operation can be applied.
                MatrixBlock mbVar = mo.acquireRead();
                moNew.acquireModify(new MatrixBlock(mbVar));
                mo.release();
            } else {
                // create empty matrix block w/ dense representation (preferred for update in-place)
                // Creating a dense matrix block is valid because empty block not allocated and transfer
                // to sparse representation happens in left indexing in place operation.
                moNew.acquireModify(new MatrixBlock((int) mo.getNumRows(), (int) mo.getNumColumns(), false));
            }
            moNew.release();
            cpec.setVariable(var, moNew);
        }
    }
    return cpec;
}

Example 74 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project systemml by apache.

the class ExternalFunctionProgramBlock method execute.

/**
 * Method to be invoked to execute instructions for the external function
 * invocation
 */
@Override
public void execute(ExecutionContext ec) {
    _runID = _idSeq.getNextID();
    changeTmpInput(_runID, ec);
    changeTmpOutput(_runID);
    // export input variables to HDFS (see RunMRJobs)
    ArrayList<DataIdentifier> inputParams = null;
    try {
        inputParams = getInputParams();
        for (DataIdentifier di : inputParams) {
            Data d = ec.getVariable(di.getName());
            if (d.getDataType().isMatrix())
                ((MatrixObject) d).exportData();
        }
    } catch (Exception e) {
        throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error exporting input variables to HDFS", e);
    }
    // convert block to cell
    if (block2CellInst != null) {
        ArrayList<Instruction> tempInst = new ArrayList<>();
        tempInst.addAll(block2CellInst);
        try {
            this.executeInstructions(tempInst, ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error executing " + tempInst.toString(), e);
        }
    }
    // now execute package function
    for (int i = 0; i < _inst.size(); i++) {
        try {
            if (_inst.get(i) instanceof ExternalFunctionInvocationInstruction)
                ((ExternalFunctionInvocationInstruction) _inst.get(i)).processInstruction(ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction " + _inst.get(i).toString(), e);
        }
    }
    // convert cell to block
    if (cell2BlockInst != null) {
        ArrayList<Instruction> tempInst = new ArrayList<>();
        try {
            tempInst.clear();
            tempInst.addAll(cell2BlockInst);
            this.executeInstructions(tempInst, ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction " + cell2BlockInst.toString(), e);
        }
    }
    // check return values
    checkOutputParameters(ec.getVariables());
}

Example 75 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project systemml by apache.

the class ParForProgramBlock method exportMatricesToHDFS.

private void exportMatricesToHDFS(ExecutionContext ec, String... blacklistNames) {
    ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
    Set<String> blacklist = UtilFunctions.asSet(blacklistNames);
    if (LIVEVAR_AWARE_EXPORT && sb != null) {
        // optimization to prevent unnecessary export of matrices
        // export only variables that are read in the body
        VariableSet varsRead = sb.variablesRead();
        for (String key : ec.getVariables().keySet()) {
            if (varsRead.containsVariable(key) && !blacklist.contains(key)) {
                Data d = ec.getVariable(key);
                if (d.getDataType() == DataType.MATRIX)
                    ((MatrixObject) d).exportData(_replicationExport);
            }
        }
    } else {
        // export all matrices in symbol table
        for (String key : ec.getVariables().keySet()) {
            if (!blacklist.contains(key)) {
                Data d = ec.getVariable(key);
                if (d.getDataType() == DataType.MATRIX)
                    ((MatrixObject) d).exportData(_replicationExport);
            }
        }
    }
}