Examples with MetaDataFormat org.apache.sysml.runtime.matrix.MetaDataFormat used on opensource projects

Example 51 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class MRJobInstruction method populateInputs.

/**
 * Auxiliary data structures that store information required to spawn MR jobs.
 * These data structures are populated by pulling out information from symbol
 * table. More specifically, from information stored in <code>inputMatrices</code>
 * and <code>outputMatrices</code>.
 */
private void populateInputs() {
    // Since inputVars can potentially contain scalar variables,
    // auxiliary data structures of size <code>inputMatrices.length</code>
    // are allocated instead of size <code>inputVars.length</code>
    // Allocate space
    inputs = new String[inputMatrices.length];
    inputInfos = new InputInfo[inputMatrices.length];
    rlens = new long[inputMatrices.length];
    clens = new long[inputMatrices.length];
    brlens = new int[inputMatrices.length];
    bclens = new int[inputMatrices.length];
    partitioned = new boolean[inputMatrices.length];
    pformats = new PDataPartitionFormat[inputMatrices.length];
    psizes = new int[inputMatrices.length];
    // populate information
    for (int i = 0; i < inputMatrices.length; i++) {
        inputs[i] = inputMatrices[i].getFileName();
        MatrixCharacteristics mc = inputMatrices[i].getMatrixCharacteristics();
        rlens[i] = mc.getRows();
        clens[i] = mc.getCols();
        brlens[i] = mc.getRowsPerBlock();
        bclens[i] = mc.getColsPerBlock();
        if (inputMatrices[i].getMetaData() instanceof MetaDataFormat) {
            inputInfos[i] = ((MetaDataFormat) inputMatrices[i].getMetaData()).getInputInfo();
        } else if (inputMatrices[i].getMetaData() instanceof MetaDataNumItemsByEachReducer) {
            inputInfos[i] = InputInfo.InputInfoForSortOutput;
            inputInfos[i].metadata = inputMatrices[i].getMetaData();
        }
        partitioned[i] = inputMatrices[i].isPartitioned();
        pformats[i] = inputMatrices[i].getPartitionFormat();
        psizes[i] = inputMatrices[i].getPartitionSize();
    }
}

Example 52 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class MRJobInstruction method populateOutputs.

/**
 * Pulls out information from symbol table for output variables (i.e., outputMatrices)
 * and populates auxiliary data structutes that are used in setting up MR jobs.
 */
private void populateOutputs() {
    // Note: (outputVars.length == outputMatrices.length) -> true
    // Allocate space
    outputs = new String[outputVars.length];
    outputInfos = new OutputInfo[outputVars.length];
    // Populate information
    for (int i = 0; i < outputVars.length; i++) {
        outputs[i] = outputMatrices[i].getFileName();
        MetaDataFormat md = (MetaDataFormat) outputMatrices[i].getMetaData();
        outputInfos[i] = md.getOutputInfo();
    }
}

Example 53 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class DataPartitionCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    // get input
    MatrixObject moIn = ec.getMatrixObject(input1.getName());
    MatrixBlock mb = moIn.acquireRead();
    // execute operations
    MatrixObject moOut = (MatrixObject) ec.getVariable(output.getName());
    String fname = moOut.getFileName();
    // modify meta data output
    moOut.setPartitioned(_pformat, -1);
    try {
        // write matrix partitions to hdfs
        WriterBinaryBlock.writePartitionedBinaryBlockMatrixToHDFS(new Path(fname), new JobConf(ConfigurationManager.getCachedJobConf()), mb, moIn.getNumRows(), moIn.getNumColumns(), (int) moIn.getNumRowsPerBlock(), (int) moIn.getNumColumnsPerBlock(), _pformat);
        // ensure correctness of output characteristics (required if input unknown during compile and no recompile)
        MatrixCharacteristics mc = new MatrixCharacteristics(moIn.getNumRows(), moIn.getNumColumns(), (int) moIn.getNumRowsPerBlock(), (int) moIn.getNumColumnsPerBlock(), moIn.getNnz());
        MetaDataFormat meta = new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
        moOut.setMetaData(meta);
    } catch (Exception ex) {
        throw new DMLRuntimeException("Failed to execute data partitioning instruction.", ex);
    }
    // release input
    ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
}

Example 54 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class Matrix method setMatrixDoubleArray.

/**
 * Method to set matrix as double array. This should only be used if the
 * user knows the matrix fits in memory. We are using the dense
 * representation.
 *
 * @param mb matrix block
 * @param oinfo output info
 * @param iinfo input info
 * @throws IOException if IOException occurs
 */
public void setMatrixDoubleArray(MatrixBlock mb, OutputInfo oinfo, InputInfo iinfo) throws IOException {
    _rows = mb.getNumRows();
    _cols = mb.getNumColumns();
    long nnz = mb.getNonZeros();
    int rblen = ConfigurationManager.getBlocksize();
    int cblen = ConfigurationManager.getBlocksize();
    MatrixCharacteristics mc = new MatrixCharacteristics(_rows, _cols, rblen, cblen, nnz);
    MetaDataFormat mfmd = new MetaDataFormat(mc, oinfo, iinfo);
    try {
        // check for correct sparse/dense representation
        if (mb.getInMemorySize() < OptimizerUtils.SAFE_REP_CHANGE_THRES)
            mb.examSparsity();
        // construct output matrix object
        _mo = new MatrixObject(Expression.ValueType.DOUBLE, _filePath, mfmd);
        _mo.acquireModify(mb);
        _mo.release();
    } catch (Exception e) {
        throw new IOException(e);
    }
}