Examples with MetaDataFormat org.apache.sysml.runtime.matrix.MetaDataFormat used on opensource projects

Example 11 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class ResultMergeRemoteSpark method executeParallelMerge.

@Override
public MatrixObject executeParallelMerge(int par) {
    // always create new matrix object (required for nested parallelism)
    MatrixObject moNew = null;
    if (LOG.isTraceEnabled())
        LOG.trace("ResultMerge (remote, spark): Execute serial merge for output " + _output.hashCode() + " (fname=" + _output.getFileName() + ")");
    try {
        if (_inputs != null && _inputs.length > 0) {
            // prepare compare
            MetaDataFormat metadata = (MetaDataFormat) _output.getMetaData();
            MatrixCharacteristics mcOld = metadata.getMatrixCharacteristics();
            MatrixObject compare = (mcOld.getNonZeros() == 0) ? null : _output;
            // actual merge
            RDDObject ro = executeMerge(compare, _inputs, mcOld.getRows(), mcOld.getCols(), mcOld.getRowsPerBlock(), mcOld.getColsPerBlock());
            // create new output matrix (e.g., to prevent potential export<->read file access conflict
            moNew = new MatrixObject(_output.getValueType(), _outputFName);
            OutputInfo oiOld = metadata.getOutputInfo();
            InputInfo iiOld = metadata.getInputInfo();
            MatrixCharacteristics mc = new MatrixCharacteristics(mcOld);
            mc.setNonZeros(_isAccum ? -1 : computeNonZeros(_output, Arrays.asList(_inputs)));
            MetaDataFormat meta = new MetaDataFormat(mc, oiOld, iiOld);
            moNew.setMetaData(meta);
            moNew.setRDDHandle(ro);
        } else {
            // return old matrix, to prevent copy
            moNew = _output;
        }
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    return moNew;
}

Example 12 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class VariableCPInstruction method writeCSVFile.

/**
 * Helper function to write CSV files to HDFS.
 *
 * @param ec execution context
 * @param fname file name
 */
private void writeCSVFile(ExecutionContext ec, String fname) {
    MatrixObject mo = ec.getMatrixObject(getInput1().getName());
    String outFmt = "csv";
    if (mo.isDirty()) {
        // there exist data computed in CP that is not backed up on HDFS
        // i.e., it is either in-memory or in evicted space
        mo.exportData(fname, outFmt, _formatProperties);
    } else {
        try {
            OutputInfo oi = ((MetaDataFormat) mo.getMetaData()).getOutputInfo();
            MatrixCharacteristics mc = ((MetaDataFormat) mo.getMetaData()).getMatrixCharacteristics();
            if (oi == OutputInfo.CSVOutputInfo) {
                WriterTextCSV writer = new WriterTextCSV((CSVFileFormatProperties) _formatProperties);
                writer.addHeaderToCSV(mo.getFileName(), fname, mc.getRows(), mc.getCols());
            } else if (oi == OutputInfo.BinaryBlockOutputInfo || oi == OutputInfo.TextCellOutputInfo) {
                mo.exportData(fname, outFmt, _formatProperties);
            } else {
                throw new DMLRuntimeException("Unexpected data format (" + OutputInfo.outputInfoToString(oi) + "): can not export into CSV format.");
            }
            // Write Metadata file
            MapReduceTool.writeMetaDataFile(fname + ".mtd", mo.getValueType(), mc, OutputInfo.CSVOutputInfo, _formatProperties);
        } catch (IOException e) {
            throw new DMLRuntimeException(e);
        }
    }
}

Example 13 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class VariableCPInstruction method writeMMFile.

/**
 * Helper function to write MM files to HDFS.
 *
 * @param ec execution context
 * @param fname file name
 */
private void writeMMFile(ExecutionContext ec, String fname) {
    MatrixObject mo = ec.getMatrixObject(getInput1().getName());
    String outFmt = "matrixmarket";
    if (mo.isDirty()) {
        // there exist data computed in CP that is not backed up on HDFS
        // i.e., it is either in-memory or in evicted space
        mo.exportData(fname, outFmt);
    } else {
        OutputInfo oi = ((MetaDataFormat) mo.getMetaData()).getOutputInfo();
        MatrixCharacteristics mc = mo.getMatrixCharacteristics();
        if (oi == OutputInfo.TextCellOutputInfo) {
            try {
                WriterMatrixMarket.mergeTextcellToMatrixMarket(mo.getFileName(), fname, mc.getRows(), mc.getCols(), mc.getNonZeros());
            } catch (IOException e) {
                throw new DMLRuntimeException(e);
            }
        } else if (oi == OutputInfo.BinaryBlockOutputInfo) {
            mo.exportData(fname, outFmt);
        } else {
            throw new DMLRuntimeException("Unexpected data format (" + OutputInfo.outputInfoToString(oi) + "): can not export into MatrixMarket format.");
        }
    }
}

Example 14 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class MatrixIndexingCPFileInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    String opcode = getOpcode();
    IndexRange ixrange = getIndexRange(ec).add(1);
    MatrixObject mo = ec.getMatrixObject(input1.getName());
    if (mo.isPartitioned() && opcode.equalsIgnoreCase(RightIndex.OPCODE)) {
        MetaDataFormat meta = (MetaDataFormat) mo.getMetaData();
        MatrixCharacteristics mc = meta.getMatrixCharacteristics();
        String pfname = mo.getPartitionFileName(ixrange, mc.getRowsPerBlock(), mc.getColsPerBlock());
        if (MapReduceTool.existsFileOnHDFS(pfname)) {
            // create output matrix object
            MatrixObject mobj = new MatrixObject(mo.getValueType(), pfname);
            MatrixCharacteristics mcNew = null;
            switch(mo.getPartitionFormat()) {
                case ROW_WISE:
                    mcNew = new MatrixCharacteristics(1, mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock());
                    break;
                case ROW_BLOCK_WISE_N:
                    mcNew = new MatrixCharacteristics(mo.getPartitionSize(), mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock());
                    break;
                case COLUMN_WISE:
                    mcNew = new MatrixCharacteristics(mc.getRows(), 1, mc.getRowsPerBlock(), mc.getColsPerBlock());
                    break;
                case COLUMN_BLOCK_WISE_N:
                    mcNew = new MatrixCharacteristics(mc.getRows(), mo.getPartitionSize(), mc.getRowsPerBlock(), mc.getColsPerBlock());
                    break;
                default:
                    throw new DMLRuntimeException("Unsupported partition format for CP_FILE " + RightIndex.OPCODE + ": " + mo.getPartitionFormat());
            }
            MetaDataFormat metaNew = new MetaDataFormat(mcNew, meta.getOutputInfo(), meta.getInputInfo());
            mobj.setMetaData(metaNew);
            // put output object into symbol table
            ec.setVariable(output.getName(), mobj);
        } else {
            // will return an empty matrix partition
            MatrixBlock resultBlock = mo.readMatrixPartition(ixrange);
            ec.setMatrixOutput(output.getName(), resultBlock, getExtendedOpcode());
        }
    } else {
        throw new DMLRuntimeException("Invalid opcode or index predicate for MatrixIndexingCPFileInstruction: " + instString);
    }
}

Example 15 with MetaDataFormat

use of org.apache.sysml.runtime.matrix.MetaDataFormat in project incubator-systemml by apache.

the class OptimizerRuleBased method determineFlagCellFormatWoCompare.

protected boolean determineFlagCellFormatWoCompare(ArrayList<ResultVar> resultVars, LocalVariableMap vars) {
    boolean ret = true;
    for (ResultVar rVar : resultVars) {
        Data dat = vars.get(rVar._name);
        if (dat == null || !(dat instanceof MatrixObject)) {
            ret = false;
            break;
        } else {
            MatrixObject mo = (MatrixObject) dat;
            MetaDataFormat meta = (MetaDataFormat) mo.getMetaData();
            OutputInfo oi = meta.getOutputInfo();
            long nnz = meta.getMatrixCharacteristics().getNonZeros();
            if (oi == OutputInfo.BinaryBlockOutputInfo || nnz != 0) {
                ret = false;
                break;
            }
        }
    }
    return ret;
}