Examples with MatrixObject org.apache.sysml.runtime.controlprogram.caching.MatrixObject used on opensource projects

Example 31 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class ResultMergeLocalAutomatic method executeSerialMerge.

@Override
public MatrixObject executeSerialMerge() {
    Timing time = new Timing(true);
    MatrixCharacteristics mc = _output.getMatrixCharacteristics();
    long rows = mc.getRows();
    long cols = mc.getCols();
    if (OptimizerRuleBased.isInMemoryResultMerge(rows, cols, OptimizerUtils.getLocalMemBudget()))
        _rm = new ResultMergeLocalMemory(_output, _inputs, _outputFName, _isAccum);
    else
        _rm = new ResultMergeLocalFile(_output, _inputs, _outputFName, _isAccum);
    MatrixObject ret = _rm.executeSerialMerge();
    LOG.trace("Automatic result merge (" + _rm.getClass().getName() + ") executed in " + time.stop() + "ms.");
    return ret;
}

Example 32 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class ResultMergeLocalFile method executeSerialMerge.

@Override
public MatrixObject executeSerialMerge() {
    // always create new matrix object (required for nested parallelism)
    MatrixObject moNew = null;
    if (LOG.isTraceEnabled())
        LOG.trace("ResultMerge (local, file): Execute serial merge for output " + _output.hashCode() + " (fname=" + _output.getFileName() + ")");
    try {
        // collect all relevant inputs
        ArrayList<MatrixObject> inMO = new ArrayList<>();
        for (MatrixObject in : _inputs) {
            // check for empty inputs (no iterations executed)
            if (in != null && in != _output) {
                // ensure that input file resides on disk
                in.exportData();
                // add to merge list
                inMO.add(in);
            }
        }
        if (!inMO.isEmpty()) {
            // ensure that outputfile (for comparison) resides on disk
            _output.exportData();
            // actual merge
            merge(_outputFName, _output, inMO);
            // create new output matrix (e.g., to prevent potential export<->read file access conflict
            moNew = createNewMatrixObject(_output, inMO);
        } else {
            // return old matrix, to prevent copy
            moNew = _output;
        }
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    return moNew;
}

Example 33 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class ResultMergeLocalFile method mergeBinaryBlockWithComp.

private void mergeBinaryBlockWithComp(String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO) {
    String fnameStaging = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE);
    String fnameStagingCompare = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE);
    try {
        // delete target file if already exists
        MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
        // Step 0) write compare blocks to staging area (if necessary)
        if (LOG.isTraceEnabled())
            LOG.trace("ResultMerge (local, file): Create merge compare matrix for output " + outMo.hashCode() + " (fname=" + outMo.getFileName() + ")");
        createBinaryBlockStagingFile(fnameStagingCompare, outMo);
        // Step 1) read and write blocks to staging area
        for (MatrixObject in : inMO) {
            if (LOG.isTraceEnabled())
                LOG.trace("ResultMerge (local, file): Merge input " + in.hashCode() + " (fname=" + in.getFileName() + ")");
            createBinaryBlockStagingFile(fnameStaging, in);
        }
        // Step 2) read blocks, consolidate, and write to HDFS
        createBinaryBlockResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MetaDataFormat) outMo.getMetaData(), true);
    } catch (Exception ex) {
        throw new DMLRuntimeException("Unable to merge binary block results.", ex);
    }
    LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
    LocalFileUtils.cleanupWorkingDirectory(fnameStagingCompare);
}

Example 34 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class ResultMergeLocalFile method copyAllFiles.

private static void copyAllFiles(String fnameNew, ArrayList<MatrixObject> inMO) throws IOException {
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path(fnameNew);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    // create output dir
    fs.mkdirs(path);
    // merge in all input matrix objects
    IDSequence seq = new IDSequence();
    for (MatrixObject in : inMO) {
        if (LOG.isTraceEnabled())
            LOG.trace("ResultMerge (local, file): Merge input " + in.hashCode() + " (fname=" + in.getFileName() + ") via file rename.");
        // copy over files (just rename file or entire dir)
        Path tmpPath = new Path(in.getFileName());
        String lname = tmpPath.getName();
        fs.rename(tmpPath, new Path(fnameNew + "/" + lname + seq.getNextID()));
    }
}

Example 35 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class ResultMergeLocalFile method mergeBinaryBlockWithoutComp.

private void mergeBinaryBlockWithoutComp(String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO) {
    String fnameStaging = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_RESULTMERGE);
    try {
        // delete target file if already exists
        MapReduceTool.deleteFileIfExistOnHDFS(fnameNew);
        // Step 1) read and write blocks to staging area
        for (MatrixObject in : inMO) {
            if (LOG.isTraceEnabled())
                LOG.trace("ResultMerge (local, file): Merge input " + in.hashCode() + " (fname=" + in.getFileName() + ")");
            createBinaryBlockStagingFile(fnameStaging, in);
        }
        // Step 2) read blocks, consolidate, and write to HDFS
        createBinaryBlockResultFile(fnameStaging, null, fnameNew, (MetaDataFormat) outMo.getMetaData(), false);
    } catch (Exception ex) {
        throw new DMLRuntimeException("Unable to merge binary block results.", ex);
    }
    LocalFileUtils.cleanupWorkingDirectory(fnameStaging);
}