Examples with MatrixObject org.apache.sysml.runtime.controlprogram.caching.MatrixObject used on opensource projects

Example 71 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class Recompiler method reconcileUpdatedCallVarsIf.

public static LocalVariableMap reconcileUpdatedCallVarsIf(LocalVariableMap oldCallVars, LocalVariableMap callVarsIf, LocalVariableMap callVarsElse, StatementBlock sb) {
    for (String varname : sb.variablesUpdated().getVariableNames()) {
        Data origVar = oldCallVars.get(varname);
        Data ifVar = callVarsIf.get(varname);
        Data elseVar = callVarsElse.get(varname);
        Data dat1 = null, dat2 = null;
        if (ifVar != null && elseVar != null) {
            // both branches exists
            dat1 = ifVar;
            dat2 = elseVar;
        } else if (ifVar != null && elseVar == null) {
            // only if
            dat1 = origVar;
            dat2 = ifVar;
        } else {
            // only else
            dat1 = origVar;
            dat2 = elseVar;
        }
        // because we do not allow data type changes)
        if (dat1 != null && dat1 instanceof MatrixObject && dat2 != null) {
            // handle matrices
            if (dat1 instanceof MatrixObject && dat2 instanceof MatrixObject) {
                MatrixObject moOld = (MatrixObject) dat1;
                MatrixObject mo = (MatrixObject) dat2;
                MatrixCharacteristics mcOld = moOld.getMatrixCharacteristics();
                MatrixCharacteristics mc = mo.getMatrixCharacteristics();
                if (mcOld.getRows() != mc.getRows() || mcOld.getCols() != mc.getCols() || mcOld.getNonZeros() != mc.getNonZeros()) {
                    long ldim1 = mc.getRows(), ldim2 = mc.getCols(), lnnz = mc.getNonZeros();
                    // handle row dimension change
                    if (mcOld.getRows() != mc.getRows()) {
                        // unknown
                        ldim1 = -1;
                    }
                    if (mcOld.getCols() != mc.getCols()) {
                        // unknown
                        ldim2 = -1;
                    }
                    // handle sparsity change
                    if (mcOld.getNonZeros() != mc.getNonZeros()) {
                        // unknown
                        lnnz = -1;
                    }
                    MatrixObject moNew = createOutputMatrix(ldim1, ldim2, lnnz);
                    callVarsIf.put(varname, moNew);
                }
            }
        }
    }
    return callVarsIf;
}

Example 72 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class Recompiler method executeInMemoryMatrixReblock.

public static void executeInMemoryMatrixReblock(ExecutionContext ec, String varin, String varout) {
    MatrixObject in = ec.getMatrixObject(varin);
    MatrixObject out = ec.getMatrixObject(varout);
    // read text input matrix (through buffer pool, matrix object carries all relevant
    // information including additional arguments for csv reblock)
    MatrixBlock mb = in.acquireRead();
    // set output (incl update matrix characteristics)
    out.acquireModify(mb);
    out.release();
    in.release();
}

Example 73 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class RunMRJobs method getNNZ.

private static long[] getNNZ(MatrixObject[] inputMatrices) {
    int len = inputMatrices.length;
    long[] ret = new long[len];
    for (int i = 0; i < len; i++) {
        MatrixObject mo = inputMatrices[i];
        if (mo != null)
            ret[i] = mo.getNnz();
        else
            ret[i] = -1;
    }
    return ret;
}

Example 74 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method changeTmpInput.

private void changeTmpInput(long id, ExecutionContext ec) {
    ArrayList<DataIdentifier> inputParams = getInputParams();
    block2CellInst = getBlock2CellInstructions(inputParams, _unblockedFileNames);
    // post processing FUNCTION PATCH
    for (String var : _skipInReblock) {
        Data dat = ec.getVariable(var);
        if (dat instanceof MatrixObject)
            _unblockedFileNames.put(var, ((MatrixObject) dat).getFileName());
    }
}

Example 75 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class ParForProgramBlock method consolidateAndCheckResults.

private void consolidateAndCheckResults(ExecutionContext ec, long expIters, long expTasks, long numIters, long numTasks, LocalVariableMap[] results) {
    Timing time = new Timing(true);
    // result merge
    if (checkParallelRemoteResultMerge()) {
        // execute result merge in parallel for all result vars
        int par = Math.min(_resultVars.size(), InfrastructureAnalyzer.getLocalParallelism());
        if (InfrastructureAnalyzer.isLocalMode()) {
            int parmem = (int) Math.floor(OptimizerUtils.getLocalMemBudget() / InfrastructureAnalyzer.getRemoteMaxMemorySortBuffer());
            // reduce k if necessary
            par = Math.min(par, Math.max(parmem, 1));
        }
        try {
            // enqueue all result vars as tasks
            LocalTaskQueue<ResultVar> q = new LocalTaskQueue<>();
            for (ResultVar var : _resultVars) {
                // foreach non-local write
                if (// robustness scalars
                ec.getVariable(var._name) instanceof MatrixObject)
                    q.enqueueTask(var);
            }
            q.closeInput();
            // run result merge workers
            ResultMergeWorker[] rmWorkers = new ResultMergeWorker[par];
            for (int i = 0; i < par; i++) rmWorkers[i] = new ResultMergeWorker(q, results, ec);
            for (// start all
            int i = 0; // start all
            i < par; // start all
            i++) rmWorkers[i].start();
            for (int i = 0; i < par; i++) {
                // wait for all
                rmWorkers[i].join();
                if (!rmWorkers[i].finishedNoError())
                    throw new DMLRuntimeException("Error occured in parallel result merge worker.");
            }
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
    } else {
        // execute result merge sequentially for all result vars
        for (// foreach non-local write
        ResultVar var : // foreach non-local write
        _resultVars) {
            Data dat = ec.getVariable(var._name);
            if (// robustness scalars
            dat instanceof MatrixObject) {
                MatrixObject out = (MatrixObject) dat;
                MatrixObject[] in = new MatrixObject[results.length];
                for (int i = 0; i < results.length; i++) in[i] = (MatrixObject) results[i].get(var._name);
                String fname = constructResultMergeFileName();
                ResultMerge rm = createResultMerge(_resultMerge, out, in, fname, var._isAccum, ec);
                MatrixObject outNew = null;
                if (USE_PARALLEL_RESULT_MERGE)
                    outNew = rm.executeParallelMerge(_numThreads);
                else
                    outNew = rm.executeSerialMerge();
                // cleanup existing var
                Data exdata = ec.removeVariable(var._name);
                if (exdata != null && exdata != outNew && exdata instanceof MatrixObject)
                    ec.cleanupCacheableData((MatrixObject) exdata);
                // cleanup of intermediate result variables
                cleanWorkerResultVariables(ec, out, in);
                // set merged result variable
                ec.setVariable(var._name, outNew);
            }
        }
    }
    // handle unscoped variables (vars created in parfor, but potentially used afterwards)
    ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
    if (// sb might be null for nested parallelism
    CREATE_UNSCOPED_RESULTVARS && sb != null && ec.getVariables() != null)
        createEmptyUnscopedVariables(ec.getVariables(), sb);
    // check expected counters
    if (// consistency check
    numTasks != expTasks || numIters != expIters)
        throw new DMLRuntimeException("PARFOR: Number of executed tasks does not match the number of created tasks: tasks " + numTasks + "/" + expTasks + ", iters " + numIters + "/" + expIters + ".");
    if (DMLScript.STATISTICS)
        Statistics.incrementParForMergeTime((long) time.stop());
}