Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 16 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class Recompiler method reconcileUpdatedCallVarsIf.

public static LocalVariableMap reconcileUpdatedCallVarsIf(LocalVariableMap oldCallVars, LocalVariableMap callVarsIf, LocalVariableMap callVarsElse, StatementBlock sb) {
    for (String varname : sb.variablesUpdated().getVariableNames()) {
        Data origVar = oldCallVars.get(varname);
        Data ifVar = callVarsIf.get(varname);
        Data elseVar = callVarsElse.get(varname);
        Data dat1 = null, dat2 = null;
        if (ifVar != null && elseVar != null) {
            // both branches exists
            dat1 = ifVar;
            dat2 = elseVar;
        } else if (ifVar != null && elseVar == null) {
            // only if
            dat1 = origVar;
            dat2 = ifVar;
        } else {
            // only else
            dat1 = origVar;
            dat2 = elseVar;
        }
        // because we do not allow data type changes)
        if (dat1 != null && dat1 instanceof MatrixObject && dat2 != null) {
            // handle matrices
            if (dat1 instanceof MatrixObject && dat2 instanceof MatrixObject) {
                MatrixObject moOld = (MatrixObject) dat1;
                MatrixObject mo = (MatrixObject) dat2;
                MatrixCharacteristics mcOld = moOld.getMatrixCharacteristics();
                MatrixCharacteristics mc = mo.getMatrixCharacteristics();
                if (mcOld.getRows() != mc.getRows() || mcOld.getCols() != mc.getCols() || mcOld.getNonZeros() != mc.getNonZeros()) {
                    long ldim1 = mc.getRows(), ldim2 = mc.getCols(), lnnz = mc.getNonZeros();
                    // handle row dimension change
                    if (mcOld.getRows() != mc.getRows()) {
                        // unknown
                        ldim1 = -1;
                    }
                    if (mcOld.getCols() != mc.getCols()) {
                        // unknown
                        ldim2 = -1;
                    }
                    // handle sparsity change
                    if (mcOld.getNonZeros() != mc.getNonZeros()) {
                        // unknown
                        lnnz = -1;
                    }
                    MatrixObject moNew = createOutputMatrix(ldim1, ldim2, lnnz);
                    callVarsIf.put(varname, moNew);
                }
            }
        }
    }
    return callVarsIf;
}

Example 17 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method changeTmpInput.

private void changeTmpInput(long id, ExecutionContext ec) {
    ArrayList<DataIdentifier> inputParams = getInputParams();
    block2CellInst = getBlock2CellInstructions(inputParams, _unblockedFileNames);
    // post processing FUNCTION PATCH
    for (String var : _skipInReblock) {
        Data dat = ec.getVariable(var);
        if (dat instanceof MatrixObject)
            _unblockedFileNames.put(var, ((MatrixObject) dat).getFileName());
    }
}

Example 18 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class FunctionProgramBlock method checkOutputParameters.

protected void checkOutputParameters(LocalVariableMap vars) {
    for (DataIdentifier diOut : _outputParams) {
        String varName = diOut.getName();
        Data dat = vars.get(varName);
        if (dat == null)
            LOG.error("Function output " + varName + " is missing.");
        else if (dat.getDataType() != diOut.getDataType())
            LOG.warn("Function output " + varName + " has wrong data type: " + dat.getDataType() + ".");
        else if (dat.getValueType() != diOut.getValueType())
            LOG.warn("Function output " + varName + " has wrong value type: " + dat.getValueType() + ".");
    }
}

Example 19 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ParForProgramBlock method consolidateAndCheckResults.

private void consolidateAndCheckResults(ExecutionContext ec, long expIters, long expTasks, long numIters, long numTasks, LocalVariableMap[] results) {
    Timing time = new Timing(true);
    // result merge
    if (checkParallelRemoteResultMerge()) {
        // execute result merge in parallel for all result vars
        int par = Math.min(_resultVars.size(), InfrastructureAnalyzer.getLocalParallelism());
        if (InfrastructureAnalyzer.isLocalMode()) {
            int parmem = (int) Math.floor(OptimizerUtils.getLocalMemBudget() / InfrastructureAnalyzer.getRemoteMaxMemorySortBuffer());
            // reduce k if necessary
            par = Math.min(par, Math.max(parmem, 1));
        }
        try {
            // enqueue all result vars as tasks
            LocalTaskQueue<ResultVar> q = new LocalTaskQueue<>();
            for (ResultVar var : _resultVars) {
                // foreach non-local write
                if (// robustness scalars
                ec.getVariable(var._name) instanceof MatrixObject)
                    q.enqueueTask(var);
            }
            q.closeInput();
            // run result merge workers
            ResultMergeWorker[] rmWorkers = new ResultMergeWorker[par];
            for (int i = 0; i < par; i++) rmWorkers[i] = new ResultMergeWorker(q, results, ec);
            for (// start all
            int i = 0; // start all
            i < par; // start all
            i++) rmWorkers[i].start();
            for (int i = 0; i < par; i++) {
                // wait for all
                rmWorkers[i].join();
                if (!rmWorkers[i].finishedNoError())
                    throw new DMLRuntimeException("Error occured in parallel result merge worker.");
            }
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
    } else {
        // execute result merge sequentially for all result vars
        for (// foreach non-local write
        ResultVar var : // foreach non-local write
        _resultVars) {
            Data dat = ec.getVariable(var._name);
            if (// robustness scalars
            dat instanceof MatrixObject) {
                MatrixObject out = (MatrixObject) dat;
                MatrixObject[] in = new MatrixObject[results.length];
                for (int i = 0; i < results.length; i++) in[i] = (MatrixObject) results[i].get(var._name);
                String fname = constructResultMergeFileName();
                ResultMerge rm = createResultMerge(_resultMerge, out, in, fname, var._isAccum, ec);
                MatrixObject outNew = null;
                if (USE_PARALLEL_RESULT_MERGE)
                    outNew = rm.executeParallelMerge(_numThreads);
                else
                    outNew = rm.executeSerialMerge();
                // cleanup existing var
                Data exdata = ec.removeVariable(var._name);
                if (exdata != null && exdata != outNew && exdata instanceof MatrixObject)
                    ec.cleanupCacheableData((MatrixObject) exdata);
                // cleanup of intermediate result variables
                cleanWorkerResultVariables(ec, out, in);
                // set merged result variable
                ec.setVariable(var._name, outNew);
            }
        }
    }
    // handle unscoped variables (vars created in parfor, but potentially used afterwards)
    ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
    if (// sb might be null for nested parallelism
    CREATE_UNSCOPED_RESULTVARS && sb != null && ec.getVariables() != null)
        createEmptyUnscopedVariables(ec.getVariables(), sb);
    // check expected counters
    if (// consistency check
    numTasks != expTasks || numIters != expIters)
        throw new DMLRuntimeException("PARFOR: Number of executed tasks does not match the number of created tasks: tasks " + numTasks + "/" + expTasks + ", iters " + numIters + "/" + expIters + ".");
    if (DMLScript.STATISTICS)
        Statistics.incrementParForMergeTime((long) time.stop());
}

Example 20 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ParForProgramBlock method exportMatricesToHDFS.

private void exportMatricesToHDFS(ExecutionContext ec, String... blacklistNames) {
    ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
    Set<String> blacklist = UtilFunctions.asSet(blacklistNames);
    if (LIVEVAR_AWARE_EXPORT && sb != null) {
        // optimization to prevent unnecessary export of matrices
        // export only variables that are read in the body
        VariableSet varsRead = sb.variablesRead();
        for (String key : ec.getVariables().keySet()) {
            if (varsRead.containsVariable(key) && !blacklist.contains(key)) {
                Data d = ec.getVariable(key);
                if (d.getDataType() == DataType.MATRIX)
                    ((MatrixObject) d).exportData(_replicationExport);
            }
        }
    } else {
        // export all matrices in symbol table
        for (String key : ec.getVariables().keySet()) {
            if (!blacklist.contains(key)) {
                Data d = ec.getVariable(key);
                if (d.getDataType() == DataType.MATRIX)
                    ((MatrixObject) d).exportData(_replicationExport);
            }
        }
    }
}