Examples with Data org.apache.sysml.runtime.instructions.cp.Data used on opensource projects

Example 36 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class RemoteParWorkerMapper method close.

@Override
public void close() throws IOException {
    // cleanup cache and local tmp dir
    RemoteParForUtils.cleanupWorkingDirectories();
    // delete evicted matrices as well.
    if (ParForProgramBlock.ALLOW_REUSE_MR_PAR_WORKER) {
        for (RemoteParWorkerMapper pw : _sCache.values()) {
            LocalVariableMap vars = pw._ec.getVariables();
            for (String varName : vars.keySet()) {
                Data dat = vars.get(varName);
                if (dat instanceof MatrixObject)
                    ((MatrixObject) dat).setEmptyStatus();
            }
        }
    }
    // ensure caching is not disabled for CP in local mode
    CacheableData.enableCaching();
}

Example 37 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class OptimizerRuleBased method rewriteSetTranposeSparseVectorOperations.

// /////
// REWRITE transpose sparse vector operations
// /
protected void rewriteSetTranposeSparseVectorOperations(OptNode pn, HashMap<String, PartitionFormat> partitionedMatrices, LocalVariableMap vars) {
    // assertions (warnings of corrupt optimizer decisions)
    if (pn.getNodeType() != NodeType.PARFOR)
        LOG.warn(getOptMode() + " OPT: Transpose sparse vector operations is only applicable for a ParFor node.");
    boolean apply = false;
    ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(pn.getID())[1];
    if (pfpb.getExecMode() == PExecMode.REMOTE_MR_DP && // general applicable
    partitionedMatrices.size() == 1) {
        String moVarname = partitionedMatrices.keySet().iterator().next();
        PartitionFormat moDpf = partitionedMatrices.get(moVarname);
        Data dat = vars.get(moVarname);
        if (dat != null && dat instanceof MatrixObject && moDpf == PartitionFormat.COLUMN_WISE && // check for sparse matrix
        ((MatrixObject) dat).getSparsity() <= MatrixBlock.SPARSITY_TURN_POINT && // tranpose-safe
        rIsTransposeSafePartition(pn, moVarname)) {
            pfpb.setTransposeSparseColumnVector(true);
            apply = true;
        }
    }
    LOG.debug(getOptMode() + " OPT: rewrite 'set transpose sparse vector operations' - result=" + apply);
}

Example 38 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method execute.

/**
 * Method to be invoked to execute instructions for the external function
 * invocation
 */
@Override
public void execute(ExecutionContext ec) {
    _runID = _idSeq.getNextID();
    changeTmpInput(_runID, ec);
    changeTmpOutput(_runID);
    // export input variables to HDFS (see RunMRJobs)
    ArrayList<DataIdentifier> inputParams = null;
    try {
        inputParams = getInputParams();
        for (DataIdentifier di : inputParams) {
            Data d = ec.getVariable(di.getName());
            if (d.getDataType().isMatrix())
                ((MatrixObject) d).exportData();
        }
    } catch (Exception e) {
        throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error exporting input variables to HDFS", e);
    }
    // convert block to cell
    if (block2CellInst != null) {
        ArrayList<Instruction> tempInst = new ArrayList<>();
        tempInst.addAll(block2CellInst);
        try {
            this.executeInstructions(tempInst, ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error executing " + tempInst.toString(), e);
        }
    }
    // now execute package function
    for (int i = 0; i < _inst.size(); i++) {
        try {
            if (_inst.get(i) instanceof ExternalFunctionInvocationInstruction)
                ((ExternalFunctionInvocationInstruction) _inst.get(i)).processInstruction(ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction " + _inst.get(i).toString(), e);
        }
    }
    // convert cell to block
    if (cell2BlockInst != null) {
        ArrayList<Instruction> tempInst = new ArrayList<>();
        try {
            tempInst.clear();
            tempInst.addAll(cell2BlockInst);
            this.executeInstructions(tempInst, ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction " + cell2BlockInst.toString(), e);
        }
    }
    // check return values
    checkOutputParameters(ec.getVariables());
}

Example 39 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class ParForProgramBlock method handleDataPartitioning.

private void handleDataPartitioning(ExecutionContext ec) {
    PDataPartitioner dataPartitioner = _dataPartitioner;
    if (dataPartitioner != PDataPartitioner.NONE) {
        ParForStatementBlock sb = (ParForStatementBlock) getStatementBlock();
        if (sb == null)
            throw new DMLRuntimeException("ParFor statement block required for reasoning about data partitioning.");
        for (String var : sb.getReadOnlyParentVars()) {
            Data dat = ec.getVariable(var);
            // partitioning but typically related branches are never executed)
            if (dat != null && dat instanceof MatrixObject) {
                // unpartitioned input
                MatrixObject moVar = (MatrixObject) dat;
                PartitionFormat dpf = sb.determineDataPartitionFormat(var);
                LOG.trace("PARFOR ID = " + _ID + ", Partitioning read-only input variable " + var + " (format=" + dpf + ", mode=" + _dataPartitioner + ")");
                if (dpf != PartitionFormat.NONE) {
                    if (dataPartitioner != PDataPartitioner.REMOTE_SPARK && dpf.isBlockwise()) {
                        LOG.warn("PARFOR ID = " + _ID + ", Switching data partitioner from " + dataPartitioner + " to " + PDataPartitioner.REMOTE_SPARK.name() + " for blockwise-n partitioning.");
                        dataPartitioner = PDataPartitioner.REMOTE_SPARK;
                    }
                    Timing ltime = new Timing(true);
                    // input data partitioning (reuse if possible)
                    Data dpdatNew = _variablesDPReuse.get(var);
                    if (// no reuse opportunity
                    dpdatNew == null) {
                        DataPartitioner dp = createDataPartitioner(dpf, dataPartitioner, ec);
                        // disable binary cell for sparse if consumed by MR jobs
                        if (!OptimizerRuleBased.allowsBinaryCellPartitions(moVar, dpf) || // TODO support for binarycell
                        OptimizerUtils.isSparkExecutionMode()) {
                            dp.disableBinaryCell();
                        }
                        MatrixObject moVarNew = dp.createPartitionedMatrixObject(moVar, constructDataPartitionsFileName());
                        dpdatNew = moVarNew;
                        // skip remaining partitioning logic if not partitioned (e.g., too small)
                        if (moVar == moVarNew)
                            // skip to next
                            continue;
                    }
                    ec.setVariable(var, dpdatNew);
                    // recompile parfor body program
                    ProgramRecompiler.rFindAndRecompileIndexingHOP(sb, this, var, ec, true);
                    // store original and partitioned matrix (for reuse if applicable)
                    _variablesDPOriginal.put(var, moVar);
                    if (ALLOW_REUSE_PARTITION_VARS && ProgramRecompiler.isApplicableForReuseVariable(sb.getDMLProg(), sb, var)) {
                        _variablesDPReuse.put(var, dpdatNew);
                    }
                    LOG.trace("Partitioning and recompilation done in " + ltime.stop() + "ms");
                }
            }
        }
    }
}

Example 40 with Data

use of org.apache.sysml.runtime.instructions.cp.Data in project incubator-systemml by apache.

the class MRJobInstruction method extractOutputMatrices.

/**
 * Extracts MatrixObject references to output variables, all of which will be
 * of MATRIX data type, and stores them in <code>outputMatrices</code>. Also,
 * populates auxiliary data structures.
 *
 * @param ec execution context
 * @return array of matrix objects
 */
public MatrixObject[] extractOutputMatrices(ExecutionContext ec) {
    outputMatrices = new MatrixObject[getOutputVars().length];
    int ind = 0;
    for (String oo : getOutputVars()) {
        Data d = ec.getVariable(oo);
        if (d.getDataType() == DataType.MATRIX) {
            outputMatrices[ind++] = (MatrixObject) d;
        } else {
            throw new DMLRuntimeException(getJobType() + ": invalid datatype (" + d.getDataType() + ") for output variable " + oo);
        }
    }
    // populate auxiliary data structures
    populateOutputs();
    return outputMatrices;
}