Search in sources :

Example 1 with DataPartitionerLocal

use of org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerLocal in project incubator-systemml by apache.

the class ParForProgramBlock method createDataPartitioner.

/**
 * Creates a new data partitioner according to the specified runtime parameter.
 *
 * @param dpf data partition format
 * @param dataPartitioner data partitioner
 * @param ec execution context
 * @return data partitioner
 */
private DataPartitioner createDataPartitioner(PartitionFormat dpf, PDataPartitioner dataPartitioner, ExecutionContext ec) {
    DataPartitioner dp = null;
    // determine max degree of parallelism
    int numReducers = ConfigurationManager.getNumReducers();
    int maxNumRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
    // correction max number of reducers on yarn clusters
    if (InfrastructureAnalyzer.isYarnEnabled())
        maxNumRed = (int) Math.max(maxNumRed, YarnClusterAnalyzer.getNumCores() / 2);
    int numRed = Math.min(numReducers, maxNumRed);
    // create data partitioner
    switch(dataPartitioner) {
        case LOCAL:
            dp = new DataPartitionerLocal(dpf, _numThreads);
            break;
        case REMOTE_MR:
            dp = new DataPartitionerRemoteMR(dpf, _ID, numRed, _replicationDP, ALLOW_REUSE_MR_JVMS, false);
            break;
        case REMOTE_SPARK:
            dp = new DataPartitionerRemoteSpark(dpf, ec, numRed, _replicationDP, false);
            break;
        default:
            throw new DMLRuntimeException("Unknown data partitioner: '" + dataPartitioner.name() + "'.");
    }
    return dp;
}
Also used : DataPartitioner(org.apache.sysml.runtime.controlprogram.parfor.DataPartitioner) DataPartitionerLocal(org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerLocal) DataPartitionerRemoteSpark(org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteSpark) DataPartitionerRemoteMR(org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteMR) DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException)

Example 2 with DataPartitionerLocal

use of org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerLocal in project systemml by apache.

the class ParForProgramBlock method createDataPartitioner.

/**
 * Creates a new data partitioner according to the specified runtime parameter.
 *
 * @param dpf data partition format
 * @param dataPartitioner data partitioner
 * @param ec execution context
 * @return data partitioner
 */
private DataPartitioner createDataPartitioner(PartitionFormat dpf, PDataPartitioner dataPartitioner, ExecutionContext ec) {
    DataPartitioner dp = null;
    // determine max degree of parallelism
    int numReducers = ConfigurationManager.getNumReducers();
    int maxNumRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
    // correction max number of reducers on yarn clusters
    if (InfrastructureAnalyzer.isYarnEnabled())
        maxNumRed = (int) Math.max(maxNumRed, YarnClusterAnalyzer.getNumCores() / 2);
    int numRed = Math.min(numReducers, maxNumRed);
    // create data partitioner
    switch(dataPartitioner) {
        case LOCAL:
            dp = new DataPartitionerLocal(dpf, _numThreads);
            break;
        case REMOTE_MR:
            dp = new DataPartitionerRemoteMR(dpf, _ID, numRed, _replicationDP, ALLOW_REUSE_MR_JVMS, false);
            break;
        case REMOTE_SPARK:
            dp = new DataPartitionerRemoteSpark(dpf, ec, numRed, _replicationDP, false);
            break;
        default:
            throw new DMLRuntimeException("Unknown data partitioner: '" + dataPartitioner.name() + "'.");
    }
    return dp;
}
Also used : DataPartitioner(org.apache.sysml.runtime.controlprogram.parfor.DataPartitioner) DataPartitionerLocal(org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerLocal) DataPartitionerRemoteSpark(org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteSpark) DataPartitionerRemoteMR(org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteMR) DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException)

Aggregations

DMLRuntimeException (org.apache.sysml.runtime.DMLRuntimeException)2 DataPartitioner (org.apache.sysml.runtime.controlprogram.parfor.DataPartitioner)2 DataPartitionerLocal (org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerLocal)2 DataPartitionerRemoteMR (org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteMR)2 DataPartitionerRemoteSpark (org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteSpark)2