Examples with DataPartitionerRemoteSpark org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteSpark used on opensource projects

Example 1 with DataPartitionerRemoteSpark

use of org.apache.sysml.runtime.controlprogram.parfor.DataPartitionerRemoteSpark in project incubator-systemml by apache.

the class ParForProgramBlock method createDataPartitioner.

/**
	 * Creates a new data partitioner according to the specified runtime parameter.
	 * 
	 * @param dpf data partition format
	 * @param dataPartitioner data partitioner
	 * @param ec execution context
	 * @return data partitioner
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
private DataPartitioner createDataPartitioner(PartitionFormat dpf, PDataPartitioner dataPartitioner, ExecutionContext ec) throws DMLRuntimeException {
    DataPartitioner dp = null;
    //determine max degree of parallelism
    int numReducers = ConfigurationManager.getNumReducers();
    int maxNumRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
    //correction max number of reducers on yarn clusters
    if (InfrastructureAnalyzer.isYarnEnabled())
        maxNumRed = (int) Math.max(maxNumRed, YarnClusterAnalyzer.getNumCores() / 2);
    int numRed = Math.min(numReducers, maxNumRed);
    //create data partitioner
    switch(dataPartitioner) {
        case LOCAL:
            dp = new DataPartitionerLocal(dpf, _numThreads);
            break;
        case REMOTE_MR:
            dp = new DataPartitionerRemoteMR(dpf, _ID, numRed, _replicationDP, ALLOW_REUSE_MR_JVMS, false);
            break;
        case REMOTE_SPARK:
            dp = new DataPartitionerRemoteSpark(dpf, ec, numRed, _replicationDP, false);
            break;
        default:
            throw new DMLRuntimeException("Unknown data partitioner: '" + dataPartitioner.name() + "'.");
    }
    return dp;
}