use of edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction in project twister2 by DSC-SPIDAL.
the class SAllReduceExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
// create the communication
reduce = new SAllReduce(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver());
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
for (int t : tasksOfExecutor) {
finishedSources.put(t, false);
}
sourcesDone = tasksOfExecutor.size() == 0;
Set<Integer> targetTasksOfExecutor = logicalPlanBuilder.getTargetsOnThisWorker();
for (int taskId : targetTasksOfExecutor) {
if (logicalPlanBuilder.getTargets().contains(taskId)) {
reduceDone = false;
}
if (workerId == 0) {
receiverInWorker0 = taskId;
}
}
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (array, args) -> {
int sourcesCount = jobParameters.getTaskStages().get(0);
return GeneratorUtils.multiplyIntArray(array, sourcesCount);
}, IntArrayComparator.getInstance());
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new MapWorker(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction in project twister2 by DSC-SPIDAL.
the class SKeyedReduceExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
keyedReduce = new SKeyedReduce(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver(jobParameters.getIterations()), new SimpleKeyBasedSelector());
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
for (int t : tasksOfExecutor) {
finishedSources.put(t, false);
}
if (tasksOfExecutor.size() == 0) {
sourcesDone = true;
}
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new KeyedBenchWorker.MapWorker(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction in project twister2 by DSC-SPIDAL.
the class BReduceExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
// create the communication
reduce = new BReduce(workerEnv.getCommunicator(), logicalPlanBuilder, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver(), MessageTypes.INTEGER_ARRAY);
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
for (int t : tasksOfExecutor) {
finishedSources.put(t, false);
}
if (tasksOfExecutor.size() == 0) {
sourcesDone = true;
}
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> GeneratorUtils.multiplyIntArray(ints, jobParameters.getTotalIterations() * logicalPlanBuilder.getSources().size()), IntArrayComparator.getInstance());
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), logicalPlanBuilder.getTargets().iterator().next(), tasksOfExecutor));
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new MapWorker(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction in project twister2 by DSC-SPIDAL.
the class SVMCommsReducer method execute.
@Override
protected void execute(WorkerEnvironment workerEnv) {
Set<Integer> sources = new HashSet<>();
Integer noOfSourceTasks = svmJobParameters.getParallelism();
for (int i = 0; i < noOfSourceTasks; i++) {
sources.add(i);
}
int target = noOfSourceTasks;
reduce = new BReduce(workerEnv.getCommunicator(), logicalPlan, sources, target, new ReduceOperationFunction(Op.SUM, MessageTypes.DOUBLE), new FinalSingularReceiver(), MessageTypes.DOUBLE);
Set<Integer> tasksOfExecutor = Utils.getTasksOfExecutor(workerId, logicalPlan, taskStages, 0);
for (int t : tasksOfExecutor) {
finishedSources.put(t, false);
}
if (tasksOfExecutor.size() == 0) {
sourcesDone = true;
}
if (!logicalPlan.getLogicalIdsOfWorker(workerId).contains(target)) {
reduceDone = true;
}
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, sources, target, tasksOfExecutor));
for (int t : tasksOfExecutor) {
Thread mapThread = new Thread(new DataStreamer(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction in project twister2 by DSC-SPIDAL.
the class BAllReduceExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
Integer noOfSourceTasks = jobParameters.getTaskStages().get(0);
Set<Integer> sources = IntStream.range(0, noOfSourceTasks).boxed().collect(Collectors.toSet());
int noOfTargetTasks = jobParameters.getTaskStages().get(1);
Set<Integer> targets = IntStream.range(noOfSourceTasks, noOfTargetTasks + noOfSourceTasks).boxed().collect(Collectors.toSet());
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
// create the communication
reduce = new BAllReduce(workerEnv.getCommunicator(), logicalPlanBuilder, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver(), MessageTypes.INTEGER_ARRAY);
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
for (int t : tasksOfExecutor) {
finishedSources.put(t, false);
}
if (tasksOfExecutor.size() == 0) {
sourcesDone = true;
}
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> GeneratorUtils.multiplyIntArray(ints, jobParameters.getIterations() * sources.size()), IntArrayComparator.getInstance());
LOG.log(Level.INFO, String.format("%d Sources %s target %s this %s", workerId, sources, targets, tasksOfExecutor));
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
mapThread.start();
}
}
Aggregations