use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.
the class BKeyedGatherExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
// create the communication
keyedGather = new BKeyedGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new FinalReduceReceiver(), new SimpleKeyBasedSelector());
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
for (int t : tasksOfExecutor) {
finishedSources.put(t, false);
}
if (tasksOfExecutor.size() == 0) {
sourcesDone = true;
}
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
int target = Integer.valueOf(args.get("target").toString());
Set<Integer> keysRoutedToThis = new HashSet<>();
for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
if (i % logicalPlanBuilder.getTargets().size() == target - lowestTarget) {
keysRoutedToThis.add(i);
}
}
List<int[]> dataForEachKey = new ArrayList<>();
for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
dataForEachKey.add(ints);
}
List<Tuple<Integer, Iterator<int[]>>> expectedData = new ArrayList<>();
for (Integer key : keysRoutedToThis) {
expectedData.add(new Tuple<>(key, dataForEachKey.iterator()));
}
return expectedData.iterator();
}, new IteratorComparator<>(new TupleComparator<>(// any int
(d1, d2) -> true, new IteratorComparator<>(IntArrayComparator.getInstance()))));
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new MapWorker(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.
the class BReduceExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
// create the communication
reduce = new BReduce(workerEnv.getCommunicator(), logicalPlanBuilder, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver(), MessageTypes.INTEGER_ARRAY);
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
for (int t : tasksOfExecutor) {
finishedSources.put(t, false);
}
if (tasksOfExecutor.size() == 0) {
sourcesDone = true;
}
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> GeneratorUtils.multiplyIntArray(ints, jobParameters.getTotalIterations() * logicalPlanBuilder.getSources().size()), IntArrayComparator.getInstance());
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), logicalPlanBuilder.getTargets().iterator().next(), tasksOfExecutor));
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new MapWorker(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.
the class SGatherExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
if (jobParameters.getTargets() != 1) {
LOG.warning("Setting targets to 1. Found, " + jobParameters.getTargets());
jobParameters.getTaskStages().set(1, 1);
}
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
// create the communication
gather = new SGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new FinalReduceReceiver());
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
for (int t : tasksOfExecutor) {
finishedSources.put(t, false);
}
if (tasksOfExecutor.size() == 0) {
sourcesDone = true;
}
if (!logicalPlan.getLogicalIdsOfWorker(workerId).contains(logicalPlanBuilder.getTargets().iterator().next())) {
gatherDone = true;
}
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (dataArray, args) -> {
List<Tuple<Integer, int[]>> listOfArrays = new ArrayList<>();
for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
listOfArrays.add(new Tuple<>(i, dataArray));
}
return listOfArrays.iterator();
}, new IteratorComparator<>(new TupleComparator<>(IntComparator.getInstance(), IntArrayComparator.getInstance())));
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.
the class SKeyedPartitionExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
// create the communication
partition = new SKeyedPartition(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new PartitionReceiver(), new HashingSelector());
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> new Tuple<>(-1, ints), new TupleComparator<>(// any int
(d1, d2) -> true, IntArrayComparator.getInstance()));
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new MapWorker(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.
the class SPartitionExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
// create the communication
partition = new SPartition(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new PartitionReceiver(), new LoadBalanceSelector());
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> ints, IntArrayComparator.getInstance());
logicalPlanBuilder.getTargetsOnThisWorker().forEach(t -> {
if (workerId == 0) {
recvrInWorker0 = t;
}
});
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new MapWorker(t));
mapThread.start();
}
}
Aggregations