use of edu.iu.dsc.tws.comms.batch.BKeyedPartition in project twister2 by DSC-SPIDAL.
the class BKeyedPartitionExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
// create the communication
partition = new BKeyedPartition(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new PartitionReceiver(), new SimpleKeyBasedSelector());
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
// now initialize the workers
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
int target = Integer.parseInt(args.get("target").toString());
Set<Integer> keysRoutedToThis = new HashSet<>();
for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
if (i % logicalPlanBuilder.getTargets().size() == target - lowestTarget) {
keysRoutedToThis.add(i);
}
}
List<Tuple<Integer, int[]>> expectedData = new ArrayList<>();
for (Integer key : keysRoutedToThis) {
for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
expectedData.add(new Tuple<>(key, ints));
}
}
return expectedData.iterator();
}, new IteratorComparator<>(new TupleComparator<>(// any int
(d1, d2) -> true, IntArrayComparator.getInstance())));
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new KeyedBenchWorker.MapWorker(t));
mapThread.start();
}
}
Aggregations