use of edu.iu.dsc.tws.comms.batch.BKeyedGather in project twister2 by DSC-SPIDAL.
the class BKeyedGatherExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
// create the communication
keyedGather = new BKeyedGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new FinalReduceReceiver(), new SimpleKeyBasedSelector());
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
for (int t : tasksOfExecutor) {
finishedSources.put(t, false);
}
if (tasksOfExecutor.size() == 0) {
sourcesDone = true;
}
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
int target = Integer.valueOf(args.get("target").toString());
Set<Integer> keysRoutedToThis = new HashSet<>();
for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
if (i % logicalPlanBuilder.getTargets().size() == target - lowestTarget) {
keysRoutedToThis.add(i);
}
}
List<int[]> dataForEachKey = new ArrayList<>();
for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
dataForEachKey.add(ints);
}
List<Tuple<Integer, Iterator<int[]>>> expectedData = new ArrayList<>();
for (Integer key : keysRoutedToThis) {
expectedData.add(new Tuple<>(key, dataForEachKey.iterator()));
}
return expectedData.iterator();
}, new IteratorComparator<>(new TupleComparator<>(// any int
(d1, d2) -> true, new IteratorComparator<>(IntArrayComparator.getInstance()))));
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new MapWorker(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.comms.batch.BKeyedGather in project twister2 by DSC-SPIDAL.
the class BDKeyedGatherExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
Set<Integer> sources = new HashSet<>();
Integer noOfSourceTasks = jobParameters.getTaskStages().get(0);
for (int i = 0; i < noOfSourceTasks; i++) {
sources.add(i);
}
Set<Integer> targets = new HashSet<>();
Integer noOfTargetTasks = jobParameters.getTaskStages().get(1);
for (int i = 0; i < noOfTargetTasks; i++) {
targets.add(noOfSourceTasks + i);
}
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
// create the communication
keyedGather = new BKeyedGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new FinalReduceReceiver(), new SimpleKeyBasedSelector(), true, Comparator.comparingInt(o -> (Integer) o), true);
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
for (int t : tasksOfExecutor) {
finishedSources.put(t, false);
}
if (tasksOfExecutor.size() == 0) {
sourcesDone = true;
}
this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
int lowestTarget = targets.stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
int target = Integer.valueOf(args.get("target").toString());
Set<Integer> keysRoutedToThis = new HashSet<>();
for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
if (i % targets.size() == target - lowestTarget) {
keysRoutedToThis.add(i);
}
}
List<int[]> dataForEachKey = new ArrayList<>();
for (int i = 0; i < sources.size(); i++) {
dataForEachKey.add(ints);
}
List<Tuple<Integer, Iterator<int[]>>> expectedData = new ArrayList<>();
for (Integer key : keysRoutedToThis) {
expectedData.add(new Tuple<>(key, dataForEachKey.iterator()));
}
return expectedData.iterator();
}, new IteratorComparator<>(new TupleComparator<>(// any int
(d1, d2) -> true, new IteratorComparator<>(IntArrayComparator.getInstance()))));
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, sources, 1, tasksOfExecutor));
// now initialize the workers
for (int t : tasksOfExecutor) {
// the map thread where data is produced
MapWorker mapWorker = new MapWorker(t);
mapWorker.setTimingForLowestTargetOnly(true);
Thread mapThread = new Thread(mapWorker);
mapThread.start();
}
}
use of edu.iu.dsc.tws.comms.batch.BKeyedGather in project twister2 by DSC-SPIDAL.
the class SortJob method execute.
@Override
public void execute(Config cfg, JobAPI.Job job, IWorkerController workerController, IPersistentVolume persistentVolume, IVolatileVolume volatileVolume) {
int workerID = workerController.getWorkerInfo().getWorkerID();
// create a worker environment & setup the network
WorkerEnvironment workerEnv = WorkerEnvironment.init(cfg, job, workerController, persistentVolume, volatileVolume);
int noOfSources = cfg.getIntegerValue(ARG_TASKS_SOURCES, 4);
int noOfTargets = cfg.getIntegerValue(ARG_TASKS_SINKS, 4);
taskStages.add(noOfSources);
taskStages.add(noOfTargets);
// lets create the task plan
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(taskStages.get(0), taskStages.get(1), workerEnv).withFairDistribution();
int valueSize = cfg.getIntegerValue(SortJob.ARG_VALUE_SIZE, 90);
int keySize = cfg.getIntegerValue(SortJob.ARG_KEY_SIZE, 10);
MessageSchema schema = MessageSchema.noSchema();
if (cfg.getBooleanValue(ARG_FIXED_SCHEMA, false)) {
LOG.info("Using fixed schema feature with message size : " + (keySize + valueSize) + " and key size : " + keySize);
schema = MessageSchema.ofSize(keySize + valueSize, keySize);
}
gather = new BKeyedGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.BYTE_ARRAY, MessageTypes.BYTE_ARRAY, new RecordSave(), new ByteSelector(), true, new IntegerComparator(), true, schema);
int thisSource = logicalPlanBuilder.getSourcesOnThisWorker().iterator().next();
RecordSource source = new RecordSource(cfg, workerID, gather, thisSource);
long start = System.currentTimeMillis();
// run until we send
source.run();
// wait until we receive
progress();
LOG.info("Time: " + (System.currentTimeMillis() - start));
}
Aggregations