use of edu.iu.dsc.tws.comms.batch.BJoin in project twister2 by DSC-SPIDAL.
the class BDJoinExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
// create the communication
join = new BJoin(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, MessageTypes.INTEGER_ARRAY, new JoinReceiver(), new SimpleKeyBasedSelector(), true, Comparator.comparingInt(o -> (Integer) o), CommunicationContext.JoinType.INNER, CommunicationContext.JoinAlgorithm.SORT);
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
// now initialize the workers
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new MapWorker(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.comms.batch.BJoin in project twister2 by DSC-SPIDAL.
the class BJoinStudentExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
// create the join communication
join = new BJoin(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.OBJECT, MessageTypes.OBJECT, new JoinReceiver(), new SimpleKeyBasedSelector(), false, (o1, o2) -> {
if (o1 instanceof String && o2 instanceof String) {
return ((String) o1).compareTo((String) o2);
}
return 0;
}, CommunicationContext.JoinType.INNER, CommunicationContext.JoinAlgorithm.SORT);
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
// now initialize the workers
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new MapWorker(t));
mapThread.start();
}
}
use of edu.iu.dsc.tws.comms.batch.BJoin in project twister2 by DSC-SPIDAL.
the class BJoinExample method compute.
@Override
protected void compute(WorkerEnvironment workerEnv) {
Set<Integer> sources = new HashSet<>();
Set<Integer> targets = new HashSet<>();
Integer noOfSourceTasks = jobParameters.getTaskStages().get(0);
for (int i = 0; i < noOfSourceTasks; i++) {
sources.add(i);
}
Integer noOfTargetTasks = jobParameters.getTaskStages().get(1);
for (int i = 0; i < noOfTargetTasks; i++) {
targets.add(noOfSourceTasks + i);
}
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
// create the communication
join = new BJoin(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, MessageTypes.INTEGER_ARRAY, new JoinReceiver(), new SimpleKeyBasedSelector(), false, Comparator.comparingInt(o -> (Integer) o), CommunicationContext.JoinType.INNER, CommunicationContext.JoinAlgorithm.SORT);
Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
// now initialize the workers
LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, sources, 1, tasksOfExecutor));
for (int t : tasksOfExecutor) {
// the map thread where data is produced
Thread mapThread = new Thread(new JoinedKeyedBenchWorker.MapWorker(t));
mapThread.start();
}
}
Aggregations