Examples with BKeyedReduce edu.iu.dsc.tws.comms.batch.BKeyedReduce used on opensource projects

Example 1 with BKeyedReduce

use of edu.iu.dsc.tws.comms.batch.BKeyedReduce in project twister2 by DSC-SPIDAL.

the class WordCountWorker method execute.

@Override
public void execute(WorkerEnvironment wEnv) {
    this.workerEnv = wEnv;
    this.workerId = workerEnv.getWorkerId();
    taskStages.add(NO_OF_TASKS);
    taskStages.add(NO_OF_TASKS);
    // lets create the task plan
    this.logicalPlan = Utils.createStageLogicalPlan(workerEnv, taskStages);
    setupTasks();
    // create the communication
    wordAggregator = new WordAggregator();
    keyGather = new BKeyedReduce(workerEnv.getCommunicator(), logicalPlan, sources, destinations, new ReduceFunction() {

        @Override
        public void init(Config cfg, DataFlowOperation op, Map<Integer, List<Integer>> expectedIds) {
        }

        @Override
        public Object reduce(Object t1, Object t2) {
            return (Integer) t1 + (Integer) t2;
        }
    }, wordAggregator, MessageTypes.OBJECT, MessageTypes.INTEGER, new HashingSelector());
    // assign the task ids to the workers, and run them using threads
    scheduleTasks();
    // progress the communication
    progress();
    // close communication
    workerEnv.close();
}

Example 2 with BKeyedReduce

use of edu.iu.dsc.tws.comms.batch.BKeyedReduce in project twister2 by DSC-SPIDAL.

the class BKeyedReduceExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    keyedReduce = new BKeyedReduce(workerEnv.getCommunicator(), logicalPlanBuilder, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalBulkReceiver(), MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new SimpleKeyBasedSelector());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
        int target = Integer.parseInt(args.get("target").toString());
        Set<Integer> keysRoutedToThis = new HashSet<>();
        for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
            if (i % logicalPlanBuilder.getTargets().size() == target - lowestTarget) {
                keysRoutedToThis.add(i);
            }
        }
        int[] reduced = GeneratorUtils.multiplyIntArray(ints, logicalPlanBuilder.getSources().size());
        List<Tuple<Integer, int[]>> expectedData = new ArrayList<>();
        for (Integer key : keysRoutedToThis) {
            expectedData.add(new Tuple<>(key, reduced));
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(IntComparator.getInstance(), IntArrayComparator.getInstance())));
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new KeyedBenchWorker.MapWorker(t));
        mapThread.start();
    }
}