Examples with BReduce edu.iu.dsc.tws.comms.batch.BReduce used on opensource projects

Example 1 with BReduce

use of edu.iu.dsc.tws.comms.batch.BReduce in project twister2 by DSC-SPIDAL.

the class BReduceExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    reduce = new BReduce(workerEnv.getCommunicator(), logicalPlanBuilder, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver(), MessageTypes.INTEGER_ARRAY);
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> GeneratorUtils.multiplyIntArray(ints, jobParameters.getTotalIterations() * logicalPlanBuilder.getSources().size()), IntArrayComparator.getInstance());
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), logicalPlanBuilder.getTargets().iterator().next(), tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}

Example 2 with BReduce

use of edu.iu.dsc.tws.comms.batch.BReduce in project twister2 by DSC-SPIDAL.

the class SVMCommsReducer method execute.

@Override
protected void execute(WorkerEnvironment workerEnv) {
    Set<Integer> sources = new HashSet<>();
    Integer noOfSourceTasks = svmJobParameters.getParallelism();
    for (int i = 0; i < noOfSourceTasks; i++) {
        sources.add(i);
    }
    int target = noOfSourceTasks;
    reduce = new BReduce(workerEnv.getCommunicator(), logicalPlan, sources, target, new ReduceOperationFunction(Op.SUM, MessageTypes.DOUBLE), new FinalSingularReceiver(), MessageTypes.DOUBLE);
    Set<Integer> tasksOfExecutor = Utils.getTasksOfExecutor(workerId, logicalPlan, taskStages, 0);
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    if (!logicalPlan.getLogicalIdsOfWorker(workerId).contains(target)) {
        reduceDone = true;
    }
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, sources, target, tasksOfExecutor));
    for (int t : tasksOfExecutor) {
        Thread mapThread = new Thread(new DataStreamer(t));
        mapThread.start();
    }
}