Examples with SReduce edu.iu.dsc.tws.comms.stream.SReduce used on opensource projects

Example 1 with SReduce

use of edu.iu.dsc.tws.comms.stream.SReduce in project twister2 by DSC-SPIDAL.

the class SReduceExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    if (jobParameters.getTargets() != 1) {
        LOG.warning("Setting targets to 1. Found, " + jobParameters.getTargets());
        jobParameters.getTaskStages().set(1, 1);
    }
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    reduce = new SReduce(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    sourcesDone = tasksOfExecutor.size() == 0;
    reduceDone = !logicalPlan.getLogicalIdsOfWorker(workerId).contains(logicalPlanBuilder.getTargets().iterator().next());
    // generating the expectedIterations results at the end
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (array, args) -> {
        int sourcesCount = jobParameters.getTaskStages().get(0);
        int[] outArray = new int[array.length];
        for (int i = 0; i < array.length; i++) {
            outArray[i] = array[i] * sourcesCount;
        }
        return outArray;
    }, IntArrayComparator.getInstance());
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}