Examples with ResultsVerifier edu.iu.dsc.tws.examples.verification.ResultsVerifier used on opensource projects

Example 1 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class SAllReduceExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    reduce = new SAllReduce(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    sourcesDone = tasksOfExecutor.size() == 0;
    Set<Integer> targetTasksOfExecutor = logicalPlanBuilder.getTargetsOnThisWorker();
    for (int taskId : targetTasksOfExecutor) {
        if (logicalPlanBuilder.getTargets().contains(taskId)) {
            reduceDone = false;
        }
        if (workerId == 0) {
            receiverInWorker0 = taskId;
        }
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (array, args) -> {
        int sourcesCount = jobParameters.getTaskStages().get(0);
        return GeneratorUtils.multiplyIntArray(array, sourcesCount);
    }, IntArrayComparator.getInstance());
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}

Example 2 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class SDirectExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    direct = new SDirect(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new PartitionReceiver());
    Set<Integer> targetTasksOfExecutor = logicalPlanBuilder.getTargetsOnThisWorker();
    for (int taskId : targetTasksOfExecutor) {
        if (logicalPlanBuilder.getTargets().contains(taskId)) {
            directDone = false;
        }
    }
    if (workerId == 0) {
        receiverInWorker0 = targetTasksOfExecutor.iterator().next();
    }
    Set<Integer> sourceTasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> ints, IntArrayComparator.getInstance());
    // now initialize the workers
    for (int t : sourceTasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
        mapThread.start();
    }
}

Example 3 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class SBroadcastExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    if (jobParameters.getTaskStages().get(0) != 1) {
        LOG.warning("Setting task stages to 1");
        jobParameters.getTaskStages().set(0, 1);
    }
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    bcast = new SBroadCast(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new BCastReceiver());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    Set<Integer> targetTasksOfExecutor = logicalPlanBuilder.getTargetsOnThisWorker();
    for (int taskId : targetTasksOfExecutor) {
        if (logicalPlanBuilder.getTargets().contains(taskId)) {
            bCastDone = false;
            if (workerId == 0) {
                receiverInWorker0 = taskId;
            }
        }
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (array, args) -> array, IntArrayComparator.getInstance());
    // the map thread where data is produced
    if (workerId == 0) {
        Thread mapThread = new Thread(new MapWorker(logicalPlanBuilder.getSources().iterator().next()));
        mapThread.start();
    }
}

Example 4 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class BBroadcastExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    if (jobParameters.getTaskStages().get(0) != 1) {
        LOG.warning("Setting no of senders to 1");
        jobParameters.getTaskStages().set(0, 1);
    }
    int source = 0;
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    bcast = new BBroadcast(workerEnv.getCommunicator(), logicalPlanBuilder, new BCastReceiver(), MessageTypes.INTEGER_ARRAY, true);
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        List<int[]> expectedData = new ArrayList<>();
        for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
            expectedData.add(ints);
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(IntArrayComparator.getInstance()));
    // the map thread where data is produced
    if (workerId == 0) {
        Thread mapThread = new Thread(new MapWorker(source));
        mapThread.start();
    }
}

Example 5 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class BKeyedPartitionExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
    // create the communication
    partition = new BKeyedPartition(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new PartitionReceiver(), new SimpleKeyBasedSelector());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    // now initialize the workers
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
        int target = Integer.parseInt(args.get("target").toString());
        Set<Integer> keysRoutedToThis = new HashSet<>();
        for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
            if (i % logicalPlanBuilder.getTargets().size() == target - lowestTarget) {
                keysRoutedToThis.add(i);
            }
        }
        List<Tuple<Integer, int[]>> expectedData = new ArrayList<>();
        for (Integer key : keysRoutedToThis) {
            for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
                expectedData.add(new Tuple<>(key, ints));
            }
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(// any int
    (d1, d2) -> true, IntArrayComparator.getInstance())));
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new KeyedBenchWorker.MapWorker(t));
        mapThread.start();
    }
}