Examples with ResultsVerifier edu.iu.dsc.tws.examples.verification.ResultsVerifier used on opensource projects

Example 11 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class BAllGatherExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    gather = new BAllGather(workerEnv.getCommunicator(), logicalPlanBuilder, new FinalSingularReceiver(), MessageTypes.INTEGER_ARRAY);
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        List<Tuple<Integer, int[]>> expectedOut = new ArrayList<>();
        for (Integer source : logicalPlanBuilder.getSources()) {
            for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
                expectedOut.add(new Tuple<>(source, ints));
            }
        }
        return expectedOut.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(IntComparator.getInstance(), IntArrayComparator.getInstance())));
    LOG.log(Level.INFO, String.format("%d Sources %s target %s this %s", workerId, logicalPlanBuilder.getSources(), logicalPlanBuilder.getTargets(), tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
        mapThread.start();
    }
}

Example 12 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class BDKeyedGatherExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    Set<Integer> sources = new HashSet<>();
    Integer noOfSourceTasks = jobParameters.getTaskStages().get(0);
    for (int i = 0; i < noOfSourceTasks; i++) {
        sources.add(i);
    }
    Set<Integer> targets = new HashSet<>();
    Integer noOfTargetTasks = jobParameters.getTaskStages().get(1);
    for (int i = 0; i < noOfTargetTasks; i++) {
        targets.add(noOfSourceTasks + i);
    }
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    keyedGather = new BKeyedGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new FinalReduceReceiver(), new SimpleKeyBasedSelector(), true, Comparator.comparingInt(o -> (Integer) o), true);
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        int lowestTarget = targets.stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
        int target = Integer.valueOf(args.get("target").toString());
        Set<Integer> keysRoutedToThis = new HashSet<>();
        for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
            if (i % targets.size() == target - lowestTarget) {
                keysRoutedToThis.add(i);
            }
        }
        List<int[]> dataForEachKey = new ArrayList<>();
        for (int i = 0; i < sources.size(); i++) {
            dataForEachKey.add(ints);
        }
        List<Tuple<Integer, Iterator<int[]>>> expectedData = new ArrayList<>();
        for (Integer key : keysRoutedToThis) {
            expectedData.add(new Tuple<>(key, dataForEachKey.iterator()));
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(// any int
    (d1, d2) -> true, new IteratorComparator<>(IntArrayComparator.getInstance()))));
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, sources, 1, tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        MapWorker mapWorker = new MapWorker(t);
        mapWorker.setTimingForLowestTargetOnly(true);
        Thread mapThread = new Thread(mapWorker);
        mapThread.start();
    }
}

Example 13 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class BGatherExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    if (jobParameters.getTargets() != 1) {
        LOG.warning("Setting no of targets to 1. Found " + jobParameters.getTargets());
        jobParameters.getTaskStages().set(1, 1);
    }
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
    // create the communication
    gather = new BGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new FinalReduceReceiver(), false);
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        List<Tuple<Integer, int[]>> expectedData = new ArrayList<>();
        for (Integer source : logicalPlanBuilder.getSources()) {
            for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
                expectedData.add(new Tuple<>(source, ints));
            }
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(IntComparator.getInstance(), IntArrayComparator.getInstance())));
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), logicalPlanBuilder.getTargets().iterator().next(), tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}

Example 14 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class BPartitionExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    partition = new BPartition(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new PartitionReceiver(), new LoadBalanceSelector(), false);
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
        int target = Integer.parseInt(args.get("target").toString());
        int toThisFromOneSource = jobParameters.getTotalIterations() / logicalPlanBuilder.getTargets().size();
        if (jobParameters.getTotalIterations() % logicalPlanBuilder.getTargets().size() > (target - lowestTarget)) {
            toThisFromOneSource++;
        }
        List<int[]> expectedData = new ArrayList<>();
        for (int i = 0; i < toThisFromOneSource * logicalPlanBuilder.getSources().size(); i++) {
            expectedData.add(ints);
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(IntArrayComparator.getInstance()));
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
        mapThread.start();
    }
}

Example 15 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class BAllReduceExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    Integer noOfSourceTasks = jobParameters.getTaskStages().get(0);
    Set<Integer> sources = IntStream.range(0, noOfSourceTasks).boxed().collect(Collectors.toSet());
    int noOfTargetTasks = jobParameters.getTaskStages().get(1);
    Set<Integer> targets = IntStream.range(noOfSourceTasks, noOfTargetTasks + noOfSourceTasks).boxed().collect(Collectors.toSet());
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    reduce = new BAllReduce(workerEnv.getCommunicator(), logicalPlanBuilder, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver(), MessageTypes.INTEGER_ARRAY);
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> GeneratorUtils.multiplyIntArray(ints, jobParameters.getIterations() * sources.size()), IntArrayComparator.getInstance());
    LOG.log(Level.INFO, String.format("%d Sources %s target %s this %s", workerId, sources, targets, tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
        mapThread.start();
    }
}