Search in sources :

Example 6 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class BKeyedGatherExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
    // create the communication
    keyedGather = new BKeyedGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new FinalReduceReceiver(), new SimpleKeyBasedSelector());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
        int target = Integer.valueOf(args.get("target").toString());
        Set<Integer> keysRoutedToThis = new HashSet<>();
        for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
            if (i % logicalPlanBuilder.getTargets().size() == target - lowestTarget) {
                keysRoutedToThis.add(i);
            }
        }
        List<int[]> dataForEachKey = new ArrayList<>();
        for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
            dataForEachKey.add(ints);
        }
        List<Tuple<Integer, Iterator<int[]>>> expectedData = new ArrayList<>();
        for (Integer key : keysRoutedToThis) {
            expectedData.add(new Tuple<>(key, dataForEachKey.iterator()));
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(// any int
    (d1, d2) -> true, new IteratorComparator<>(IntArrayComparator.getInstance()))));
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}
Also used : SimpleKeyBasedSelector(edu.iu.dsc.tws.comms.selectors.SimpleKeyBasedSelector) IntArrayComparator(edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) Config(edu.iu.dsc.tws.api.config.Config) SimpleKeyBasedSelector(edu.iu.dsc.tws.comms.selectors.SimpleKeyBasedSelector) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) HashSet(java.util.HashSet) TupleComparator(edu.iu.dsc.tws.examples.verification.comparators.TupleComparator) KeyedBenchWorker(edu.iu.dsc.tws.examples.comms.KeyedBenchWorker) BenchmarkUtils(edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils) ResultsVerifier(edu.iu.dsc.tws.examples.verification.ResultsVerifier) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) BulkReceiver(edu.iu.dsc.tws.api.comms.BulkReceiver) Iterator(java.util.Iterator) Set(java.util.Set) Timing(edu.iu.dsc.tws.examples.utils.bench.Timing) MessageTypes(edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes) Logger(java.util.logging.Logger) BenchmarkConstants(edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants) List(java.util.List) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) IteratorComparator(edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator) Comparator(java.util.Comparator) Collections(java.util.Collections) BKeyedGather(edu.iu.dsc.tws.comms.batch.BKeyedGather) HashSet(java.util.HashSet) Set(java.util.Set) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) BKeyedGather(edu.iu.dsc.tws.comms.batch.BKeyedGather) IteratorComparator(edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator) Iterator(java.util.Iterator) ArrayList(java.util.ArrayList) List(java.util.List) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Example 7 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class BReduceExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    reduce = new BReduce(workerEnv.getCommunicator(), logicalPlanBuilder, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver(), MessageTypes.INTEGER_ARRAY);
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> GeneratorUtils.multiplyIntArray(ints, jobParameters.getTotalIterations() * logicalPlanBuilder.getSources().size()), IntArrayComparator.getInstance());
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), logicalPlanBuilder.getTargets().iterator().next(), tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}
Also used : BReduce(edu.iu.dsc.tws.comms.batch.BReduce) IntArrayComparator(edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator) SingularReceiver(edu.iu.dsc.tws.api.comms.SingularReceiver) GeneratorUtils(edu.iu.dsc.tws.examples.verification.GeneratorUtils) Set(java.util.Set) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) Config(edu.iu.dsc.tws.api.config.Config) Timing(edu.iu.dsc.tws.examples.utils.bench.Timing) MessageTypes(edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes) Logger(java.util.logging.Logger) BenchWorker(edu.iu.dsc.tws.examples.comms.BenchWorker) BenchmarkConstants(edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants) Level(java.util.logging.Level) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) Op(edu.iu.dsc.tws.api.comms.Op) ReduceOperationFunction(edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction) BenchmarkUtils(edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils) ResultsVerifier(edu.iu.dsc.tws.examples.verification.ResultsVerifier) Comparator(java.util.Comparator) ReduceOperationFunction(edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) BReduce(edu.iu.dsc.tws.comms.batch.BReduce)

Example 8 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class SGatherExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    if (jobParameters.getTargets() != 1) {
        LOG.warning("Setting targets to 1. Found, " + jobParameters.getTargets());
        jobParameters.getTaskStages().set(1, 1);
    }
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
    // create the communication
    gather = new SGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new FinalReduceReceiver());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    if (!logicalPlan.getLogicalIdsOfWorker(workerId).contains(logicalPlanBuilder.getTargets().iterator().next())) {
        gatherDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (dataArray, args) -> {
        List<Tuple<Integer, int[]>> listOfArrays = new ArrayList<>();
        for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
            listOfArrays.add(new Tuple<>(i, dataArray));
        }
        return listOfArrays.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(IntComparator.getInstance(), IntArrayComparator.getInstance())));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
        mapThread.start();
    }
}
Also used : IntArrayComparator(edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator) TIMING_MESSAGE_RECV(edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants.TIMING_MESSAGE_RECV) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) BulkReceiver(edu.iu.dsc.tws.api.comms.BulkReceiver) Iterator(java.util.Iterator) Set(java.util.Set) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) Config(edu.iu.dsc.tws.api.config.Config) Timing(edu.iu.dsc.tws.examples.utils.bench.Timing) MessageTypes(edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes) Logger(java.util.logging.Logger) SGather(edu.iu.dsc.tws.comms.stream.SGather) BenchWorker(edu.iu.dsc.tws.examples.comms.BenchWorker) TIMING_ALL_RECV(edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants.TIMING_ALL_RECV) ArrayList(java.util.ArrayList) TupleComparator(edu.iu.dsc.tws.examples.verification.comparators.TupleComparator) List(java.util.List) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) IteratorComparator(edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator) IntComparator(edu.iu.dsc.tws.examples.verification.comparators.IntComparator) BenchmarkUtils(edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils) ResultsVerifier(edu.iu.dsc.tws.examples.verification.ResultsVerifier) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) SGather(edu.iu.dsc.tws.comms.stream.SGather) BenchWorker(edu.iu.dsc.tws.examples.comms.BenchWorker) IteratorComparator(edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator) ArrayList(java.util.ArrayList) List(java.util.List) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Example 9 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class SKeyedPartitionExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    partition = new SKeyedPartition(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new PartitionReceiver(), new HashingSelector());
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> new Tuple<>(-1, ints), new TupleComparator<>(// any int
    (d1, d2) -> true, IntArrayComparator.getInstance()));
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}
Also used : IntArrayComparator(edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) SingularReceiver(edu.iu.dsc.tws.api.comms.SingularReceiver) Set(java.util.Set) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) Config(edu.iu.dsc.tws.api.config.Config) MessageTypes(edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes) Logger(java.util.logging.Logger) Level(java.util.logging.Level) TupleComparator(edu.iu.dsc.tws.examples.verification.comparators.TupleComparator) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) KeyedBenchWorker(edu.iu.dsc.tws.examples.comms.KeyedBenchWorker) SKeyedPartition(edu.iu.dsc.tws.comms.stream.SKeyedPartition) ResultsVerifier(edu.iu.dsc.tws.examples.verification.ResultsVerifier) HashingSelector(edu.iu.dsc.tws.comms.selectors.HashingSelector) SKeyedPartition(edu.iu.dsc.tws.comms.stream.SKeyedPartition) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) TupleComparator(edu.iu.dsc.tws.examples.verification.comparators.TupleComparator) HashingSelector(edu.iu.dsc.tws.comms.selectors.HashingSelector) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Example 10 with ResultsVerifier

use of edu.iu.dsc.tws.examples.verification.ResultsVerifier in project twister2 by DSC-SPIDAL.

the class SPartitionExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    partition = new SPartition(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new PartitionReceiver(), new LoadBalanceSelector());
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> ints, IntArrayComparator.getInstance());
    logicalPlanBuilder.getTargetsOnThisWorker().forEach(t -> {
        if (workerId == 0) {
            recvrInWorker0 = t;
        }
    });
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}
Also used : IntArrayComparator(edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator) TIMING_MESSAGE_RECV(edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants.TIMING_MESSAGE_RECV) LoadBalanceSelector(edu.iu.dsc.tws.comms.selectors.LoadBalanceSelector) SingularReceiver(edu.iu.dsc.tws.api.comms.SingularReceiver) Set(java.util.Set) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) Config(edu.iu.dsc.tws.api.config.Config) Timing(edu.iu.dsc.tws.examples.utils.bench.Timing) MessageTypes(edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes) Logger(java.util.logging.Logger) BenchWorker(edu.iu.dsc.tws.examples.comms.BenchWorker) TIMING_ALL_RECV(edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants.TIMING_ALL_RECV) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) SPartition(edu.iu.dsc.tws.comms.stream.SPartition) BenchmarkUtils(edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils) ResultsVerifier(edu.iu.dsc.tws.examples.verification.ResultsVerifier) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) SPartition(edu.iu.dsc.tws.comms.stream.SPartition) LoadBalanceSelector(edu.iu.dsc.tws.comms.selectors.LoadBalanceSelector)

Aggregations

MessageTypes (edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes)19 Config (edu.iu.dsc.tws.api.config.Config)19 WorkerEnvironment (edu.iu.dsc.tws.api.resource.WorkerEnvironment)19 LogicalPlanBuilder (edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder)19 ResultsVerifier (edu.iu.dsc.tws.examples.verification.ResultsVerifier)19 IntArrayComparator (edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator)19 Set (java.util.Set)19 Logger (java.util.logging.Logger)19 BenchmarkUtils (edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils)18 Timing (edu.iu.dsc.tws.examples.utils.bench.Timing)18 BenchWorker (edu.iu.dsc.tws.examples.comms.BenchWorker)14 BenchmarkConstants (edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants)13 BulkReceiver (edu.iu.dsc.tws.api.comms.BulkReceiver)11 IteratorComparator (edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator)11 ArrayList (java.util.ArrayList)11 Comparator (java.util.Comparator)11 Iterator (java.util.Iterator)11 List (java.util.List)11 Tuple (edu.iu.dsc.tws.api.comms.structs.Tuple)9 TupleComparator (edu.iu.dsc.tws.examples.verification.comparators.TupleComparator)9