Search in sources :

Example 41 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class BJoinStudentExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the join communication
    join = new BJoin(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.OBJECT, MessageTypes.OBJECT, new JoinReceiver(), new SimpleKeyBasedSelector(), false, (o1, o2) -> {
        if (o1 instanceof String && o2 instanceof String) {
            return ((String) o1).compareTo((String) o2);
        }
        return 0;
    }, CommunicationContext.JoinType.INNER, CommunicationContext.JoinAlgorithm.SORT);
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    // now initialize the workers
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}
Also used : SimpleKeyBasedSelector(edu.iu.dsc.tws.comms.selectors.SimpleKeyBasedSelector) DataGenerator(edu.iu.dsc.tws.examples.comms.DataGenerator) CommunicationContext(edu.iu.dsc.tws.api.comms.CommunicationContext) BulkReceiver(edu.iu.dsc.tws.api.comms.BulkReceiver) Iterator(java.util.Iterator) ReentrantLock(java.util.concurrent.locks.ReentrantLock) Set(java.util.Set) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) JoinedTuple(edu.iu.dsc.tws.api.comms.structs.JoinedTuple) Config(edu.iu.dsc.tws.api.config.Config) MessageTypes(edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes) SimpleKeyBasedSelector(edu.iu.dsc.tws.comms.selectors.SimpleKeyBasedSelector) Logger(java.util.logging.Logger) Level(java.util.logging.Level) Lock(java.util.concurrent.locks.Lock) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) KeyedBenchWorker(edu.iu.dsc.tws.examples.comms.KeyedBenchWorker) BJoin(edu.iu.dsc.tws.comms.batch.BJoin) Map(java.util.Map) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) BJoin(edu.iu.dsc.tws.comms.batch.BJoin)

Example 42 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class BPartitionExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    partition = new BPartition(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new PartitionReceiver(), new LoadBalanceSelector(), false);
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
        int target = Integer.parseInt(args.get("target").toString());
        int toThisFromOneSource = jobParameters.getTotalIterations() / logicalPlanBuilder.getTargets().size();
        if (jobParameters.getTotalIterations() % logicalPlanBuilder.getTargets().size() > (target - lowestTarget)) {
            toThisFromOneSource++;
        }
        List<int[]> expectedData = new ArrayList<>();
        for (int i = 0; i < toThisFromOneSource * logicalPlanBuilder.getSources().size(); i++) {
            expectedData.add(ints);
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(IntArrayComparator.getInstance()));
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
        mapThread.start();
    }
}
Also used : IntArrayComparator(edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator) LoadBalanceSelector(edu.iu.dsc.tws.comms.selectors.LoadBalanceSelector) BulkReceiver(edu.iu.dsc.tws.api.comms.BulkReceiver) Iterator(java.util.Iterator) BPartition(edu.iu.dsc.tws.comms.batch.BPartition) Set(java.util.Set) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) Config(edu.iu.dsc.tws.api.config.Config) Timing(edu.iu.dsc.tws.examples.utils.bench.Timing) MessageTypes(edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes) Logger(java.util.logging.Logger) BenchWorker(edu.iu.dsc.tws.examples.comms.BenchWorker) BenchmarkConstants(edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants) ArrayList(java.util.ArrayList) List(java.util.List) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) IteratorComparator(edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator) BenchmarkUtils(edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils) ResultsVerifier(edu.iu.dsc.tws.examples.verification.ResultsVerifier) Comparator(java.util.Comparator) Collections(java.util.Collections) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) BPartition(edu.iu.dsc.tws.comms.batch.BPartition) BenchWorker(edu.iu.dsc.tws.examples.comms.BenchWorker) IteratorComparator(edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator) ArrayList(java.util.ArrayList) List(java.util.List) LoadBalanceSelector(edu.iu.dsc.tws.comms.selectors.LoadBalanceSelector)

Example 43 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class BAllReduceExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    Integer noOfSourceTasks = jobParameters.getTaskStages().get(0);
    Set<Integer> sources = IntStream.range(0, noOfSourceTasks).boxed().collect(Collectors.toSet());
    int noOfTargetTasks = jobParameters.getTaskStages().get(1);
    Set<Integer> targets = IntStream.range(noOfSourceTasks, noOfTargetTasks + noOfSourceTasks).boxed().collect(Collectors.toSet());
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    reduce = new BAllReduce(workerEnv.getCommunicator(), logicalPlanBuilder, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver(), MessageTypes.INTEGER_ARRAY);
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> GeneratorUtils.multiplyIntArray(ints, jobParameters.getIterations() * sources.size()), IntArrayComparator.getInstance());
    LOG.log(Level.INFO, String.format("%d Sources %s target %s this %s", workerId, sources, targets, tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
        mapThread.start();
    }
}
Also used : IntStream(java.util.stream.IntStream) IntArrayComparator(edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator) SingularReceiver(edu.iu.dsc.tws.api.comms.SingularReceiver) BAllReduce(edu.iu.dsc.tws.comms.batch.BAllReduce) GeneratorUtils(edu.iu.dsc.tws.examples.verification.GeneratorUtils) Set(java.util.Set) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) Config(edu.iu.dsc.tws.api.config.Config) Timing(edu.iu.dsc.tws.examples.utils.bench.Timing) MessageTypes(edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes) Logger(java.util.logging.Logger) BenchWorker(edu.iu.dsc.tws.examples.comms.BenchWorker) Collectors(java.util.stream.Collectors) BenchmarkConstants(edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants) Level(java.util.logging.Level) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) Op(edu.iu.dsc.tws.api.comms.Op) ReduceOperationFunction(edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction) BenchmarkUtils(edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils) ResultsVerifier(edu.iu.dsc.tws.examples.verification.ResultsVerifier) Comparator(java.util.Comparator) ReduceOperationFunction(edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction) BenchWorker(edu.iu.dsc.tws.examples.comms.BenchWorker) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) BAllReduce(edu.iu.dsc.tws.comms.batch.BAllReduce)

Example 44 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class BDirectExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    if (!jobParameters.getTaskStages().get(0).equals(jobParameters.getTaskStages().get(1))) {
        int min = Math.min(jobParameters.getTaskStages().get(0), jobParameters.getTaskStages().get(1));
        LOG.warning("Setting sources and sinks to " + min);
        jobParameters.getTaskStages().set(0, min);
        jobParameters.getTaskStages().set(1, min);
    }
    Integer noOfSourceTasks = jobParameters.getTaskStages().get(0);
    List<Integer> sources = IntStream.range(0, noOfSourceTasks).boxed().collect(Collectors.toList());
    Integer noOfTargetTasks = jobParameters.getTaskStages().get(1);
    List<Integer> targets = IntStream.range(0, noOfTargetTasks).boxed().collect(Collectors.toList());
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    direct = new BDirect(workerEnv.getCommunicator(), logicalPlanBuilder, new DirectReceiver(), MessageTypes.INTEGER_ARRAY, false);
    resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        List<int[]> expectedData = new ArrayList<>();
        for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
            expectedData.add(ints);
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(IntArrayComparator.getInstance()));
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
        mapThread.start();
    }
}
Also used : IntStream(java.util.stream.IntStream) IntArrayComparator(edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator) BulkReceiver(edu.iu.dsc.tws.api.comms.BulkReceiver) Iterator(java.util.Iterator) BDirect(edu.iu.dsc.tws.comms.batch.BDirect) Set(java.util.Set) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) Config(edu.iu.dsc.tws.api.config.Config) Timing(edu.iu.dsc.tws.examples.utils.bench.Timing) MessageTypes(edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes) Logger(java.util.logging.Logger) BenchWorker(edu.iu.dsc.tws.examples.comms.BenchWorker) Collectors(java.util.stream.Collectors) BenchmarkConstants(edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants) ArrayList(java.util.ArrayList) List(java.util.List) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) IteratorComparator(edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator) BenchmarkUtils(edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils) ResultsVerifier(edu.iu.dsc.tws.examples.verification.ResultsVerifier) Comparator(java.util.Comparator) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) BDirect(edu.iu.dsc.tws.comms.batch.BDirect) BenchWorker(edu.iu.dsc.tws.examples.comms.BenchWorker) IteratorComparator(edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator) ArrayList(java.util.ArrayList) List(java.util.List)

Example 45 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class BKeyedReduceExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    keyedReduce = new BKeyedReduce(workerEnv.getCommunicator(), logicalPlanBuilder, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalBulkReceiver(), MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new SimpleKeyBasedSelector());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
        int target = Integer.parseInt(args.get("target").toString());
        Set<Integer> keysRoutedToThis = new HashSet<>();
        for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
            if (i % logicalPlanBuilder.getTargets().size() == target - lowestTarget) {
                keysRoutedToThis.add(i);
            }
        }
        int[] reduced = GeneratorUtils.multiplyIntArray(ints, logicalPlanBuilder.getSources().size());
        List<Tuple<Integer, int[]>> expectedData = new ArrayList<>();
        for (Integer key : keysRoutedToThis) {
            expectedData.add(new Tuple<>(key, reduced));
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(IntComparator.getInstance(), IntArrayComparator.getInstance())));
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new KeyedBenchWorker.MapWorker(t));
        mapThread.start();
    }
}
Also used : SimpleKeyBasedSelector(edu.iu.dsc.tws.comms.selectors.SimpleKeyBasedSelector) IntArrayComparator(edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator) GeneratorUtils(edu.iu.dsc.tws.examples.verification.GeneratorUtils) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) Config(edu.iu.dsc.tws.api.config.Config) SimpleKeyBasedSelector(edu.iu.dsc.tws.comms.selectors.SimpleKeyBasedSelector) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) HashSet(java.util.HashSet) TupleComparator(edu.iu.dsc.tws.examples.verification.comparators.TupleComparator) KeyedBenchWorker(edu.iu.dsc.tws.examples.comms.KeyedBenchWorker) Op(edu.iu.dsc.tws.api.comms.Op) ReduceOperationFunction(edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction) BenchmarkUtils(edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils) ResultsVerifier(edu.iu.dsc.tws.examples.verification.ResultsVerifier) BKeyedReduce(edu.iu.dsc.tws.comms.batch.BKeyedReduce) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) BulkReceiver(edu.iu.dsc.tws.api.comms.BulkReceiver) Iterator(java.util.Iterator) Set(java.util.Set) Timing(edu.iu.dsc.tws.examples.utils.bench.Timing) MessageTypes(edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes) Logger(java.util.logging.Logger) BenchmarkConstants(edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants) List(java.util.List) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) IteratorComparator(edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator) IntComparator(edu.iu.dsc.tws.examples.verification.comparators.IntComparator) Comparator(java.util.Comparator) Collections(java.util.Collections) ReduceOperationFunction(edu.iu.dsc.tws.comms.functions.reduction.ReduceOperationFunction) KeyedBenchWorker(edu.iu.dsc.tws.examples.comms.KeyedBenchWorker) HashSet(java.util.HashSet) Set(java.util.Set) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) BKeyedReduce(edu.iu.dsc.tws.comms.batch.BKeyedReduce) IteratorComparator(edu.iu.dsc.tws.examples.verification.comparators.IteratorComparator) ArrayList(java.util.ArrayList) List(java.util.List) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Aggregations

WorkerEnvironment (edu.iu.dsc.tws.api.resource.WorkerEnvironment)49 Logger (java.util.logging.Logger)46 Config (edu.iu.dsc.tws.api.config.Config)42 Iterator (java.util.Iterator)27 TSetEnvironment (edu.iu.dsc.tws.tset.env.TSetEnvironment)26 JobConfig (edu.iu.dsc.tws.api.JobConfig)25 Tuple (edu.iu.dsc.tws.api.comms.structs.Tuple)24 ResourceAllocator (edu.iu.dsc.tws.rsched.core.ResourceAllocator)23 BatchEnvironment (edu.iu.dsc.tws.tset.env.BatchEnvironment)23 SourceTSet (edu.iu.dsc.tws.tset.sets.batch.SourceTSet)23 HashMap (java.util.HashMap)22 LogicalPlanBuilder (edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder)21 MessageTypes (edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes)20 Set (java.util.Set)20 ResultsVerifier (edu.iu.dsc.tws.examples.verification.ResultsVerifier)19 IntArrayComparator (edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator)19 BenchmarkUtils (edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils)18 Timing (edu.iu.dsc.tws.examples.utils.bench.Timing)18 BenchWorker (edu.iu.dsc.tws.examples.comms.BenchWorker)14 BenchmarkConstants (edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants)13