Examples with BatchEnvironment edu.iu.dsc.tws.tset.env.BatchEnvironment used on opensource projects

Example 51 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class BroadcastExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    SourceTSet<Integer> src = dummySource(env, COUNT, 1);
    ReplicateTLink<Integer> replicate = src.replicate(PARALLELISM);
    LOG.info("test foreach");
    replicate.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    replicate.map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    replicate.flatmap((i, c) -> c.collect(i.toString() + "##")).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    replicate.compute((ComputeFunc<Iterator<Integer>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        return "sum" + sum;
    }).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    replicate.compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        output.collect("sum" + sum);
    }).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test sink");
    SinkTSet<Iterator<Integer>> sink = replicate.sink((SinkFunc<Iterator<Integer>>) value -> {
        while (value.hasNext()) {
            LOG.info("val =" + value.next());
        }
        return true;
    });
    env.run(sink);
}

Example 52 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class ComputeCollectExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM).setName("src");
    ComputeTSet<String> modify = src.direct().compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, collector) -> {
        while (input.hasNext()) {
            collector.collect(input.next() + "##");
        }
    }).setName("modify");
    modify.direct().forEach(data -> LOG.info("val: " + data));
}

Example 53 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class AllReduceExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    LOG.info("test foreach");
    src.allReduce(Integer::sum).forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    src.allReduce(Integer::sum).map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    src.allReduce(Integer::sum).flatmap((i, c) -> c.collect(i.toString() + "$$")).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    src.allReduce(Integer::sum).compute(i -> i * 2).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    src.allReduce(Integer::sum).compute((ComputeCollectorFunc<Integer, String>) (input, output) -> output.collect("sum=" + input)).direct().forEach(s -> LOG.info("computec: " + s));
}

Example 54 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class CacheExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    // test direct().cache() which has IterLink semantics
    CachedTSet<Integer> cache = src.direct().cache();
    runOps(env, cache);
    // test reduce().cache() which has SingleLink semantics
    LOG.info("test caching after reduce");
    CachedTSet<Integer> cache1 = src.reduce(Integer::sum).cache();
    runOps(env, cache1);
    // test gather.cache() which has TupleValueIterLink
    LOG.info("test caching after gather");
    CachedTSet<Integer> cache2 = src.gather().cache();
    runOps(env, cache2);
}

Example 55 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class ComputeExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM).setName("src").withSchema(PrimitiveSchemas.INTEGER);
    ComputeTSet<Integer> sum = src.direct().compute((ComputeFunc<Iterator<Integer>, Integer>) input -> {
        int s = 0;
        while (input.hasNext()) {
            s += input.next();
        }
        return s;
    }).withSchema(PrimitiveSchemas.INTEGER).setName("sum");
    sum.direct().forEach(data -> LOG.info("val: " + data));
    sum.reduce(Integer::sum).forEach(i -> LOG.info("red: " + i));
}