Examples with SourceTSet edu.iu.dsc.tws.tset.sets.batch.SourceTSet used on opensource projects

Example 21 with SourceTSet

use of edu.iu.dsc.tws.tset.sets.batch.SourceTSet in project twister2 by DSC-SPIDAL.

the class AllReduceExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    LOG.info("test foreach");
    src.allReduce(Integer::sum).forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    src.allReduce(Integer::sum).map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    src.allReduce(Integer::sum).flatmap((i, c) -> c.collect(i.toString() + "$$")).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    src.allReduce(Integer::sum).compute(i -> i * 2).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    src.allReduce(Integer::sum).compute((ComputeCollectorFunc<Integer, String>) (input, output) -> output.collect("sum=" + input)).direct().forEach(s -> LOG.info("computec: " + s));
}

Example 22 with SourceTSet

use of edu.iu.dsc.tws.tset.sets.batch.SourceTSet in project twister2 by DSC-SPIDAL.

the class ComputeExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM).setName("src").withSchema(PrimitiveSchemas.INTEGER);
    ComputeTSet<Integer> sum = src.direct().compute((ComputeFunc<Iterator<Integer>, Integer>) input -> {
        int s = 0;
        while (input.hasNext()) {
            s += input.next();
        }
        return s;
    }).withSchema(PrimitiveSchemas.INTEGER).setName("sum");
    sum.direct().forEach(data -> LOG.info("val: " + data));
    sum.reduce(Integer::sum).forEach(i -> LOG.info("red: " + i));
}

Example 23 with SourceTSet

use of edu.iu.dsc.tws.tset.sets.batch.SourceTSet in project twister2 by DSC-SPIDAL.

the class FileBasedWordCount method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int sourcePar = (int) env.getConfig().get("PAR");
    // read the file line by line by using a single worker
    SourceTSet<String> lines = env.createSource(new WordCountFileSource(), 1);
    // distribute the lines among the workers and performs a flatmap operation to extract words
    ComputeTSet<String> words = lines.partition(new HashingPartitioner<>(), sourcePar).flatmap((FlatMapFunc<String, String>) (l, collector) -> {
        StringTokenizer itr = new StringTokenizer(l);
        while (itr.hasMoreTokens()) {
            collector.collect(itr.nextToken());
        }
    });
    // attach count as 1 for each word
    KeyedTSet<String, Integer> groupedWords = words.mapToTuple(w -> new Tuple<>(w, 1));
    // performs reduce by key at each worker
    KeyedReduceTLink<String, Integer> keyedReduce = groupedWords.keyedReduce(Integer::sum);
    // gather the results to worker0 (there is a dummy map op here to pass the values to edges)
    // and write to a file
    keyedReduce.map(i -> i).gather().forEach(new WordcountFileWriter());
}