Examples with StreamingEnvironment edu.iu.dsc.tws.tset.env.StreamingEnvironment used on opensource projects

Example 1 with StreamingEnvironment

use of edu.iu.dsc.tws.tset.env.StreamingEnvironment in project twister2 by DSC-SPIDAL.

the class SReduceWindowExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    StreamingEnvironment env = TSetEnvironment.initStreaming(workerEnv);
    SSourceTSet<Integer> src = dummySource(env, ELEMENTS_IN_STREAM, PARALLELISM);
    SDirectTLink<Integer> link = src.direct();
    if (COUNT_WINDOWS) {
        if (PROCESS_WINDOW) {
            WindowComputeTSet<Iterator<Integer>> winTSet = link.countWindow(2);
            WindowComputeTSet<Iterator<Integer>> processedTSet = winTSet.process((WindowComputeFunc<Iterator<Integer>, Iterator<Integer>>) input -> {
                List<Integer> list = new ArrayList<>();
                while (input.hasNext()) {
                    list.add(input.next());
                }
                return list.iterator();
            });
            processedTSet.direct().forEach((ApplyFunc<Iterator<Integer>>) data -> {
                while (data.hasNext()) {
                    System.out.println(data.next());
                }
            });
        }
        if (REDUCE_WINDOW) {
            WindowComputeTSet<Integer> winTSet = link.countWindow(2);
            WindowComputeTSet<Integer> localReducedTSet = winTSet.aggregate((AggregateFunc<Integer>) Integer::sum);
            localReducedTSet.direct().forEach(x -> System.out.println(x));
        }
    }
    if (DURATION_WINDOWS) {
        if (PROCESS_WINDOW) {
            System.out.println("DURATION PROCESS WINDOW");
            WindowComputeTSet<Iterator<Integer>> winTSet = link.timeWindow(2, TimeUnit.MILLISECONDS);
            WindowComputeTSet<Iterator<Integer>> processedTSet = winTSet.process((WindowComputeFunc<Iterator<Integer>, Iterator<Integer>>) input -> {
                List<Integer> list = new ArrayList<>();
                while (input.hasNext()) {
                    list.add(input.next());
                }
                return list.iterator();
            });
            processedTSet.direct().forEach((ApplyFunc<Iterator<Integer>>) data -> {
                while (data.hasNext()) {
                    System.out.println(data.next());
                }
            });
        }
        if (REDUCE_WINDOW) {
            WindowComputeTSet<Integer> winTSet = link.timeWindow(2, TimeUnit.MILLISECONDS);
            WindowComputeTSet<Integer> localReducedTSet = winTSet.aggregate((AggregateFunc<Integer>) Integer::sum);
            localReducedTSet.direct().forEach(x -> System.out.println(x));
        // link.countWindow().reduce(a,b-> a + b)
        }
    }
    // Runs the entire TSet graph
    env.run();
}

Example 2 with StreamingEnvironment

use of edu.iu.dsc.tws.tset.env.StreamingEnvironment in project twister2 by DSC-SPIDAL.

the class SDirectExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    StreamingEnvironment env = TSetEnvironment.initStreaming(workerEnv);
    SSourceTSet<Integer> src = dummySource(env, COUNT, PARALLELISM).setName("src");
    SDirectTLink<Integer> link = src.direct().setName("dir");
    link.map(i -> i * 2).setName("map").direct().forEach(i -> LOG.info("m" + i.toString()));
    link.flatmap((i, c) -> c.collect("fm" + i)).setName("flatmap").direct().forEach(i -> LOG.info(i.toString()));
    link.compute(i -> i + "C").setName("compute").direct().forEach(i -> LOG.info(i));
    link.mapToTuple(i -> new Tuple<>(i, i.toString())).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
    link.compute((input, output) -> output.collect(input + "DD")).setName("computec").direct().forEach(s -> LOG.info(s.toString()));
    // Runs the entire TSet graph
    env.run();
}

Example 3 with StreamingEnvironment

use of edu.iu.dsc.tws.tset.env.StreamingEnvironment in project twister2 by DSC-SPIDAL.

the class SUnionExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    StreamingEnvironment env = TSetEnvironment.initStreaming(workerEnv);
    // SourceTSet<Integer> src = dummySource(env, COUNT, PARALLELISM).setName("src");
    SSourceTSet<Integer> src1 = dummySource(env, COUNT, PARALLELISM).setName("src1");
    SSourceTSet<Integer> src2 = dummySourceOther(env, COUNT, PARALLELISM).setName("src2");
    // src.direct().forEach(s -> LOG.info("map sssss: " + s));
    SComputeTSet<Integer> unionTSet = src1.union(src2);
    LOG.info("test source union");
    unionTSet.direct().forEach(s -> LOG.info("map: " + s));
    // Runs the entire TSet graph
    env.run();
}

Example 4 with StreamingEnvironment

use of edu.iu.dsc.tws.tset.env.StreamingEnvironment in project twister2 by DSC-SPIDAL.

the class WordCount method execute.

@Override
public void execute(WorkerEnvironment workerEnvironment) {
    StreamingEnvironment cEnv = TSetEnvironment.initStreaming(workerEnvironment);
    // create source and aggregator
    cEnv.createSource(new SourceFunc<String>() {

        // sample words
        private List<String> sampleWords = new ArrayList<>();

        // the random used to pick he words
        private Random random;

        @Override
        public void prepare(TSetContext context) {
            this.random = new Random();
            RandomString randomString = new RandomString(MAX_CHARS, random, RandomString.ALPHANUM);
            for (int i = 0; i < NO_OF_SAMPLE_WORDS; i++) {
                sampleWords.add(randomString.nextRandomSizeString());
            }
        }

        @Override
        public boolean hasNext() {
            return true;
        }

        @Override
        public String next() {
            return sampleWords.get(random.nextInt(sampleWords.size()));
        }
    }, 4).partition(new HashingPartitioner<>()).sink(new SinkFunc<String>() {

        // keep track of the counts
        private Map<String, Integer> counts = new HashMap<>();

        private TSetContext context;

        @Override
        public void prepare(TSetContext context) {
            this.context = context;
        }

        @Override
        public boolean add(String word) {
            int count = 1;
            if (counts.containsKey(word)) {
                count = counts.get(word);
                count++;
            }
            counts.put(word, count);
            LOG.log(Level.INFO, String.format("%d Word %s count %s", context.getIndex(), word, count));
            return true;
        }
    });
    // start executing the streaming graph
    cEnv.run();
}

Example 5 with StreamingEnvironment

use of edu.iu.dsc.tws.tset.env.StreamingEnvironment in project twister2 by DSC-SPIDAL.

the class SReduceExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    StreamingEnvironment env = TSetEnvironment.initStreaming(workerEnv);
    SSourceTSet<Integer> src = dummySource(env, 8, PARALLELISM);
    SDirectTLink<Integer> link = src.direct();
    link.map(i -> i * 2).direct().forEach(i -> LOG.info("m" + i.toString()));
    link.flatmap((i, c) -> c.collect("fm" + i)).direct().forEach(i -> LOG.info(i.toString()));
    link.compute(i -> i + "C").direct().forEach(i -> LOG.info(i));
    link.mapToTuple(i -> new Tuple<>(i, i.toString())).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
    link.compute((input, output) -> output.collect(input + "DD")).direct().forEach(s -> LOG.info(s.toString()));
    // Runs the entire TSet graph
    env.run();
}