Search in sources :

Example 1 with SDirectTLink

use of edu.iu.dsc.tws.tset.links.streaming.SDirectTLink in project twister2 by DSC-SPIDAL.

the class SReduceWindowExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    StreamingEnvironment env = TSetEnvironment.initStreaming(workerEnv);
    SSourceTSet<Integer> src = dummySource(env, ELEMENTS_IN_STREAM, PARALLELISM);
    SDirectTLink<Integer> link = src.direct();
    if (COUNT_WINDOWS) {
        if (PROCESS_WINDOW) {
            WindowComputeTSet<Iterator<Integer>> winTSet = link.countWindow(2);
            WindowComputeTSet<Iterator<Integer>> processedTSet = winTSet.process((WindowComputeFunc<Iterator<Integer>, Iterator<Integer>>) input -> {
                List<Integer> list = new ArrayList<>();
                while (input.hasNext()) {
                    list.add(input.next());
                }
                return list.iterator();
            });
            processedTSet.direct().forEach((ApplyFunc<Iterator<Integer>>) data -> {
                while (data.hasNext()) {
                    System.out.println(data.next());
                }
            });
        }
        if (REDUCE_WINDOW) {
            WindowComputeTSet<Integer> winTSet = link.countWindow(2);
            WindowComputeTSet<Integer> localReducedTSet = winTSet.aggregate((AggregateFunc<Integer>) Integer::sum);
            localReducedTSet.direct().forEach(x -> System.out.println(x));
        }
    }
    if (DURATION_WINDOWS) {
        if (PROCESS_WINDOW) {
            System.out.println("DURATION PROCESS WINDOW");
            WindowComputeTSet<Iterator<Integer>> winTSet = link.timeWindow(2, TimeUnit.MILLISECONDS);
            WindowComputeTSet<Iterator<Integer>> processedTSet = winTSet.process((WindowComputeFunc<Iterator<Integer>, Iterator<Integer>>) input -> {
                List<Integer> list = new ArrayList<>();
                while (input.hasNext()) {
                    list.add(input.next());
                }
                return list.iterator();
            });
            processedTSet.direct().forEach((ApplyFunc<Iterator<Integer>>) data -> {
                while (data.hasNext()) {
                    System.out.println(data.next());
                }
            });
        }
        if (REDUCE_WINDOW) {
            WindowComputeTSet<Integer> winTSet = link.timeWindow(2, TimeUnit.MILLISECONDS);
            WindowComputeTSet<Integer> localReducedTSet = winTSet.aggregate((AggregateFunc<Integer>) Integer::sum);
            localReducedTSet.direct().forEach(x -> System.out.println(x));
        // link.countWindow().reduce(a,b-> a + b)
        }
    }
    // Runs the entire TSet graph
    env.run();
}
Also used : SSourceTSet(edu.iu.dsc.tws.tset.sets.streaming.SSourceTSet) Iterator(java.util.Iterator) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) AggregateFunc(edu.iu.dsc.tws.tset.fn.AggregateFunc) Logger(java.util.logging.Logger) JobConfig(edu.iu.dsc.tws.api.JobConfig) ArrayList(java.util.ArrayList) StreamingEnvironment(edu.iu.dsc.tws.tset.env.StreamingEnvironment) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) WindowComputeFunc(edu.iu.dsc.tws.tset.fn.WindowComputeFunc) SDirectTLink(edu.iu.dsc.tws.tset.links.streaming.SDirectTLink) WindowComputeTSet(edu.iu.dsc.tws.tset.sets.streaming.WindowComputeTSet) ApplyFunc(edu.iu.dsc.tws.api.tset.fn.ApplyFunc) BatchTsetExample(edu.iu.dsc.tws.examples.tset.batch.BatchTsetExample) StreamingEnvironment(edu.iu.dsc.tws.tset.env.StreamingEnvironment) Iterator(java.util.Iterator) ArrayList(java.util.ArrayList) List(java.util.List)

Example 2 with SDirectTLink

use of edu.iu.dsc.tws.tset.links.streaming.SDirectTLink in project twister2 by DSC-SPIDAL.

the class SDirectExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    StreamingEnvironment env = TSetEnvironment.initStreaming(workerEnv);
    SSourceTSet<Integer> src = dummySource(env, COUNT, PARALLELISM).setName("src");
    SDirectTLink<Integer> link = src.direct().setName("dir");
    link.map(i -> i * 2).setName("map").direct().forEach(i -> LOG.info("m" + i.toString()));
    link.flatmap((i, c) -> c.collect("fm" + i)).setName("flatmap").direct().forEach(i -> LOG.info(i.toString()));
    link.compute(i -> i + "C").setName("compute").direct().forEach(i -> LOG.info(i));
    link.mapToTuple(i -> new Tuple<>(i, i.toString())).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
    link.compute((input, output) -> output.collect(input + "DD")).setName("computec").direct().forEach(s -> LOG.info(s.toString()));
    // Runs the entire TSet graph
    env.run();
}
Also used : SSourceTSet(edu.iu.dsc.tws.tset.sets.streaming.SSourceTSet) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) SDirectTLink(edu.iu.dsc.tws.tset.links.streaming.SDirectTLink) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) Logger(java.util.logging.Logger) JobConfig(edu.iu.dsc.tws.api.JobConfig) BatchTsetExample(edu.iu.dsc.tws.examples.tset.batch.BatchTsetExample) StreamingEnvironment(edu.iu.dsc.tws.tset.env.StreamingEnvironment) StreamingEnvironment(edu.iu.dsc.tws.tset.env.StreamingEnvironment)

Aggregations

JobConfig (edu.iu.dsc.tws.api.JobConfig)2 Config (edu.iu.dsc.tws.api.config.Config)2 WorkerEnvironment (edu.iu.dsc.tws.api.resource.WorkerEnvironment)2 BatchTsetExample (edu.iu.dsc.tws.examples.tset.batch.BatchTsetExample)2 ResourceAllocator (edu.iu.dsc.tws.rsched.core.ResourceAllocator)2 StreamingEnvironment (edu.iu.dsc.tws.tset.env.StreamingEnvironment)2 TSetEnvironment (edu.iu.dsc.tws.tset.env.TSetEnvironment)2 SDirectTLink (edu.iu.dsc.tws.tset.links.streaming.SDirectTLink)2 SSourceTSet (edu.iu.dsc.tws.tset.sets.streaming.SSourceTSet)2 HashMap (java.util.HashMap)2 Logger (java.util.logging.Logger)2 Tuple (edu.iu.dsc.tws.api.comms.structs.Tuple)1 ApplyFunc (edu.iu.dsc.tws.api.tset.fn.ApplyFunc)1 AggregateFunc (edu.iu.dsc.tws.tset.fn.AggregateFunc)1 WindowComputeFunc (edu.iu.dsc.tws.tset.fn.WindowComputeFunc)1 WindowComputeTSet (edu.iu.dsc.tws.tset.sets.streaming.WindowComputeTSet)1 ArrayList (java.util.ArrayList)1 Iterator (java.util.Iterator)1 List (java.util.List)1 TimeUnit (java.util.concurrent.TimeUnit)1