Examples with ApplyFunc edu.iu.dsc.tws.api.tset.fn.ApplyFunc used on opensource projects

Example 1 with ApplyFunc

use of edu.iu.dsc.tws.api.tset.fn.ApplyFunc in project twister2 by DSC-SPIDAL.

the class KGatherUngroupedExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    SourceTSet<Integer> src = dummySource(env, COUNT, PARALLELISM);
    KeyedGatherUngroupedTLink<Integer, Integer> klink = src.mapToTuple(i -> new Tuple<>(i % 4, i)).keyedGatherUngrouped();
    LOG.info("test foreach");
    klink.forEach((ApplyFunc<Tuple<Integer, Integer>>) data -> LOG.info(data.getKey() + " -> " + data.getValue()));
    LOG.info("test map");
    klink.map((MapFunc<Tuple<Integer, Integer>, String>) input -> input.getKey() + " -> " + input.getValue()).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test compute");
    klink.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
        StringBuilder sb = new StringBuilder();
        while (input.hasNext()) {
            Tuple<Integer, Integer> next = input.next();
            sb.append("[").append(next.getKey()).append("->").append(next.getValue()).append("]");
        }
        return sb.toString();
    }).direct().forEach(s -> LOG.info("compute: " + s));
    LOG.info("test computec");
    klink.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
        while (input.hasNext()) {
            Tuple<Integer, Integer> next = input.next();
            output.collect(next.getKey() + " -> " + next.getValue() * 2);
        }
    }).direct().forEach(s -> LOG.info("computec: " + s));
}

Example 2 with ApplyFunc

use of edu.iu.dsc.tws.api.tset.fn.ApplyFunc in project twister2 by DSC-SPIDAL.

the class SReduceWindowExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    StreamingEnvironment env = TSetEnvironment.initStreaming(workerEnv);
    SSourceTSet<Integer> src = dummySource(env, ELEMENTS_IN_STREAM, PARALLELISM);
    SDirectTLink<Integer> link = src.direct();
    if (COUNT_WINDOWS) {
        if (PROCESS_WINDOW) {
            WindowComputeTSet<Iterator<Integer>> winTSet = link.countWindow(2);
            WindowComputeTSet<Iterator<Integer>> processedTSet = winTSet.process((WindowComputeFunc<Iterator<Integer>, Iterator<Integer>>) input -> {
                List<Integer> list = new ArrayList<>();
                while (input.hasNext()) {
                    list.add(input.next());
                }
                return list.iterator();
            });
            processedTSet.direct().forEach((ApplyFunc<Iterator<Integer>>) data -> {
                while (data.hasNext()) {
                    System.out.println(data.next());
                }
            });
        }
        if (REDUCE_WINDOW) {
            WindowComputeTSet<Integer> winTSet = link.countWindow(2);
            WindowComputeTSet<Integer> localReducedTSet = winTSet.aggregate((AggregateFunc<Integer>) Integer::sum);
            localReducedTSet.direct().forEach(x -> System.out.println(x));
        }
    }
    if (DURATION_WINDOWS) {
        if (PROCESS_WINDOW) {
            System.out.println("DURATION PROCESS WINDOW");
            WindowComputeTSet<Iterator<Integer>> winTSet = link.timeWindow(2, TimeUnit.MILLISECONDS);
            WindowComputeTSet<Iterator<Integer>> processedTSet = winTSet.process((WindowComputeFunc<Iterator<Integer>, Iterator<Integer>>) input -> {
                List<Integer> list = new ArrayList<>();
                while (input.hasNext()) {
                    list.add(input.next());
                }
                return list.iterator();
            });
            processedTSet.direct().forEach((ApplyFunc<Iterator<Integer>>) data -> {
                while (data.hasNext()) {
                    System.out.println(data.next());
                }
            });
        }
        if (REDUCE_WINDOW) {
            WindowComputeTSet<Integer> winTSet = link.timeWindow(2, TimeUnit.MILLISECONDS);
            WindowComputeTSet<Integer> localReducedTSet = winTSet.aggregate((AggregateFunc<Integer>) Integer::sum);
            localReducedTSet.direct().forEach(x -> System.out.println(x));
        // link.countWindow().reduce(a,b-> a + b)
        }
    }
    // Runs the entire TSet graph
    env.run();
}

Example 3 with ApplyFunc

use of edu.iu.dsc.tws.api.tset.fn.ApplyFunc in project twister2 by DSC-SPIDAL.

the class KeyedOperationsExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    KeyedSourceTSet<String, Integer> kSource = dummyKeyedSource(env, COUNT, PARALLELISM);
    KeyedDirectTLink<String, Integer> kDirect = kSource.keyedDirect();
    kDirect.forEach(i -> LOG.info("d_fe: " + i.toString()));
    KeyedCachedTSet<String, Integer> cache = kDirect.cache();
    cache.keyedDirect().forEach(i -> LOG.info("c_d_fe: " + i.toString()));
    cache.keyedReduce(Integer::sum).forEach(i -> LOG.info("c_r_fe: " + i.toString()));
    cache.keyedGather().forEach((ApplyFunc<Tuple<String, Iterator<Integer>>>) data -> {
        StringBuilder sb = new StringBuilder();
        sb.append("c_g_fe: key ").append(data.getKey()).append("->");
        while (data.getValue().hasNext()) {
            sb.append(" ").append(data.getValue().next());
        }
        LOG.info(sb.toString());
    });
}

Example 4 with ApplyFunc

use of edu.iu.dsc.tws.api.tset.fn.ApplyFunc in project twister2 by DSC-SPIDAL.

the class KGatherExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    KeyedGatherTLink<Integer, Integer> klink = src.mapToTuple(i -> new Tuple<>(i % 10, i)).keyedGather();
    LOG.info("test foreach");
    klink.forEach((ApplyFunc<Tuple<Integer, Iterator<Integer>>>) data -> LOG.info(data.getKey() + " -> " + iterToString(data.getValue())));
    LOG.info("test map");
    klink.map((MapFunc<Tuple<Integer, Iterator<Integer>>, String>) input -> {
        int s = 0;
        while (input.getValue().hasNext()) {
            s += input.getValue().next();
        }
        return input.getKey() + " -> " + s;
    }).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test compute");
    klink.compute((ComputeFunc<Iterator<Tuple<Integer, Iterator<Integer>>>, String>) input -> {
        StringBuilder s = new StringBuilder();
        while (input.hasNext()) {
            Tuple<Integer, Iterator<Integer>> next = input.next();
            s.append(" [").append(next.getKey()).append(" -> ").append(iterToString(next.getValue())).append("] ");
        }
        return s.toString();
    }).direct().forEach(s -> LOG.info("compute: concat " + s));
    LOG.info("test computec");
    klink.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Iterator<Integer>>>, String>) (input, output) -> {
        while (input.hasNext()) {
            Tuple<Integer, Iterator<Integer>> next = input.next();
            output.collect(next.getKey() + " -> " + iterToString(next.getValue()));
        }
    }).direct().forEach(s -> LOG.info("computec: " + s));
    // Test byte[] key value pairs for KeyedGather
    SourceTSet<String> srcString = dummyStringSource(env, 25, PARALLELISM);
    KeyedGatherTLink<byte[], Integer> keyedGatherLink = srcString.mapToTuple(s -> new Tuple<>(s.getBytes(), 1)).keyedGather();
    LOG.info("test foreach");
    keyedGatherLink.forEach((ApplyFunc<Tuple<byte[], Iterator<Integer>>>) data -> LOG.info(new String(data.getKey()) + " -> " + iterToString(data.getValue())));
}