Examples with ComputeCollectorFunc edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc used on opensource projects

Example 6 with ComputeCollectorFunc

use of edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc in project twister2 by DSC-SPIDAL.

the class DirectExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM).setName("src");
    DirectTLink<Integer> direct = src.direct().setName("direct");
    LOG.info("test foreach");
    direct.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    direct.map(i -> i.toString() + "$$").setName("map").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    direct.flatmap((i, c) -> c.collect(i.toString() + "##")).setName("flatmap").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    direct.compute((ComputeFunc<Iterator<Integer>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        return "sum" + sum;
    }).setName("compute").withSchema(PrimitiveSchemas.STRING).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    direct.compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        output.collect("sum" + sum);
    }).setName("ccompute").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test map2tup");
    direct.mapToTuple(i -> new Tuple<>(i, i.toString())).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
    LOG.info("test sink");
    SinkTSet<Iterator<Integer>> sink = direct.sink((SinkFunc<Iterator<Integer>>) value -> {
        while (value.hasNext()) {
            LOG.info("val =" + value.next());
        }
        return true;
    });
    env.run(sink);
}

Example 7 with ComputeCollectorFunc

use of edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc in project twister2 by DSC-SPIDAL.

the class GatherExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    GatherTLink<Integer> gather = src.gather();
    LOG.info("test foreach");
    gather.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    gather.map(i -> i.toString() + "$$").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test compute");
    gather.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        return "sum=" + sum;
    }).withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("compute: " + s));
    LOG.info("test computec");
    gather.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        output.collect("sum=" + sum);
    }).withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("computec: " + s));
    gather.mapToTuple(i -> new Tuple<>(i % 2, i)).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
}

Example 8 with ComputeCollectorFunc

use of edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc in project twister2 by DSC-SPIDAL.

the class KGatherExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    KeyedGatherTLink<Integer, Integer> klink = src.mapToTuple(i -> new Tuple<>(i % 10, i)).keyedGather();
    LOG.info("test foreach");
    klink.forEach((ApplyFunc<Tuple<Integer, Iterator<Integer>>>) data -> LOG.info(data.getKey() + " -> " + iterToString(data.getValue())));
    LOG.info("test map");
    klink.map((MapFunc<Tuple<Integer, Iterator<Integer>>, String>) input -> {
        int s = 0;
        while (input.getValue().hasNext()) {
            s += input.getValue().next();
        }
        return input.getKey() + " -> " + s;
    }).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test compute");
    klink.compute((ComputeFunc<Iterator<Tuple<Integer, Iterator<Integer>>>, String>) input -> {
        StringBuilder s = new StringBuilder();
        while (input.hasNext()) {
            Tuple<Integer, Iterator<Integer>> next = input.next();
            s.append(" [").append(next.getKey()).append(" -> ").append(iterToString(next.getValue())).append("] ");
        }
        return s.toString();
    }).direct().forEach(s -> LOG.info("compute: concat " + s));
    LOG.info("test computec");
    klink.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Iterator<Integer>>>, String>) (input, output) -> {
        while (input.hasNext()) {
            Tuple<Integer, Iterator<Integer>> next = input.next();
            output.collect(next.getKey() + " -> " + iterToString(next.getValue()));
        }
    }).direct().forEach(s -> LOG.info("computec: " + s));
    // Test byte[] key value pairs for KeyedGather
    SourceTSet<String> srcString = dummyStringSource(env, 25, PARALLELISM);
    KeyedGatherTLink<byte[], Integer> keyedGatherLink = srcString.mapToTuple(s -> new Tuple<>(s.getBytes(), 1)).keyedGather();
    LOG.info("test foreach");
    keyedGatherLink.forEach((ApplyFunc<Tuple<byte[], Iterator<Integer>>>) data -> LOG.info(new String(data.getKey()) + " -> " + iterToString(data.getValue())));
}

Example 9 with ComputeCollectorFunc

use of edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc in project twister2 by DSC-SPIDAL.

the class KPartitionExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    KeyedPartitionTLink<Integer, Integer> klink = src.mapToTuple(i -> new Tuple<>(i % 10, i)).keyedPartition(new LoadBalancePartitioner<>());
    LOG.info("test foreach");
    klink.forEach(t -> LOG.info(t.getKey() + "_" + t.getValue()));
    LOG.info("test map");
    klink.map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test compute");
    klink.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
        StringBuilder s = new StringBuilder();
        while (input.hasNext()) {
            s.append(input.next().toString()).append(" ");
        }
        return s.toString();
    }).direct().forEach(s -> LOG.info("compute: concat " + s));
    LOG.info("test computec");
    klink.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
        while (input.hasNext()) {
            output.collect(input.next().toString());
        }
    }).direct().forEach(s -> LOG.info("computec: " + s));
}

Example 10 with ComputeCollectorFunc

use of edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc in project twister2 by DSC-SPIDAL.

the class BroadcastExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    SourceTSet<Integer> src = dummySource(env, COUNT, 1);
    ReplicateTLink<Integer> replicate = src.replicate(PARALLELISM);
    LOG.info("test foreach");
    replicate.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    replicate.map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    replicate.flatmap((i, c) -> c.collect(i.toString() + "##")).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    replicate.compute((ComputeFunc<Iterator<Integer>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        return "sum" + sum;
    }).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    replicate.compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        output.collect("sum" + sum);
    }).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test sink");
    SinkTSet<Iterator<Integer>> sink = replicate.sink((SinkFunc<Iterator<Integer>>) value -> {
        while (value.hasNext()) {
            LOG.info("val =" + value.next());
        }
        return true;
    });
    env.run(sink);
}