Examples with SinkTSet edu.iu.dsc.tws.tset.sets.batch.SinkTSet used on opensource projects

Example 6 with SinkTSet

use of edu.iu.dsc.tws.tset.sets.batch.SinkTSet in project twister2 by DSC-SPIDAL.

the class HelloTSet method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    LOG.info("Strating Hello TSet Example");
    int para = env.getConfig().getIntegerValue("para", 4);
    SourceTSet<int[]> source = env.createSource(new SourceFunc<int[]>() {

        private int count = 0;

        @Override
        public boolean hasNext() {
            return count < para;
        }

        @Override
        public int[] next() {
            count++;
            return new int[] { 1, 1, 1 };
        }
    }, para).setName("source");
    PartitionTLink<int[]> partitioned = source.partition(new LoadBalancePartitioner<>());
    ComputeTSet<int[]> mapedPartition = partitioned.map((MapFunc<int[], int[]>) input -> Arrays.stream(input).map(a -> a * 2).toArray());
    ReduceTLink<int[]> reduce = mapedPartition.reduce((t1, t2) -> {
        int[] ret = new int[t1.length];
        for (int i = 0; i < t1.length; i++) {
            ret[i] = t1[i] + t2[i];
        }
        return ret;
    });
    SinkTSet<int[]> sink = reduce.sink(value -> {
        LOG.info("Results " + Arrays.toString(value));
        return false;
    });
    env.run(sink);
    LOG.info("Ending  Hello TSet Example");
}

Example 7 with SinkTSet

use of edu.iu.dsc.tws.tset.sets.batch.SinkTSet in project twister2 by DSC-SPIDAL.

the class DirectExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM).setName("src");
    DirectTLink<Integer> direct = src.direct().setName("direct");
    LOG.info("test foreach");
    direct.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    direct.map(i -> i.toString() + "$$").setName("map").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    direct.flatmap((i, c) -> c.collect(i.toString() + "##")).setName("flatmap").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    direct.compute((ComputeFunc<Iterator<Integer>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        return "sum" + sum;
    }).setName("compute").withSchema(PrimitiveSchemas.STRING).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    direct.compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        output.collect("sum" + sum);
    }).setName("ccompute").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test map2tup");
    direct.mapToTuple(i -> new Tuple<>(i, i.toString())).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
    LOG.info("test sink");
    SinkTSet<Iterator<Integer>> sink = direct.sink((SinkFunc<Iterator<Integer>>) value -> {
        while (value.hasNext()) {
            LOG.info("val =" + value.next());
        }
        return true;
    });
    env.run(sink);
}

Example 8 with SinkTSet

use of edu.iu.dsc.tws.tset.sets.batch.SinkTSet in project beam by apache.

the class BeamBatchWorker method executePipeline.

public void executePipeline(BatchTSetEnvironment env) {
    Map<String, CachedTSet> sideInputTSets = new HashMap<>();
    for (Map.Entry<String, BatchTSet<?>> sides : sideInputDataSets.entrySet()) {
        BatchTSet<?> sideTSet = sides.getValue();
        addInputs((BaseTSet) sideTSet, sideInputTSets);
        CachedTSet tempCache = (CachedTSet) sideTSet.cache();
        sideInputTSets.put(sides.getKey(), tempCache);
    }
    for (TSet leaf : leaves) {
        SinkTSet sinkTSet = (SinkTSet) leaf.direct().sink(new Twister2SinkFunction());
        addInputs(sinkTSet, sideInputTSets);
        eval(env, sinkTSet);
    }
}

Example 9 with SinkTSet

use of edu.iu.dsc.tws.tset.sets.batch.SinkTSet in project twister2 by DSC-SPIDAL.

the class BroadcastExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    SourceTSet<Integer> src = dummySource(env, COUNT, 1);
    ReplicateTLink<Integer> replicate = src.replicate(PARALLELISM);
    LOG.info("test foreach");
    replicate.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    replicate.map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    replicate.flatmap((i, c) -> c.collect(i.toString() + "##")).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    replicate.compute((ComputeFunc<Iterator<Integer>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        return "sum" + sum;
    }).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    replicate.compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        output.collect("sum" + sum);
    }).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test sink");
    SinkTSet<Iterator<Integer>> sink = replicate.sink((SinkFunc<Iterator<Integer>>) value -> {
        while (value.hasNext()) {
            LOG.info("val =" + value.next());
        }
        return true;
    });
    env.run(sink);
}