Examples with SourceFunc edu.iu.dsc.tws.api.tset.fn.SourceFunc used on opensource projects

Example 1 with SourceFunc

use of edu.iu.dsc.tws.api.tset.fn.SourceFunc in project twister2 by DSC-SPIDAL.

the class TSetCheckptExample method execute.

@Override
public void execute(WorkerEnvironment workerEnvironment) {
    BatchChkPntEnvironment env = TSetEnvironment.initCheckpointing(workerEnvironment);
    LOG.info(String.format("Hello from worker %d", env.getWorkerID()));
    SourceTSet<Integer> sourceX = env.createSource(new SourceFunc<Integer>() {

        private int count = 0;

        @Override
        public boolean hasNext() {
            return count < 10000;
        }

        @Override
        public Integer next() {
            return count++;
        }
    }, 4);
    long t1 = System.currentTimeMillis();
    ComputeTSet<Object> twoComputes = sourceX.direct().compute((itr, c) -> {
        itr.forEachRemaining(i -> {
            c.collect(i * 5);
        });
    }).direct().compute((itr, c) -> {
        itr.forEachRemaining(i -> {
            c.collect((int) i + 2);
        });
    });
    LOG.info("Time for two computes : " + (System.currentTimeMillis() - t1));
    t1 = System.currentTimeMillis();
    PersistedTSet<Object> persist = twoComputes.persist();
    LOG.info("Time for persist : " + (System.currentTimeMillis() - t1) / 1000);
    // When persist() is called, twister2 performs all the computations/communication
    // upto this point and persists the result into the disk.
    // This makes previous data garbage collectible and frees some memory.
    // If persist() is called in a checkpointing enabled job, this will create
    // a snapshot at this point and will start straightaway from this point if the
    // job is restarted.
    // Similar to CachedTSets, PersistedTSets can be added as inputs for other TSets and
    // operations
    persist.reduce((i1, i2) -> {
        return (int) i1 + (int) i2;
    }).forEach(i -> {
        LOG.info("SUM=" + i);
    });
}

Example 2 with SourceFunc

use of edu.iu.dsc.tws.api.tset.fn.SourceFunc in project twister2 by DSC-SPIDAL.

the class PartitionExample method execute.

@Override
public void execute(WorkerEnvironment workerEnvironment) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnvironment);
    List<TField> fieldList = new ArrayList<>();
    fieldList.add(new TField("first", MessageTypes.INTEGER));
    fieldList.add(new TField("second", MessageTypes.DOUBLE));
    RowSourceTSet src = env.createRowSource("row", new SourceFunc<Row>() {

        private int count = 0;

        @Override
        public boolean hasNext() {
            return count++ < 1000;
        }

        @Override
        public Row next() {
            return new TwoRow(1, 4.1);
        }
    }, 4).withSchema(new RowSchema(fieldList));
    BatchRowTLink partition = src.partition(new PartitionFunc<Row>() {

        private List<Integer> targets;

        private Random random;

        private int c = 0;

        private Map<Integer, Integer> counts = new HashMap<>();

        @Override
        public void prepare(Set<Integer> sources, Set<Integer> destinations) {
            targets = new ArrayList<>(destinations);
            random = new Random();
            for (int t : targets) {
                counts.put(t, 0);
            }
        }

        @Override
        public int partition(int sourceIndex, Row val) {
            int index = random.nextInt(targets.size());
            int count = counts.get(index);
            counts.put(index, count + 1);
            c++;
            if (c == 1000) {
                LOG.info("COUNTS " + counts);
            }
            return targets.get(index);
        }
    }, 4, 0);
    partition.forEach(new ApplyFunc<Row>() {

        private TSetContext ctx;

        private int count;

        @Override
        public void prepare(TSetContext context) {
            ctx = context;
        }

        @Override
        public void apply(Row data) {
            LOG.info(ctx.getIndex() + " Data " + data.get(0) + ", " + data.get(1) + ", count " + count++);
        }
    });
}

Example 3 with SourceFunc

use of edu.iu.dsc.tws.api.tset.fn.SourceFunc in project twister2 by DSC-SPIDAL.

the class TSetCommunicationExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    LOG.info(String.format("Hello from worker %d", env.getWorkerID()));
    SourceTSet<Integer> sourceX = env.createSource(new SourceFunc<Integer>() {

        private int count = 0;

        @Override
        public boolean hasNext() {
            return count < 10;
        }

        @Override
        public Integer next() {
            return count++;
        }
    }, 4);
    sourceX.direct().compute((itr, collector) -> {
        itr.forEachRemaining(i -> {
            collector.collect(i * 5);
        });
    }).direct().compute((itr, collector) -> {
        itr.forEachRemaining(i -> {
            collector.collect((int) i + 2);
        });
    }).reduce((i1, i2) -> {
        return (int) i1 + (int) i2;
    }).forEach(i -> {
        LOG.info("SUM=" + i);
    });
}

Example 4 with SourceFunc

use of edu.iu.dsc.tws.api.tset.fn.SourceFunc in project twister2 by DSC-SPIDAL.

the class HelloTSet method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    LOG.info("Strating Hello TSet Example");
    int para = env.getConfig().getIntegerValue("para", 4);
    SourceTSet<int[]> source = env.createSource(new SourceFunc<int[]>() {

        private int count = 0;

        @Override
        public boolean hasNext() {
            return count < para;
        }

        @Override
        public int[] next() {
            count++;
            return new int[] { 1, 1, 1 };
        }
    }, para).setName("source");
    PartitionTLink<int[]> partitioned = source.partition(new LoadBalancePartitioner<>());
    ComputeTSet<int[]> mapedPartition = partitioned.map((MapFunc<int[], int[]>) input -> Arrays.stream(input).map(a -> a * 2).toArray());
    ReduceTLink<int[]> reduce = mapedPartition.reduce((t1, t2) -> {
        int[] ret = new int[t1.length];
        for (int i = 0; i < t1.length; i++) {
            ret[i] = t1[i] + t2[i];
        }
        return ret;
    });
    SinkTSet<int[]> sink = reduce.sink(value -> {
        LOG.info("Results " + Arrays.toString(value));
        return false;
    });
    env.run(sink);
    LOG.info("Ending  Hello TSet Example");
}