Examples with BaseComputeCollectorFunc edu.iu.dsc.tws.api.tset.fn.BaseComputeCollectorFunc used on opensource projects

Example 1 with BaseComputeCollectorFunc

use of edu.iu.dsc.tws.api.tset.fn.BaseComputeCollectorFunc in project twister2 by DSC-SPIDAL.

the class KeyedAddInputsExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    KeyedSourceTSet<String, Integer> src0 = dummyKeyedSource(env, COUNT, PARALLELISM);
    KeyedSourceTSet<String, Integer> src1 = dummyKeyedSourceOther(env, COUNT, PARALLELISM);
    KeyedCachedTSet<String, Integer> cache0 = src0.cache();
    KeyedCachedTSet<String, Integer> cache1 = src1.cache();
    ComputeTSet<String> comp = cache0.keyedDirect().compute(new BaseComputeCollectorFunc<Iterator<Tuple<String, Integer>>, String>() {

        private Map<String, Integer> input1 = new HashMap<>();

        @Override
        public void prepare(TSetContext ctx) {
            super.prepare(ctx);
            // populate the hashmap with values from the input
            DataPartitionConsumer<Tuple<String, Integer>> part = (DataPartitionConsumer<Tuple<String, Integer>>) getInput("input").getConsumer();
            while (part.hasNext()) {
                Tuple<String, Integer> next = part.next();
                input1.put(next.getKey(), next.getValue());
            }
        }

        @Override
        public void compute(Iterator<Tuple<String, Integer>> input, RecordCollector<String> output) {
            while (input.hasNext()) {
                Tuple<String, Integer> next = input.next();
                output.collect(next.getKey() + " -> " + next.getValue() + ", " + input1.get(next.getKey()));
            }
        }
    }).addInput("input", cache1);
    comp.direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("Test lazy cache!");
    ComputeTSet<Object> forEach = comp.direct().lazyForEach(i -> LOG.info("comp-lazy: " + i));
    for (int i = 0; i < 4; i++) {
        LOG.info("iter: " + i);
        env.eval(forEach);
        try {
            Thread.sleep(2000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    env.finishEval(forEach);
}