Examples with BatchEnvironment edu.iu.dsc.tws.tset.env.BatchEnvironment used on opensource projects

Example 6 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class KReduceExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    KeyedReduceTLink<Integer, Integer> kreduce = src.mapToTuple(i -> new Tuple<>(i % 10, i)).keyedReduce(Integer::sum);
    LOG.info("test foreach");
    kreduce.forEach(t -> LOG.info("sum by key=" + t.getKey() + ", " + t.getValue()));
    LOG.info("test map");
    kreduce.map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test compute");
    kreduce.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
        StringBuilder s = new StringBuilder();
        while (input.hasNext()) {
            s.append(input.next().toString()).append(" ");
        }
        return s.toString();
    }).direct().forEach(s -> LOG.info("compute: concat " + s));
    LOG.info("test computec");
    kreduce.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
        while (input.hasNext()) {
            output.collect(input.next().toString());
        }
    }).direct().forEach(s -> LOG.info("computec: " + s));
}

Example 7 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class TSetPartitionExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    super.execute(workerEnv);
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
/*    super.execute(tc);

    // set the parallelism of source to task stage 0
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    BatchSourceTSet<int[]> source = tc.createSource(new TestBaseSource(),
        sourceParallelism).setName("Source");
    PartitionTLink<int[]> partition = source.partition(new OneToOnePartitioner<>());

    partition.sink(new BaseSink<int[]>() {
      @Override
      public boolean add(int[] value) {
        LOG.info("Task Id : " + context.getIndex() + " Results " + Arrays.toString(value));
        experimentData.setOutput(value);
        try {
          verify(OperationNames.PARTITION);
        } catch (VerificationException e) {
          LOG.info("Exception Message : " + e.getMessage());
        }
        return true;
      }

      @Override
      public void prepare() {
      }
    }, sinkParallelism);*/
}

Example 8 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class TSetSourceExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    LOG.info(String.format("Hello from worker %d", env.getWorkerID()));
    SourceTSet<Integer> sourceX = env.createSource(new SourceFunc<Integer>() {

        private int count = 0;

        @Override
        public boolean hasNext() {
            return count < 10;
        }

        @Override
        public Integer next() {
            return count++;
        }
    }, 4);
    sourceX.direct().forEach(i -> {
        LOG.info("i : " + i);
    });
}

Example 9 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class TSetAllGatherExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    super.execute(workerEnv);
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    // set the parallelism of source to task stage 0
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    SourceTSet<int[]> source = env.createSource(new TestBaseSource(), sourceParallelism).setName("Source");
    AllGatherTLink<int[]> gather = source.allGather();
    SinkTSet<Iterator<Tuple<Integer, int[]>>> sink = gather.sink(new SinkFunc<Iterator<Tuple<Integer, int[]>>>() {

        private TSetContext context;

        @Override
        public boolean add(Iterator<Tuple<Integer, int[]>> value) {
            // todo: check this!
            int[] result = new int[0];
            while (value.hasNext()) {
                Tuple<Integer, int[]> t = value.next();
                if (t.getKey().equals(context.getIndex())) {
                    result = t.getValue();
                    break;
                }
            }
            LOG.info("Task Id : " + context.getIndex() + " Results " + Arrays.toString(result));
            experimentData.setOutput(value);
            try {
                verify(OperationNames.ALLGATHER);
            } catch (VerificationException e) {
                LOG.info("Exception Message : " + e.getMessage());
            }
            return true;
        }

        @Override
        public void prepare(TSetContext ctx) {
            this.context = ctx;
        }
    });
    env.run(sink);
}

Example 10 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class PipeExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    SourceTSet<Integer> src = dummySource(env, COUNT, PARALLELISM).setName("src");
    src.pipe().forEach(input -> {
        LOG.info("Pipe input " + input.toString());
    });
    // cache the src input (first task graph will be created and executed)
    CachedTSet<Integer> cachedInput = src.pipe().cache();
    LOG.info("test direct iteration");
    ComputeTSet<Object> lazyForEach = cachedInput.pipe().lazyForEach(input -> {
        LOG.info("##" + input.toString());
    });
    // will be executed 4 times)
    for (int i = 0; i < 4; i++) {
        env.eval(lazyForEach);
    }
    env.finishEval(lazyForEach);
}