Examples with BatchEnvironment edu.iu.dsc.tws.tset.env.BatchEnvironment used on opensource projects

Example 46 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class UnionExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src1 = dummySource(env, start, COUNT, PARALLELISM).setName("src1");
    SourceTSet<Integer> src2 = dummySourceOther(env, COUNT, PARALLELISM).setName("src2");
    ComputeTSet<Integer> unionTSet = src1.union(src2);
    LOG.info("test source union");
    unionTSet.direct().forEach(s -> LOG.info("map: " + s));
}

Example 47 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class TSetKeyedReduceExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    super.execute(workerEnv);
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
/*    super.execute(tc);

    // set the parallelism of source to task stage 0
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    BatchSourceTSet<int[]> source = tc.createSource(new TestBaseSource(),
        sourceParallelism).setName("Source");
    KeyedReduceTLink<int[], int[]> reduce = source.
        groupBy(new LoadBalancePartitioner<>(), new IdentitySelector<>()).
          keyedReduce((t1, t2) -> {
            int[] val = new int[t1.length];
            for (int i = 0; i < t1.length; i++) {
              val[i] = t1[i] + t2[i];
            }
            return val;
          });

    reduce.sink(new BaseSink<int[]>() {
      @Override
      public boolean add(int[] value) {
        experimentData.setOutput(value);
        try {
          LOG.info("Task Id : " + context.getIndex() + " Results " + Arrays.toString(value));
          verify(OperationNames.REDUCE);
        } catch (VerificationException e) {
          LOG.info("Exception Message : " + e.getMessage());
        }
        return true;
      }

      @Override
      public void prepare() {
      }
    }, sinkParallelism);*/
}

Example 48 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class TSetReduceExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    super.execute(workerEnv);
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    // set the parallelism of source to task stage 0
    int srcPara = jobParameters.getTaskStages().get(0);
    int sinkPara = jobParameters.getTaskStages().get(1);
    SourceTSet<int[]> source = env.createSource(new TestBaseSource(), srcPara).setName("Source");
    ReduceTLink<int[]> reduce = source.reduce((t1, t2) -> {
        int[] val = new int[t1.length];
        for (int i = 0; i < t1.length; i++) {
            val[i] = t1[i] + t2[i];
        }
        return val;
    });
    SinkTSet<int[]> sink = reduce.sink(value -> {
        experimentData.setOutput(value);
        LOG.info("Result : " + Arrays.toString(value));
        try {
            verify(OperationNames.REDUCE);
        } catch (VerificationException e) {
            LOG.info("Exception Message : " + e.getMessage());
        }
        return true;
    });
    env.run(sink);
}

Example 49 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class TSetReplicateExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    super.execute(workerEnv);
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
/*    super.execute(tc);

    // set the parallelism of source to task stage 0
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    BatchSourceTSet<int[]> source = tc.createSource(new TestBaseSource(), sourceParallelism).
        setName("Source");
    ReplicateTLink<int[]> replicate = source.replicate(10);

    replicate.sink(new BaseSink<int[]>() {
      @Override
      public boolean add(int[] value) {
        experimentData.setOutput(value);
        LOG.info("Task Id : " + context.getIndex() + " Results " + Arrays.toString(value));
        try {
          verify(OperationNames.BROADCAST);
        } catch (VerificationException e) {
          LOG.info("Exception Message : " + e.getMessage());
        }
        return true;
      }

      @Override
      public void prepare() {
      }
    });*/
}

Example 50 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class TSetComputeExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    LOG.info(String.format("Hello from worker %d", env.getWorkerID()));
    SourceTSet<Integer> sourceX = env.createSource(new SourceFunc<Integer>() {

        private int count = 0;

        @Override
        public boolean hasNext() {
            return count < 10;
        }

        @Override
        public Integer next() {
            return count++;
        }
    }, 4);
    sourceX.direct().compute((itr, collector) -> {
        // if each element of the iterator should be processed individually, compute
        // function which accepts a ComputeCollector can be used.
        itr.forEachRemaining(i -> {
            collector.collect(i * 5);
        });
    }).direct().compute((itr, collector) -> {
        itr.forEachRemaining(i -> {
            collector.collect((int) i + 2);
        });
    }).direct().forEach(i -> {
        LOG.info("(x * 5 ) + 2 = " + i);
    });
}