Examples with BatchEnvironment edu.iu.dsc.tws.tset.env.BatchEnvironment used on opensource projects

Example 26 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class AllGatherExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    LOG.info("test foreach");
    src.allGather().forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    src.allGather().map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    src.allGather().flatmap((i, c) -> c.collect(i.toString() + "##")).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test computec");
    src.allGather().compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        output.collect("sum=" + sum);
    }).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test compute");
    src.allGather().compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        return "sum=" + sum;
    }).direct().forEach(s -> LOG.info("compute: " + s));
}

Example 27 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class DirectExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM).setName("src");
    DirectTLink<Integer> direct = src.direct().setName("direct");
    LOG.info("test foreach");
    direct.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    direct.map(i -> i.toString() + "$$").setName("map").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    direct.flatmap((i, c) -> c.collect(i.toString() + "##")).setName("flatmap").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    direct.compute((ComputeFunc<Iterator<Integer>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        return "sum" + sum;
    }).setName("compute").withSchema(PrimitiveSchemas.STRING).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    direct.compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        output.collect("sum" + sum);
    }).setName("ccompute").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test map2tup");
    direct.mapToTuple(i -> new Tuple<>(i, i.toString())).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
    LOG.info("test sink");
    SinkTSet<Iterator<Integer>> sink = direct.sink((SinkFunc<Iterator<Integer>>) value -> {
        while (value.hasNext()) {
            LOG.info("val =" + value.next());
        }
        return true;
    });
    env.run(sink);
}

Example 28 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class EvaluateExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    SourceTSet<Integer> src = dummyReplayableSource(env, COUNT, PARALLELISM).setName("src");
    DirectTLink<Integer> direct = src.direct().setName("direct");
    LOG.info("test foreach");
    ComputeTSet<Object> tset1 = direct.lazyForEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    ComputeTSet<Object> tset2 = direct.map(i -> i.toString() + "$$").setName("map").direct().lazyForEach(s -> LOG.info("map: " + s));
    for (int i = 0; i < 4; i++) {
        LOG.info("iter " + i);
        env.eval(tset1);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    // env.eval(tset2);
    }
    env.finishEval(tset1);
// env.finishEval(tset2);
/*    LOG.info("test flat map");
    direct.flatmap((i, c) -> c.collect(i.toString() + "##")).setName("flatmap")
        .direct()
        .lazyForEach(s -> LOG.info("flat:" + s));

    LOG.info("test compute");
    direct.compute((ComputeFunc<String, Iterator<Integer>>) input -> {
      int sum = 0;
      while (input.hasNext()) {
        sum += input.next();
      }
      return "sum" + sum;
    }).setName("compute")
        .direct()
        .lazyForEach(i -> LOG.info("comp: " + i));

    LOG.info("test computec");
    direct.compute((ComputeCollectorFunc<String, Iterator<Integer>>)
        (input, output) -> {
          int sum = 0;
          while (input.hasNext()) {
            sum += input.next();
          }
          output.collect("sum" + sum);
        }).setName("ccompute")
        .direct()
        .lazyForEach(s -> LOG.info("computec: " + s));

    LOG.info("test sink");
    direct.sink((SinkFunc<Iterator<Integer>>) value -> {
      while (value.hasNext()) {
        LOG.info("val =" + value.next());
      }
      return true;
    });*/
}

Example 29 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class SvmSgdTsetRunner method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    // Method 1
    // initializeParameters();
    // trainingData = loadTrainingData();
    // testingData = loadTestingData();
    // trainedWeightVector = loadWeightVector();
    // TSetUtils.printCachedTset(trainedWeightVector,
    // doubles -> System.out.println(Arrays.toString(doubles)));
    // executeTraining();
    // executePredict();
    // Method 2
    executeAll(env);
}

Example 30 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class BeamBatchWorker method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    Config config = env.getConfig();
    Map<PCollectionView<?>, BatchTSet<?>> sideInputs = (Map<PCollectionView<?>, BatchTSet<?>>) config.get(SIDEINPUTS);
    Set<TSet> leaves = (Set<TSet>) config.get(LEAVES);
}