Examples with BatchEnvironment edu.iu.dsc.tws.tset.env.BatchEnvironment used on opensource projects

Example 31 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class ReadSourceTest method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    System.out.println("Rank " + env.getWorkerID());
    Twister2PipelineOptions options = PipelineOptionsFactory.as(Twister2PipelineOptions.class);
    options.setTSetEnvironment(env);
    options.as(Twister2PipelineOptions.class).setRunner(Twister2LegacyRunner.class);
    String resultPath = "/tmp/testdir";
    Pipeline p = Pipeline.create(options);
    PCollection<String> result = p.apply(GenerateSequence.from(0).to(10)).apply(ParDo.of(new DoFn<Long, String>() {

        @ProcessElement
        public void processElement(ProcessContext c) throws Exception {
            c.output(c.element().toString());
        }
    }));
    try {
        result.apply(TextIO.write().to(new URI(resultPath).getPath() + "/part"));
    } catch (URISyntaxException e) {
        LOG.info(e.getMessage());
    }
    p.run();
    System.out.println("Result " + result.toString());
}

Example 32 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class UnionExample2 method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src1 = dummySource(env, start, COUNT, PARALLELISM).setName("src1");
    ComputeTSet<Integer> map = src1.direct().map(i -> i + 50);
    ComputeTSet<Integer> unionTSet = src1.union(map);
    LOG.info("test source union");
    unionTSet.direct().forEach(s -> LOG.info("union: " + s));
}

Example 33 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class KeyedOperationsExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    KeyedSourceTSet<String, Integer> kSource = dummyKeyedSource(env, COUNT, PARALLELISM);
    KeyedDirectTLink<String, Integer> kDirect = kSource.keyedDirect();
    kDirect.forEach(i -> LOG.info("d_fe: " + i.toString()));
    KeyedCachedTSet<String, Integer> cache = kDirect.cache();
    cache.keyedDirect().forEach(i -> LOG.info("c_d_fe: " + i.toString()));
    cache.keyedReduce(Integer::sum).forEach(i -> LOG.info("c_r_fe: " + i.toString()));
    cache.keyedGather().forEach((ApplyFunc<Tuple<String, Iterator<Integer>>>) data -> {
        StringBuilder sb = new StringBuilder();
        sb.append("c_g_fe: key ").append(data.getKey()).append("->");
        while (data.getValue().hasNext()) {
            sb.append(" ").append(data.getValue().next());
        }
        LOG.info(sb.toString());
    });
}

Example 34 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class JoinExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int para = 2;
    int workerID = env.getWorkerID();
    SourceTSet<Integer> src0 = dummySource(env, COUNT, para).setName("src0");
    KeyedTSet<Integer, Integer> left = src0.mapToTuple(i -> new Tuple<>(i % 2, i)).setName("left");
    left.keyedDirect().forEach(i -> LOG.info(workerID + "L " + i.toString()));
    SourceTSet<Integer> src1 = dummySource(env, COUNT, para).setName("src1");
    KeyedTSet<Integer, Integer> right = src1.mapToTuple(i -> new Tuple<>(i % 2, i)).setName("right");
    right.keyedDirect().forEach(i -> LOG.info(workerID + "R " + i.toString()));
    JoinTLink<Integer, Integer, Integer> join = left.join(right, CommunicationContext.JoinType.INNER, Integer::compareTo).setName("join");
    join.forEach(t -> LOG.info(workerID + "out: " + t.toString()));
}

Example 35 with BatchEnvironment

use of edu.iu.dsc.tws.tset.env.BatchEnvironment in project twister2 by DSC-SPIDAL.

the class GatherExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    GatherTLink<Integer> gather = src.gather();
    LOG.info("test foreach");
    gather.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    gather.map(i -> i.toString() + "$$").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test compute");
    gather.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        return "sum=" + sum;
    }).withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("compute: " + s));
    LOG.info("test computec");
    gather.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        output.collect("sum=" + sum);
    }).withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("computec: " + s));
    gather.mapToTuple(i -> new Tuple<>(i % 2, i)).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
}