Search in sources :

Example 11 with SourceTSet

use of edu.iu.dsc.tws.tset.sets.batch.SourceTSet in project twister2 by DSC-SPIDAL.

the class BranchingExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int para = 2;
    SourceTSet<Integer> src = dummySource(env, COUNT, para).setName("src0");
    KeyedTSet<Integer, Integer> left = src.mapToTuple(i -> new Tuple<>(i % 2, i)).setName("left");
    KeyedTSet<Integer, Integer> right = src.mapToTuple(i -> new Tuple<>(i % 2, i + 1)).setName("right");
    JoinTLink<Integer, Integer, Integer> join = left.join(right, CommunicationContext.JoinType.INNER, Integer::compareTo).setName("join");
    ComputeTSet<String> map = join.map(t -> "(" + t.getKey() + " " + t.getLeftValue() + " " + t.getRightValue() + ")").setName("map***");
    ComputeTSet<String> map1 = map.direct().map(s -> "###" + s).setName("map@@");
    ComputeTSet<String> union = map.union(map1).setName("union");
    union.direct().forEach(s -> LOG.info(s));
}
Also used : CommunicationContext(edu.iu.dsc.tws.api.comms.CommunicationContext) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) ComputeTSet(edu.iu.dsc.tws.tset.sets.batch.ComputeTSet) SourceTSet(edu.iu.dsc.tws.tset.sets.batch.SourceTSet) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) Logger(java.util.logging.Logger) KeyedTSet(edu.iu.dsc.tws.tset.sets.batch.KeyedTSet) JobConfig(edu.iu.dsc.tws.api.JobConfig) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) JoinTLink(edu.iu.dsc.tws.tset.links.batch.JoinTLink) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Example 12 with SourceTSet

use of edu.iu.dsc.tws.tset.sets.batch.SourceTSet in project twister2 by DSC-SPIDAL.

the class HelloTSet method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    LOG.info("Strating Hello TSet Example");
    int para = env.getConfig().getIntegerValue("para", 4);
    SourceTSet<int[]> source = env.createSource(new SourceFunc<int[]>() {

        private int count = 0;

        @Override
        public boolean hasNext() {
            return count < para;
        }

        @Override
        public int[] next() {
            count++;
            return new int[] { 1, 1, 1 };
        }
    }, para).setName("source");
    PartitionTLink<int[]> partitioned = source.partition(new LoadBalancePartitioner<>());
    ComputeTSet<int[]> mapedPartition = partitioned.map((MapFunc<int[], int[]>) input -> Arrays.stream(input).map(a -> a * 2).toArray());
    ReduceTLink<int[]> reduce = mapedPartition.reduce((t1, t2) -> {
        int[] ret = new int[t1.length];
        for (int i = 0; i < t1.length; i++) {
            ret[i] = t1[i] + t2[i];
        }
        return ret;
    });
    SinkTSet<int[]> sink = reduce.sink(value -> {
        LOG.info("Results " + Arrays.toString(value));
        return false;
    });
    env.run(sink);
    LOG.info("Ending  Hello TSet Example");
}
Also used : Twister2Job(edu.iu.dsc.tws.api.Twister2Job) Arrays(java.util.Arrays) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) Options(org.apache.commons.cli.Options) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) MapFunc(edu.iu.dsc.tws.api.tset.fn.MapFunc) JobConfig(edu.iu.dsc.tws.api.JobConfig) DefaultParser(org.apache.commons.cli.DefaultParser) ReduceTLink(edu.iu.dsc.tws.tset.links.batch.ReduceTLink) CommandLine(org.apache.commons.cli.CommandLine) ComputeTSet(edu.iu.dsc.tws.tset.sets.batch.ComputeTSet) SourceTSet(edu.iu.dsc.tws.tset.sets.batch.SourceTSet) CommandLineParser(org.apache.commons.cli.CommandLineParser) SinkTSet(edu.iu.dsc.tws.tset.sets.batch.SinkTSet) SourceFunc(edu.iu.dsc.tws.api.tset.fn.SourceFunc) LoadBalancePartitioner(edu.iu.dsc.tws.tset.fn.LoadBalancePartitioner) Logger(java.util.logging.Logger) Serializable(java.io.Serializable) Twister2Submitter(edu.iu.dsc.tws.rsched.job.Twister2Submitter) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) ParseException(org.apache.commons.cli.ParseException) PartitionTLink(edu.iu.dsc.tws.tset.links.batch.PartitionTLink) Twister2Worker(edu.iu.dsc.tws.api.resource.Twister2Worker) SourceFunc(edu.iu.dsc.tws.api.tset.fn.SourceFunc) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment)

Example 13 with SourceTSet

use of edu.iu.dsc.tws.tset.sets.batch.SourceTSet in project twister2 by DSC-SPIDAL.

the class AllGatherExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    LOG.info("test foreach");
    src.allGather().forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    src.allGather().map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    src.allGather().flatmap((i, c) -> c.collect(i.toString() + "##")).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test computec");
    src.allGather().compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        output.collect("sum=" + sum);
    }).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test compute");
    src.allGather().compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        return "sum=" + sum;
    }).direct().forEach(s -> LOG.info("compute: " + s));
}
Also used : Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) Iterator(java.util.Iterator) ComputeCollectorFunc(edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc) SourceTSet(edu.iu.dsc.tws.tset.sets.batch.SourceTSet) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) Logger(java.util.logging.Logger) JobConfig(edu.iu.dsc.tws.api.JobConfig) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) ComputeFunc(edu.iu.dsc.tws.api.tset.fn.ComputeFunc) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) Iterator(java.util.Iterator)

Example 14 with SourceTSet

use of edu.iu.dsc.tws.tset.sets.batch.SourceTSet in project twister2 by DSC-SPIDAL.

the class DirectExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM).setName("src");
    DirectTLink<Integer> direct = src.direct().setName("direct");
    LOG.info("test foreach");
    direct.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    direct.map(i -> i.toString() + "$$").setName("map").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    direct.flatmap((i, c) -> c.collect(i.toString() + "##")).setName("flatmap").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    direct.compute((ComputeFunc<Iterator<Integer>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        return "sum" + sum;
    }).setName("compute").withSchema(PrimitiveSchemas.STRING).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    direct.compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        output.collect("sum" + sum);
    }).setName("ccompute").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test map2tup");
    direct.mapToTuple(i -> new Tuple<>(i, i.toString())).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
    LOG.info("test sink");
    SinkTSet<Iterator<Integer>> sink = direct.sink((SinkFunc<Iterator<Integer>>) value -> {
        while (value.hasNext()) {
            LOG.info("val =" + value.next());
        }
        return true;
    });
    env.run(sink);
}
Also used : Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) Iterator(java.util.Iterator) ComputeCollectorFunc(edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc) SourceTSet(edu.iu.dsc.tws.tset.sets.batch.SourceTSet) DirectTLink(edu.iu.dsc.tws.tset.links.batch.DirectTLink) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) SinkTSet(edu.iu.dsc.tws.tset.sets.batch.SinkTSet) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) Logger(java.util.logging.Logger) JobConfig(edu.iu.dsc.tws.api.JobConfig) SinkFunc(edu.iu.dsc.tws.api.tset.fn.SinkFunc) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) ComputeFunc(edu.iu.dsc.tws.api.tset.fn.ComputeFunc) PrimitiveSchemas(edu.iu.dsc.tws.api.tset.schema.PrimitiveSchemas) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) Iterator(java.util.Iterator)

Example 15 with SourceTSet

use of edu.iu.dsc.tws.tset.sets.batch.SourceTSet in project twister2 by DSC-SPIDAL.

the class EvaluateExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    SourceTSet<Integer> src = dummyReplayableSource(env, COUNT, PARALLELISM).setName("src");
    DirectTLink<Integer> direct = src.direct().setName("direct");
    LOG.info("test foreach");
    ComputeTSet<Object> tset1 = direct.lazyForEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    ComputeTSet<Object> tset2 = direct.map(i -> i.toString() + "$$").setName("map").direct().lazyForEach(s -> LOG.info("map: " + s));
    for (int i = 0; i < 4; i++) {
        LOG.info("iter " + i);
        env.eval(tset1);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    // env.eval(tset2);
    }
    env.finishEval(tset1);
// env.finishEval(tset2);
/*    LOG.info("test flat map");
    direct.flatmap((i, c) -> c.collect(i.toString() + "##")).setName("flatmap")
        .direct()
        .lazyForEach(s -> LOG.info("flat:" + s));

    LOG.info("test compute");
    direct.compute((ComputeFunc<String, Iterator<Integer>>) input -> {
      int sum = 0;
      while (input.hasNext()) {
        sum += input.next();
      }
      return "sum" + sum;
    }).setName("compute")
        .direct()
        .lazyForEach(i -> LOG.info("comp: " + i));

    LOG.info("test computec");
    direct.compute((ComputeCollectorFunc<String, Iterator<Integer>>)
        (input, output) -> {
          int sum = 0;
          while (input.hasNext()) {
            sum += input.next();
          }
          output.collect("sum" + sum);
        }).setName("ccompute")
        .direct()
        .lazyForEach(s -> LOG.info("computec: " + s));

    LOG.info("test sink");
    direct.sink((SinkFunc<Iterator<Integer>>) value -> {
      while (value.hasNext()) {
        LOG.info("val =" + value.next());
      }
      return true;
    });*/
}
Also used : WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) ComputeTSet(edu.iu.dsc.tws.tset.sets.batch.ComputeTSet) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) SourceTSet(edu.iu.dsc.tws.tset.sets.batch.SourceTSet) DirectTLink(edu.iu.dsc.tws.tset.links.batch.DirectTLink) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) Logger(java.util.logging.Logger) JobConfig(edu.iu.dsc.tws.api.JobConfig) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment)

Aggregations

WorkerEnvironment (edu.iu.dsc.tws.api.resource.WorkerEnvironment)23 TSetEnvironment (edu.iu.dsc.tws.tset.env.TSetEnvironment)23 SourceTSet (edu.iu.dsc.tws.tset.sets.batch.SourceTSet)23 Logger (java.util.logging.Logger)23 JobConfig (edu.iu.dsc.tws.api.JobConfig)22 BatchEnvironment (edu.iu.dsc.tws.tset.env.BatchEnvironment)22 ResourceAllocator (edu.iu.dsc.tws.rsched.core.ResourceAllocator)20 Config (edu.iu.dsc.tws.api.config.Config)19 HashMap (java.util.HashMap)19 Tuple (edu.iu.dsc.tws.api.comms.structs.Tuple)13 Iterator (java.util.Iterator)13 ComputeFunc (edu.iu.dsc.tws.api.tset.fn.ComputeFunc)12 ComputeCollectorFunc (edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc)11 Twister2Job (edu.iu.dsc.tws.api.Twister2Job)6 Twister2Submitter (edu.iu.dsc.tws.rsched.job.Twister2Submitter)6 ComputeTSet (edu.iu.dsc.tws.tset.sets.batch.ComputeTSet)6 SinkTSet (edu.iu.dsc.tws.tset.sets.batch.SinkTSet)6 Twister2Worker (edu.iu.dsc.tws.api.resource.Twister2Worker)5 MapFunc (edu.iu.dsc.tws.api.tset.fn.MapFunc)5 SinkFunc (edu.iu.dsc.tws.api.tset.fn.SinkFunc)5