use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class BranchingExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
int para = 2;
SourceTSet<Integer> src = dummySource(env, COUNT, para).setName("src0");
KeyedTSet<Integer, Integer> left = src.mapToTuple(i -> new Tuple<>(i % 2, i)).setName("left");
KeyedTSet<Integer, Integer> right = src.mapToTuple(i -> new Tuple<>(i % 2, i + 1)).setName("right");
JoinTLink<Integer, Integer, Integer> join = left.join(right, CommunicationContext.JoinType.INNER, Integer::compareTo).setName("join");
ComputeTSet<String> map = join.map(t -> "(" + t.getKey() + " " + t.getLeftValue() + " " + t.getRightValue() + ")").setName("map***");
ComputeTSet<String> map1 = map.direct().map(s -> "###" + s).setName("map@@");
ComputeTSet<String> union = map.union(map1).setName("union");
union.direct().forEach(s -> LOG.info(s));
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class HelloTSet method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
LOG.info("Strating Hello TSet Example");
int para = env.getConfig().getIntegerValue("para", 4);
SourceTSet<int[]> source = env.createSource(new SourceFunc<int[]>() {
private int count = 0;
@Override
public boolean hasNext() {
return count < para;
}
@Override
public int[] next() {
count++;
return new int[] { 1, 1, 1 };
}
}, para).setName("source");
PartitionTLink<int[]> partitioned = source.partition(new LoadBalancePartitioner<>());
ComputeTSet<int[]> mapedPartition = partitioned.map((MapFunc<int[], int[]>) input -> Arrays.stream(input).map(a -> a * 2).toArray());
ReduceTLink<int[]> reduce = mapedPartition.reduce((t1, t2) -> {
int[] ret = new int[t1.length];
for (int i = 0; i < t1.length; i++) {
ret[i] = t1[i] + t2[i];
}
return ret;
});
SinkTSet<int[]> sink = reduce.sink(value -> {
LOG.info("Results " + Arrays.toString(value));
return false;
});
env.run(sink);
LOG.info("Ending Hello TSet Example");
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class AllGatherExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
int start = env.getWorkerID() * 100;
SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
LOG.info("test foreach");
src.allGather().forEach(i -> LOG.info("foreach: " + i));
LOG.info("test map");
src.allGather().map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
LOG.info("test flat map");
src.allGather().flatmap((i, c) -> c.collect(i.toString() + "##")).direct().forEach(s -> LOG.info("flat:" + s));
LOG.info("test computec");
src.allGather().compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
int sum = 0;
while (input.hasNext()) {
sum += input.next().getValue();
}
output.collect("sum=" + sum);
}).direct().forEach(s -> LOG.info("computec: " + s));
LOG.info("test compute");
src.allGather().compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
int sum = 0;
while (input.hasNext()) {
sum += input.next().getValue();
}
return "sum=" + sum;
}).direct().forEach(s -> LOG.info("compute: " + s));
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class DirectExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
int start = env.getWorkerID() * 100;
SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM).setName("src");
DirectTLink<Integer> direct = src.direct().setName("direct");
LOG.info("test foreach");
direct.forEach(i -> LOG.info("foreach: " + i));
LOG.info("test map");
direct.map(i -> i.toString() + "$$").setName("map").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("map: " + s));
LOG.info("test flat map");
direct.flatmap((i, c) -> c.collect(i.toString() + "##")).setName("flatmap").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("flat:" + s));
LOG.info("test compute");
direct.compute((ComputeFunc<Iterator<Integer>, String>) input -> {
int sum = 0;
while (input.hasNext()) {
sum += input.next();
}
return "sum" + sum;
}).setName("compute").withSchema(PrimitiveSchemas.STRING).direct().forEach(i -> LOG.info("comp: " + i));
LOG.info("test computec");
direct.compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, output) -> {
int sum = 0;
while (input.hasNext()) {
sum += input.next();
}
output.collect("sum" + sum);
}).setName("ccompute").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("computec: " + s));
LOG.info("test map2tup");
direct.mapToTuple(i -> new Tuple<>(i, i.toString())).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
LOG.info("test sink");
SinkTSet<Iterator<Integer>> sink = direct.sink((SinkFunc<Iterator<Integer>>) value -> {
while (value.hasNext()) {
LOG.info("val =" + value.next());
}
return true;
});
env.run(sink);
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class EvaluateExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
SourceTSet<Integer> src = dummyReplayableSource(env, COUNT, PARALLELISM).setName("src");
DirectTLink<Integer> direct = src.direct().setName("direct");
LOG.info("test foreach");
ComputeTSet<Object> tset1 = direct.lazyForEach(i -> LOG.info("foreach: " + i));
LOG.info("test map");
ComputeTSet<Object> tset2 = direct.map(i -> i.toString() + "$$").setName("map").direct().lazyForEach(s -> LOG.info("map: " + s));
for (int i = 0; i < 4; i++) {
LOG.info("iter " + i);
env.eval(tset1);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
// env.eval(tset2);
}
env.finishEval(tset1);
// env.finishEval(tset2);
/* LOG.info("test flat map");
direct.flatmap((i, c) -> c.collect(i.toString() + "##")).setName("flatmap")
.direct()
.lazyForEach(s -> LOG.info("flat:" + s));
LOG.info("test compute");
direct.compute((ComputeFunc<String, Iterator<Integer>>) input -> {
int sum = 0;
while (input.hasNext()) {
sum += input.next();
}
return "sum" + sum;
}).setName("compute")
.direct()
.lazyForEach(i -> LOG.info("comp: " + i));
LOG.info("test computec");
direct.compute((ComputeCollectorFunc<String, Iterator<Integer>>)
(input, output) -> {
int sum = 0;
while (input.hasNext()) {
sum += input.next();
}
output.collect("sum" + sum);
}).setName("ccompute")
.direct()
.lazyForEach(s -> LOG.info("computec: " + s));
LOG.info("test sink");
direct.sink((SinkFunc<Iterator<Integer>>) value -> {
while (value.hasNext()) {
LOG.info("val =" + value.next());
}
return true;
});*/
}
Aggregations