use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class KGatherUngroupedExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
SourceTSet<Integer> src = dummySource(env, COUNT, PARALLELISM);
KeyedGatherUngroupedTLink<Integer, Integer> klink = src.mapToTuple(i -> new Tuple<>(i % 4, i)).keyedGatherUngrouped();
LOG.info("test foreach");
klink.forEach((ApplyFunc<Tuple<Integer, Integer>>) data -> LOG.info(data.getKey() + " -> " + data.getValue()));
LOG.info("test map");
klink.map((MapFunc<Tuple<Integer, Integer>, String>) input -> input.getKey() + " -> " + input.getValue()).direct().forEach(s -> LOG.info("map: " + s));
LOG.info("test compute");
klink.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
StringBuilder sb = new StringBuilder();
while (input.hasNext()) {
Tuple<Integer, Integer> next = input.next();
sb.append("[").append(next.getKey()).append("->").append(next.getValue()).append("]");
}
return sb.toString();
}).direct().forEach(s -> LOG.info("compute: " + s));
LOG.info("test computec");
klink.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
while (input.hasNext()) {
Tuple<Integer, Integer> next = input.next();
output.collect(next.getKey() + " -> " + next.getValue() * 2);
}
}).direct().forEach(s -> LOG.info("computec: " + s));
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class KReduceExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
int start = env.getWorkerID() * 100;
SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
KeyedReduceTLink<Integer, Integer> kreduce = src.mapToTuple(i -> new Tuple<>(i % 10, i)).keyedReduce(Integer::sum);
LOG.info("test foreach");
kreduce.forEach(t -> LOG.info("sum by key=" + t.getKey() + ", " + t.getValue()));
LOG.info("test map");
kreduce.map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
LOG.info("test compute");
kreduce.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
StringBuilder s = new StringBuilder();
while (input.hasNext()) {
s.append(input.next().toString()).append(" ");
}
return s.toString();
}).direct().forEach(s -> LOG.info("compute: concat " + s));
LOG.info("test computec");
kreduce.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
while (input.hasNext()) {
output.collect(input.next().toString());
}
}).direct().forEach(s -> LOG.info("computec: " + s));
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class SReduceWindowExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
StreamingEnvironment env = TSetEnvironment.initStreaming(workerEnv);
SSourceTSet<Integer> src = dummySource(env, ELEMENTS_IN_STREAM, PARALLELISM);
SDirectTLink<Integer> link = src.direct();
if (COUNT_WINDOWS) {
if (PROCESS_WINDOW) {
WindowComputeTSet<Iterator<Integer>> winTSet = link.countWindow(2);
WindowComputeTSet<Iterator<Integer>> processedTSet = winTSet.process((WindowComputeFunc<Iterator<Integer>, Iterator<Integer>>) input -> {
List<Integer> list = new ArrayList<>();
while (input.hasNext()) {
list.add(input.next());
}
return list.iterator();
});
processedTSet.direct().forEach((ApplyFunc<Iterator<Integer>>) data -> {
while (data.hasNext()) {
System.out.println(data.next());
}
});
}
if (REDUCE_WINDOW) {
WindowComputeTSet<Integer> winTSet = link.countWindow(2);
WindowComputeTSet<Integer> localReducedTSet = winTSet.aggregate((AggregateFunc<Integer>) Integer::sum);
localReducedTSet.direct().forEach(x -> System.out.println(x));
}
}
if (DURATION_WINDOWS) {
if (PROCESS_WINDOW) {
System.out.println("DURATION PROCESS WINDOW");
WindowComputeTSet<Iterator<Integer>> winTSet = link.timeWindow(2, TimeUnit.MILLISECONDS);
WindowComputeTSet<Iterator<Integer>> processedTSet = winTSet.process((WindowComputeFunc<Iterator<Integer>, Iterator<Integer>>) input -> {
List<Integer> list = new ArrayList<>();
while (input.hasNext()) {
list.add(input.next());
}
return list.iterator();
});
processedTSet.direct().forEach((ApplyFunc<Iterator<Integer>>) data -> {
while (data.hasNext()) {
System.out.println(data.next());
}
});
}
if (REDUCE_WINDOW) {
WindowComputeTSet<Integer> winTSet = link.timeWindow(2, TimeUnit.MILLISECONDS);
WindowComputeTSet<Integer> localReducedTSet = winTSet.aggregate((AggregateFunc<Integer>) Integer::sum);
localReducedTSet.direct().forEach(x -> System.out.println(x));
// link.countWindow().reduce(a,b-> a + b)
}
}
// Runs the entire TSet graph
env.run();
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class TSetCheckptExample method execute.
@Override
public void execute(WorkerEnvironment workerEnvironment) {
BatchChkPntEnvironment env = TSetEnvironment.initCheckpointing(workerEnvironment);
LOG.info(String.format("Hello from worker %d", env.getWorkerID()));
SourceTSet<Integer> sourceX = env.createSource(new SourceFunc<Integer>() {
private int count = 0;
@Override
public boolean hasNext() {
return count < 10000;
}
@Override
public Integer next() {
return count++;
}
}, 4);
long t1 = System.currentTimeMillis();
ComputeTSet<Object> twoComputes = sourceX.direct().compute((itr, c) -> {
itr.forEachRemaining(i -> {
c.collect(i * 5);
});
}).direct().compute((itr, c) -> {
itr.forEachRemaining(i -> {
c.collect((int) i + 2);
});
});
LOG.info("Time for two computes : " + (System.currentTimeMillis() - t1));
t1 = System.currentTimeMillis();
PersistedTSet<Object> persist = twoComputes.persist();
LOG.info("Time for persist : " + (System.currentTimeMillis() - t1) / 1000);
// When persist() is called, twister2 performs all the computations/communication
// upto this point and persists the result into the disk.
// This makes previous data garbage collectible and frees some memory.
// If persist() is called in a checkpointing enabled job, this will create
// a snapshot at this point and will start straightaway from this point if the
// job is restarted.
// Similar to CachedTSets, PersistedTSets can be added as inputs for other TSets and
// operations
persist.reduce((i1, i2) -> {
return (int) i1 + (int) i2;
}).forEach(i -> {
LOG.info("SUM=" + i);
});
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class SDirectExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
StreamingEnvironment env = TSetEnvironment.initStreaming(workerEnv);
SSourceTSet<Integer> src = dummySource(env, COUNT, PARALLELISM).setName("src");
SDirectTLink<Integer> link = src.direct().setName("dir");
link.map(i -> i * 2).setName("map").direct().forEach(i -> LOG.info("m" + i.toString()));
link.flatmap((i, c) -> c.collect("fm" + i)).setName("flatmap").direct().forEach(i -> LOG.info(i.toString()));
link.compute(i -> i + "C").setName("compute").direct().forEach(i -> LOG.info(i));
link.mapToTuple(i -> new Tuple<>(i, i.toString())).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
link.compute((input, output) -> output.collect(input + "DD")).setName("computec").direct().forEach(s -> LOG.info(s.toString()));
// Runs the entire TSet graph
env.run();
}
Aggregations