use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class KeyedOperationsExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
KeyedSourceTSet<String, Integer> kSource = dummyKeyedSource(env, COUNT, PARALLELISM);
KeyedDirectTLink<String, Integer> kDirect = kSource.keyedDirect();
kDirect.forEach(i -> LOG.info("d_fe: " + i.toString()));
KeyedCachedTSet<String, Integer> cache = kDirect.cache();
cache.keyedDirect().forEach(i -> LOG.info("c_d_fe: " + i.toString()));
cache.keyedReduce(Integer::sum).forEach(i -> LOG.info("c_r_fe: " + i.toString()));
cache.keyedGather().forEach((ApplyFunc<Tuple<String, Iterator<Integer>>>) data -> {
StringBuilder sb = new StringBuilder();
sb.append("c_g_fe: key ").append(data.getKey()).append("->");
while (data.getValue().hasNext()) {
sb.append(" ").append(data.getValue().next());
}
LOG.info(sb.toString());
});
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class JoinExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
int para = 2;
int workerID = env.getWorkerID();
SourceTSet<Integer> src0 = dummySource(env, COUNT, para).setName("src0");
KeyedTSet<Integer, Integer> left = src0.mapToTuple(i -> new Tuple<>(i % 2, i)).setName("left");
left.keyedDirect().forEach(i -> LOG.info(workerID + "L " + i.toString()));
SourceTSet<Integer> src1 = dummySource(env, COUNT, para).setName("src1");
KeyedTSet<Integer, Integer> right = src1.mapToTuple(i -> new Tuple<>(i % 2, i)).setName("right");
right.keyedDirect().forEach(i -> LOG.info(workerID + "R " + i.toString()));
JoinTLink<Integer, Integer, Integer> join = left.join(right, CommunicationContext.JoinType.INNER, Integer::compareTo).setName("join");
join.forEach(t -> LOG.info(workerID + "out: " + t.toString()));
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class GatherExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
int start = env.getWorkerID() * 100;
SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
GatherTLink<Integer> gather = src.gather();
LOG.info("test foreach");
gather.forEach(i -> LOG.info("foreach: " + i));
LOG.info("test map");
gather.map(i -> i.toString() + "$$").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("map: " + s));
LOG.info("test compute");
gather.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
int sum = 0;
while (input.hasNext()) {
sum += input.next().getValue();
}
return "sum=" + sum;
}).withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("compute: " + s));
LOG.info("test computec");
gather.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
int sum = 0;
while (input.hasNext()) {
sum += input.next().getValue();
}
output.collect("sum=" + sum);
}).withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("computec: " + s));
gather.mapToTuple(i -> new Tuple<>(i % 2, i)).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class KGatherExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
int start = env.getWorkerID() * 100;
SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
KeyedGatherTLink<Integer, Integer> klink = src.mapToTuple(i -> new Tuple<>(i % 10, i)).keyedGather();
LOG.info("test foreach");
klink.forEach((ApplyFunc<Tuple<Integer, Iterator<Integer>>>) data -> LOG.info(data.getKey() + " -> " + iterToString(data.getValue())));
LOG.info("test map");
klink.map((MapFunc<Tuple<Integer, Iterator<Integer>>, String>) input -> {
int s = 0;
while (input.getValue().hasNext()) {
s += input.getValue().next();
}
return input.getKey() + " -> " + s;
}).direct().forEach(s -> LOG.info("map: " + s));
LOG.info("test compute");
klink.compute((ComputeFunc<Iterator<Tuple<Integer, Iterator<Integer>>>, String>) input -> {
StringBuilder s = new StringBuilder();
while (input.hasNext()) {
Tuple<Integer, Iterator<Integer>> next = input.next();
s.append(" [").append(next.getKey()).append(" -> ").append(iterToString(next.getValue())).append("] ");
}
return s.toString();
}).direct().forEach(s -> LOG.info("compute: concat " + s));
LOG.info("test computec");
klink.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Iterator<Integer>>>, String>) (input, output) -> {
while (input.hasNext()) {
Tuple<Integer, Iterator<Integer>> next = input.next();
output.collect(next.getKey() + " -> " + iterToString(next.getValue()));
}
}).direct().forEach(s -> LOG.info("computec: " + s));
// Test byte[] key value pairs for KeyedGather
SourceTSet<String> srcString = dummyStringSource(env, 25, PARALLELISM);
KeyedGatherTLink<byte[], Integer> keyedGatherLink = srcString.mapToTuple(s -> new Tuple<>(s.getBytes(), 1)).keyedGather();
LOG.info("test foreach");
keyedGatherLink.forEach((ApplyFunc<Tuple<byte[], Iterator<Integer>>>) data -> LOG.info(new String(data.getKey()) + " -> " + iterToString(data.getValue())));
}
use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.
the class KPartitionExample method execute.
@Override
public void execute(WorkerEnvironment workerEnv) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
int start = env.getWorkerID() * 100;
SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
KeyedPartitionTLink<Integer, Integer> klink = src.mapToTuple(i -> new Tuple<>(i % 10, i)).keyedPartition(new LoadBalancePartitioner<>());
LOG.info("test foreach");
klink.forEach(t -> LOG.info(t.getKey() + "_" + t.getValue()));
LOG.info("test map");
klink.map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
LOG.info("test compute");
klink.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
StringBuilder s = new StringBuilder();
while (input.hasNext()) {
s.append(input.next().toString()).append(" ");
}
return s.toString();
}).direct().forEach(s -> LOG.info("compute: concat " + s));
LOG.info("test computec");
klink.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
while (input.hasNext()) {
output.collect(input.next().toString());
}
}).direct().forEach(s -> LOG.info("computec: " + s));
}
Aggregations