use of edu.iu.dsc.tws.api.tset.link.batch.BatchRowTLink in project twister2 by DSC-SPIDAL.
the class PartitionExample method execute.
@Override
public void execute(WorkerEnvironment workerEnvironment) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnvironment);
List<TField> fieldList = new ArrayList<>();
fieldList.add(new TField("first", MessageTypes.INTEGER));
fieldList.add(new TField("second", MessageTypes.DOUBLE));
RowSourceTSet src = env.createRowSource("row", new SourceFunc<Row>() {
private int count = 0;
@Override
public boolean hasNext() {
return count++ < 1000;
}
@Override
public Row next() {
return new TwoRow(1, 4.1);
}
}, 4).withSchema(new RowSchema(fieldList));
BatchRowTLink partition = src.partition(new PartitionFunc<Row>() {
private List<Integer> targets;
private Random random;
private int c = 0;
private Map<Integer, Integer> counts = new HashMap<>();
@Override
public void prepare(Set<Integer> sources, Set<Integer> destinations) {
targets = new ArrayList<>(destinations);
random = new Random();
for (int t : targets) {
counts.put(t, 0);
}
}
@Override
public int partition(int sourceIndex, Row val) {
int index = random.nextInt(targets.size());
int count = counts.get(index);
counts.put(index, count + 1);
c++;
if (c == 1000) {
LOG.info("COUNTS " + counts);
}
return targets.get(index);
}
}, 4, 0);
partition.forEach(new ApplyFunc<Row>() {
private TSetContext ctx;
private int count;
@Override
public void prepare(TSetContext context) {
ctx = context;
}
@Override
public void apply(Row data) {
LOG.info(ctx.getIndex() + " Data " + data.get(0) + ", " + data.get(1) + ", count " + count++);
}
});
}
use of edu.iu.dsc.tws.api.tset.link.batch.BatchRowTLink in project twister2 by DSC-SPIDAL.
the class TBaseGraph method conditionalBFS.
/**
* Same case as above this but can handle multiple roots
*
* @return build seq
*/
private Set<TBase> conditionalBFS(Collection<? extends BuildableTSet> roots, AdjNodesExtractor adjNodesExtractor) {
Set<TBase> buildSequence = new LinkedHashSet<>();
Set<TBase> visited = new HashSet<>();
Deque<TBase> queue = new LinkedList<>();
for (TBase root : roots) {
visited.add(root);
queue.add(root);
while (queue.size() != 0) {
TBase t = queue.poll();
for (TBase adj : adjNodesExtractor.extract(t)) {
if (!visited.contains(adj)) {
visited.add(adj);
queue.add(adj);
}
}
if (t instanceof TLink && !allAdjNodesTraversed((TLink) t, buildSequence, adjNodesExtractor)) {
queue.add(t);
continue;
} else if (t instanceof BatchRowTLink && !allAdjNodesTraversed((BatchRowTLink) t, buildSequence, adjNodesExtractor)) {
queue.add(t);
continue;
}
buildSequence.add(t);
}
}
return buildSequence;
}
Aggregations