use of edu.iu.dsc.tws.api.tset.schema.RowSchema in project twister2 by DSC-SPIDAL.
the class PartitionExample method execute.
@Override
public void execute(WorkerEnvironment workerEnvironment) {
BatchEnvironment env = TSetEnvironment.initBatch(workerEnvironment);
List<TField> fieldList = new ArrayList<>();
fieldList.add(new TField("first", MessageTypes.INTEGER));
fieldList.add(new TField("second", MessageTypes.DOUBLE));
RowSourceTSet src = env.createRowSource("row", new SourceFunc<Row>() {
private int count = 0;
@Override
public boolean hasNext() {
return count++ < 1000;
}
@Override
public Row next() {
return new TwoRow(1, 4.1);
}
}, 4).withSchema(new RowSchema(fieldList));
BatchRowTLink partition = src.partition(new PartitionFunc<Row>() {
private List<Integer> targets;
private Random random;
private int c = 0;
private Map<Integer, Integer> counts = new HashMap<>();
@Override
public void prepare(Set<Integer> sources, Set<Integer> destinations) {
targets = new ArrayList<>(destinations);
random = new Random();
for (int t : targets) {
counts.put(t, 0);
}
}
@Override
public int partition(int sourceIndex, Row val) {
int index = random.nextInt(targets.size());
int count = counts.get(index);
counts.put(index, count + 1);
c++;
if (c == 1000) {
LOG.info("COUNTS " + counts);
}
return targets.get(index);
}
}, 4, 0);
partition.forEach(new ApplyFunc<Row>() {
private TSetContext ctx;
private int count;
@Override
public void prepare(TSetContext context) {
ctx = context;
}
@Override
public void apply(Row data) {
LOG.info(ctx.getIndex() + " Data " + data.get(0) + ", " + data.get(1) + ", count " + count++);
}
});
}
use of edu.iu.dsc.tws.api.tset.schema.RowSchema in project twister2 by DSC-SPIDAL.
the class RowBatchTLinkImpl method compute.
protected RowComputeTSet compute(String n, ComputeCollectorFunc<Iterator<Row>, Row> computeFunction) {
RowComputeTSet set;
if (n != null && !n.isEmpty()) {
set = new RowComputeTSet(getTSetEnv(), n, computeFunction, getTargetParallelism(), (RowSchema) getSchema(), false);
} else {
set = new RowComputeTSet(getTSetEnv(), computeFunction, getTargetParallelism(), (RowSchema) getSchema(), false);
}
addChildToGraph(set);
return set;
}
use of edu.iu.dsc.tws.api.tset.schema.RowSchema in project twister2 by DSC-SPIDAL.
the class RowPartitionTLink method compute.
protected RowComputeTSet compute(String n, ComputeCollectorFunc<Iterator<Row>, Row> computeFunction) {
RowComputeTSet set;
if (n != null && !n.isEmpty()) {
set = new RowComputeTSet(getTSetEnv(), n, computeFunction, getTargetParallelism(), (RowSchema) getSchema(), true);
} else {
set = new RowComputeTSet(getTSetEnv(), computeFunction, getTargetParallelism(), (RowSchema) getSchema(), true);
}
addChildToGraph(set);
return set;
}
Aggregations