use of edu.iu.dsc.tws.tset.env.TSetEnvironment in project beam by apache.
the class FlattenTranslatorBatch method translateNode.
@Override
public void translateNode(Flatten.PCollections<T> transform, Twister2BatchTranslationContext context) {
Collection<PCollection<?>> pcs = context.getInputs().values();
List<BatchTSetImpl<WindowedValue<T>>> tSets = new ArrayList<>();
BatchTSetImpl<WindowedValue<T>> unionTSet;
if (pcs.isEmpty()) {
final TSetEnvironment tsetEnv = context.getEnvironment();
unionTSet = ((BatchTSetEnvironment) tsetEnv).createSource(new Twister2EmptySource(), context.getOptions().getParallelism());
} else {
for (PValue pc : pcs) {
BatchTSetImpl<WindowedValue<T>> curr = context.getInputDataSet(pc);
tSets.add(curr);
}
BatchTSetImpl<WindowedValue<T>> first = tSets.remove(0);
Collection<TSet<WindowedValue<T>>> others = new ArrayList<>();
others.addAll(tSets);
if (tSets.size() > 0) {
unionTSet = first.union(others);
} else {
unionTSet = first;
}
}
context.setOutputDataSet(context.getOutput(transform), unionTSet);
}
use of edu.iu.dsc.tws.tset.env.TSetEnvironment in project twister2 by DSC-SPIDAL.
the class ReadSourceTranslatorBatch method translateNode.
@Override
public void translateNode(Read.Bounded<T> transform, Twister2BatchTranslationContext context) {
BoundedSource<T> boundedSource = transform.getSource();
Twister2BoundedSource<T> twister2BoundedSource = new Twister2BoundedSource<T>(boundedSource, context, context.getOptions());
final TSetEnvironment tsetEnv = context.getEnvironment();
// TODO: need to set paralliem value
SourceTSet<WindowedValue<T>> sourceTSet = ((BatchEnvironment) tsetEnv).createSource(twister2BoundedSource, 1);
PCollection<T> output = context.getOutput(transform);
context.setOutputDataSet(output, sourceTSet);
}
use of edu.iu.dsc.tws.tset.env.TSetEnvironment in project beam by apache.
the class ReadSourceTranslatorBatch method translateNode.
@Override
public void translateNode(SplittableParDo.PrimitiveBoundedRead<T> transform, Twister2BatchTranslationContext context) {
BoundedSource<T> boundedSource = transform.getSource();
Twister2BoundedSource<T> twister2BoundedSource = new Twister2BoundedSource<T>(boundedSource, context, context.getOptions());
final TSetEnvironment tsetEnv = context.getEnvironment();
SourceTSet<WindowedValue<T>> sourceTSet = ((BatchTSetEnvironment) tsetEnv).createSource(twister2BoundedSource, context.getOptions().getParallelism());
PCollection<T> output = context.getOutput(transform);
context.setOutputDataSet(output, sourceTSet);
}
use of edu.iu.dsc.tws.tset.env.TSetEnvironment in project beam by apache.
the class ImpulseTranslatorBatch method translateNode.
@Override
public void translateNode(Impulse transform, Twister2BatchTranslationContext context) {
final TSetEnvironment tsetEnv = context.getEnvironment();
SourceTSet<WindowedValue<byte[]>> sourceTSet = ((BatchTSetEnvironment) tsetEnv).createSource(new ImpulseSource(), 1);
PCollection<byte[]> output = context.getOutput(transform);
context.setOutputDataSet(output, sourceTSet);
}
Aggregations