Examples with TSet edu.iu.dsc.tws.api.tset.sets.TSet used on opensource projects

Example 1 with TSet

use of edu.iu.dsc.tws.api.tset.sets.TSet in project beam by apache.

the class FlattenTranslatorBatch method translateNode.

@Override
public void translateNode(Flatten.PCollections<T> transform, Twister2BatchTranslationContext context) {
    Collection<PCollection<?>> pcs = context.getInputs().values();
    List<BatchTSetImpl<WindowedValue<T>>> tSets = new ArrayList<>();
    BatchTSetImpl<WindowedValue<T>> unionTSet;
    if (pcs.isEmpty()) {
        final TSetEnvironment tsetEnv = context.getEnvironment();
        unionTSet = ((BatchTSetEnvironment) tsetEnv).createSource(new Twister2EmptySource(), context.getOptions().getParallelism());
    } else {
        for (PValue pc : pcs) {
            BatchTSetImpl<WindowedValue<T>> curr = context.getInputDataSet(pc);
            tSets.add(curr);
        }
        BatchTSetImpl<WindowedValue<T>> first = tSets.remove(0);
        Collection<TSet<WindowedValue<T>>> others = new ArrayList<>();
        others.addAll(tSets);
        if (tSets.size() > 0) {
            unionTSet = first.union(others);
        } else {
            unionTSet = first;
        }
    }
    context.setOutputDataSet(context.getOutput(transform), unionTSet);
}

Example 2 with TSet

use of edu.iu.dsc.tws.api.tset.sets.TSet in project beam by apache.

the class BeamBatchWorker method executePipeline.

public void executePipeline(BatchTSetEnvironment env) {
    Map<String, CachedTSet> sideInputTSets = new HashMap<>();
    for (Map.Entry<String, BatchTSet<?>> sides : sideInputDataSets.entrySet()) {
        BatchTSet<?> sideTSet = sides.getValue();
        addInputs((BaseTSet) sideTSet, sideInputTSets);
        CachedTSet tempCache = (CachedTSet) sideTSet.cache();
        sideInputTSets.put(sides.getKey(), tempCache);
    }
    for (TSet leaf : leaves) {
        SinkTSet sinkTSet = (SinkTSet) leaf.direct().sink(new Twister2SinkFunction());
        addInputs(sinkTSet, sideInputTSets);
        eval(env, sinkTSet);
    }
}