Examples with BatchTSet edu.iu.dsc.tws.api.tset.sets.batch.BatchTSet used on opensource projects

Example 1 with BatchTSet

use of edu.iu.dsc.tws.api.tset.sets.batch.BatchTSet in project beam by apache.

the class BeamBatchWorker method executePipeline.

public void executePipeline(BatchTSetEnvironment env) {
    Map<String, CachedTSet> sideInputTSets = new HashMap<>();
    for (Map.Entry<String, BatchTSet<?>> sides : sideInputDataSets.entrySet()) {
        BatchTSet<?> sideTSet = sides.getValue();
        addInputs((BaseTSet) sideTSet, sideInputTSets);
        CachedTSet tempCache = (CachedTSet) sideTSet.cache();
        sideInputTSets.put(sides.getKey(), tempCache);
    }
    for (TSet leaf : leaves) {
        SinkTSet sinkTSet = (SinkTSet) leaf.direct().sink(new Twister2SinkFunction());
        addInputs(sinkTSet, sideInputTSets);
        eval(env, sinkTSet);
    }
}

Example 2 with BatchTSet

use of edu.iu.dsc.tws.api.tset.sets.batch.BatchTSet in project beam by apache.

the class BeamBatchWorker method setupTSets.

/**
 * Extract the sideInput TSets and the Leaves from the graph.
 *
 * @param env BatchTSetEnvironment which contains the tSet Graph
 * @param sideInputIds the id's of the side Inputs
 * @param leaveIds the id's of the leaf
 */
private void setupTSets(BatchTSetEnvironment env, Map<String, String> sideInputIds, Set<String> leaveIds) {
    sideInputDataSets = new LinkedHashMap<>();
    leaves = new HashSet<>();
    // reset sources, so that the graph does not have two source objects
    // created during deserialization
    Set<BuildableTSet> newSources = new HashSet<>();
    for (BuildableTSet source : env.getGraph().getSources()) {
        newSources.add((BuildableTSet) env.getGraph().getNodeById(source.getId()));
    }
    env.getGraph().setSources(newSources);
    for (Map.Entry<String, String> entry : sideInputIds.entrySet()) {
        BatchTSet curr = (BatchTSet) env.getGraph().getNodeById(entry.getValue());
        sideInputDataSets.put(entry.getKey(), curr);
    }
    for (String leaveId : leaveIds) {
        leaves.add((TSet) env.getGraph().getNodeById(leaveId));
    }
}