Search in sources :

Example 1 with BatchRowTLink

use of edu.iu.dsc.tws.api.tset.link.batch.BatchRowTLink in project twister2 by DSC-SPIDAL.

the class PartitionExample method execute.

@Override
public void execute(WorkerEnvironment workerEnvironment) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnvironment);
    List<TField> fieldList = new ArrayList<>();
    fieldList.add(new TField("first", MessageTypes.INTEGER));
    fieldList.add(new TField("second", MessageTypes.DOUBLE));
    RowSourceTSet src = env.createRowSource("row", new SourceFunc<Row>() {

        private int count = 0;

        @Override
        public boolean hasNext() {
            return count++ < 1000;
        }

        @Override
        public Row next() {
            return new TwoRow(1, 4.1);
        }
    }, 4).withSchema(new RowSchema(fieldList));
    BatchRowTLink partition = src.partition(new PartitionFunc<Row>() {

        private List<Integer> targets;

        private Random random;

        private int c = 0;

        private Map<Integer, Integer> counts = new HashMap<>();

        @Override
        public void prepare(Set<Integer> sources, Set<Integer> destinations) {
            targets = new ArrayList<>(destinations);
            random = new Random();
            for (int t : targets) {
                counts.put(t, 0);
            }
        }

        @Override
        public int partition(int sourceIndex, Row val) {
            int index = random.nextInt(targets.size());
            int count = counts.get(index);
            counts.put(index, count + 1);
            c++;
            if (c == 1000) {
                LOG.info("COUNTS " + counts);
            }
            return targets.get(index);
        }
    }, 4, 0);
    partition.forEach(new ApplyFunc<Row>() {

        private TSetContext ctx;

        private int count;

        @Override
        public void prepare(TSetContext context) {
            ctx = context;
        }

        @Override
        public void apply(Row data) {
            LOG.info(ctx.getIndex() + " Data " + data.get(0) + ", " + data.get(1) + ", count " + count++);
        }
    });
}
Also used : RowSchema(edu.iu.dsc.tws.api.tset.schema.RowSchema) RowSourceTSet(edu.iu.dsc.tws.tset.sets.batch.row.RowSourceTSet) TField(edu.iu.dsc.tws.common.table.TField) HashMap(java.util.HashMap) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) ArrayList(java.util.ArrayList) SourceFunc(edu.iu.dsc.tws.api.tset.fn.SourceFunc) TSetContext(edu.iu.dsc.tws.api.tset.TSetContext) Random(java.util.Random) TwoRow(edu.iu.dsc.tws.common.table.TwoRow) BatchRowTLink(edu.iu.dsc.tws.api.tset.link.batch.BatchRowTLink) Row(edu.iu.dsc.tws.common.table.Row) TwoRow(edu.iu.dsc.tws.common.table.TwoRow)

Example 2 with BatchRowTLink

use of edu.iu.dsc.tws.api.tset.link.batch.BatchRowTLink in project twister2 by DSC-SPIDAL.

the class TBaseGraph method conditionalBFS.

/**
 * Same case as above this but can handle multiple roots
 *
 * @return build seq
 */
private Set<TBase> conditionalBFS(Collection<? extends BuildableTSet> roots, AdjNodesExtractor adjNodesExtractor) {
    Set<TBase> buildSequence = new LinkedHashSet<>();
    Set<TBase> visited = new HashSet<>();
    Deque<TBase> queue = new LinkedList<>();
    for (TBase root : roots) {
        visited.add(root);
        queue.add(root);
        while (queue.size() != 0) {
            TBase t = queue.poll();
            for (TBase adj : adjNodesExtractor.extract(t)) {
                if (!visited.contains(adj)) {
                    visited.add(adj);
                    queue.add(adj);
                }
            }
            if (t instanceof TLink && !allAdjNodesTraversed((TLink) t, buildSequence, adjNodesExtractor)) {
                queue.add(t);
                continue;
            } else if (t instanceof BatchRowTLink && !allAdjNodesTraversed((BatchRowTLink) t, buildSequence, adjNodesExtractor)) {
                queue.add(t);
                continue;
            }
            buildSequence.add(t);
        }
    }
    return buildSequence;
}
Also used : LinkedHashSet(java.util.LinkedHashSet) TBase(edu.iu.dsc.tws.api.tset.TBase) BatchRowTLink(edu.iu.dsc.tws.api.tset.link.batch.BatchRowTLink) TLink(edu.iu.dsc.tws.api.tset.link.TLink) BatchRowTLink(edu.iu.dsc.tws.api.tset.link.batch.BatchRowTLink) LinkedList(java.util.LinkedList) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet)

Aggregations

BatchRowTLink (edu.iu.dsc.tws.api.tset.link.batch.BatchRowTLink)2 TBase (edu.iu.dsc.tws.api.tset.TBase)1 TSetContext (edu.iu.dsc.tws.api.tset.TSetContext)1 SourceFunc (edu.iu.dsc.tws.api.tset.fn.SourceFunc)1 TLink (edu.iu.dsc.tws.api.tset.link.TLink)1 RowSchema (edu.iu.dsc.tws.api.tset.schema.RowSchema)1 Row (edu.iu.dsc.tws.common.table.Row)1 TField (edu.iu.dsc.tws.common.table.TField)1 TwoRow (edu.iu.dsc.tws.common.table.TwoRow)1 BatchEnvironment (edu.iu.dsc.tws.tset.env.BatchEnvironment)1 RowSourceTSet (edu.iu.dsc.tws.tset.sets.batch.row.RowSourceTSet)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 LinkedHashSet (java.util.LinkedHashSet)1 LinkedList (java.util.LinkedList)1 Random (java.util.Random)1