Examples with Row edu.iu.dsc.tws.common.table.Row used on opensource projects

Example 1 with Row

use of edu.iu.dsc.tws.common.table.Row in project twister2 by DSC-SPIDAL.

the class STPartition method isComplete.

@Override
public boolean isComplete() {
    for (Map.Entry<Integer, Queue<Table>> e : inputs.entrySet()) {
        if (e.getValue().isEmpty()) {
            continue;
        }
        // partition the table, default
        Table t = e.getValue().poll();
        List<ArrowColumn> columns = t.getColumns();
        ArrowColumn col = columns.get(indexes[0]);
        for (int i = 0; i < col.getVector().getValueCount(); i++) {
            Row row = new OneRow(col.get(i));
            int target = selector.next(e.getKey(), row);
            TableBuilder builder = partitionedTables.get(target);
            if (builder == null) {
                builder = new ArrowTableBuilder(schema, allocator);
                this.partitionedTables.put(target, builder);
            }
            for (int j = 0; j < columns.size(); j++) {
                builder.getColumns().get(j).addValue(columns.get(j).get(i));
            }
        }
    }
    if (finished) {
        for (Map.Entry<Integer, TableBuilder> e : partitionedTables.entrySet()) {
            Table t = e.getValue().build();
            allToAll.insert(t, e.getKey());
        }
        // clear the tables, so we won't build the tables again
        partitionedTables.clear();
        for (int s : finishedSources) {
            allToAll.finish(s);
        }
        // clear so, we won't call finish again
        finishedSources.clear();
        return allToAll.isComplete();
    }
    return false;
}

Example 2 with Row

use of edu.iu.dsc.tws.common.table.Row in project twister2 by DSC-SPIDAL.

the class PartitionExample method execute.

@Override
public void execute(WorkerEnvironment workerEnvironment) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnvironment);
    List<TField> fieldList = new ArrayList<>();
    fieldList.add(new TField("first", MessageTypes.INTEGER));
    fieldList.add(new TField("second", MessageTypes.DOUBLE));
    RowSourceTSet src = env.createRowSource("row", new SourceFunc<Row>() {

        private int count = 0;

        @Override
        public boolean hasNext() {
            return count++ < 1000;
        }

        @Override
        public Row next() {
            return new TwoRow(1, 4.1);
        }
    }, 4).withSchema(new RowSchema(fieldList));
    BatchRowTLink partition = src.partition(new PartitionFunc<Row>() {

        private List<Integer> targets;

        private Random random;

        private int c = 0;

        private Map<Integer, Integer> counts = new HashMap<>();

        @Override
        public void prepare(Set<Integer> sources, Set<Integer> destinations) {
            targets = new ArrayList<>(destinations);
            random = new Random();
            for (int t : targets) {
                counts.put(t, 0);
            }
        }

        @Override
        public int partition(int sourceIndex, Row val) {
            int index = random.nextInt(targets.size());
            int count = counts.get(index);
            counts.put(index, count + 1);
            c++;
            if (c == 1000) {
                LOG.info("COUNTS " + counts);
            }
            return targets.get(index);
        }
    }, 4, 0);
    partition.forEach(new ApplyFunc<Row>() {

        private TSetContext ctx;

        private int count;

        @Override
        public void prepare(TSetContext context) {
            ctx = context;
        }

        @Override
        public void apply(Row data) {
            LOG.info(ctx.getIndex() + " Data " + data.get(0) + ", " + data.get(1) + ", count " + count++);
        }
    });
}

Example 3 with Row

use of edu.iu.dsc.tws.common.table.Row in project twister2 by DSC-SPIDAL.

the class RowBatchTLinkImpl method lazyPersist.

public StorableTBase<Row> lazyPersist() {
    DiskPersistSingleSink<Row> diskPersistSingleSink = new DiskPersistSingleSink<>(this.getId());
    RowPersistedTSet persistedTSet = new RowPersistedTSet(getTSetEnv(), diskPersistSingleSink, getTargetParallelism(), (RowSchema) getSchema());
    addChildToGraph(persistedTSet);
    return persistedTSet;
}

Example 4 with Row

use of edu.iu.dsc.tws.common.table.Row in project twister2 by DSC-SPIDAL.

the class RowMapCompute method compute.

@Override
public void compute(Iterator<Row> input, RecordCollector<Row> output) {
    while (input.hasNext()) {
        Row next = input.next();
        Row out = mapFn.map(next);
        output.collect(out);
    }
}

Example 5 with Row

use of edu.iu.dsc.tws.common.table.Row in project twister2 by DSC-SPIDAL.

the class RowBatchTLinkImpl method lazyCache.

public StorableTBase<Row> lazyCache() {
    RowCachedTSet cacheTSet = new RowCachedTSet(getTSetEnv(), new CacheSingleSink<Row>(), getTargetParallelism(), (RowSchema) getSchema());
    addChildToGraph(cacheTSet);
    return cacheTSet;
}