Examples with ArrowTableBuilder edu.iu.dsc.tws.common.table.ArrowTableBuilder used on opensource projects

Example 1 with ArrowTableBuilder

use of edu.iu.dsc.tws.common.table.ArrowTableBuilder in project twister2 by DSC-SPIDAL.

the class STPartition method isComplete.

@Override
public boolean isComplete() {
    for (Map.Entry<Integer, Queue<Table>> e : inputs.entrySet()) {
        if (e.getValue().isEmpty()) {
            continue;
        }
        // partition the table, default
        Table t = e.getValue().poll();
        List<ArrowColumn> columns = t.getColumns();
        ArrowColumn col = columns.get(indexes[0]);
        for (int i = 0; i < col.getVector().getValueCount(); i++) {
            Row row = new OneRow(col.get(i));
            int target = selector.next(e.getKey(), row);
            TableBuilder builder = partitionedTables.get(target);
            if (builder == null) {
                builder = new ArrowTableBuilder(schema, allocator);
                this.partitionedTables.put(target, builder);
            }
            for (int j = 0; j < columns.size(); j++) {
                builder.getColumns().get(j).addValue(columns.get(j).get(i));
            }
        }
    }
    if (finished) {
        for (Map.Entry<Integer, TableBuilder> e : partitionedTables.entrySet()) {
            Table t = e.getValue().build();
            allToAll.insert(t, e.getKey());
        }
        // clear the tables, so we won't build the tables again
        partitionedTables.clear();
        for (int s : finishedSources) {
            allToAll.finish(s);
        }
        // clear so, we won't call finish again
        finishedSources.clear();
        return allToAll.isComplete();
    }
    return false;
}

Example 2 with ArrowTableBuilder

use of edu.iu.dsc.tws.common.table.ArrowTableBuilder in project twister2 by DSC-SPIDAL.

the class RowItrComputeCollectorOp method prepare.

@Override
public void prepare(Config cfg, TaskContext ctx) {
    super.prepare(cfg, ctx);
    runtime = WorkerEnvironment.getSharedValue(TableRuntime.TABLE_RUNTIME_CONF, TableRuntime.class);
    if (runtime == null) {
        throw new Twister2RuntimeException("Table runtime must be set");
    }
    schema = (RowSchema) ctx.getConfig(TSetConstants.OUTPUT_SCHEMA_KEY);
    tableMaxSize = cfg.getLongValue("twister2.table.max.size", tableMaxSize);
    builder = new ArrowTableBuilder(schema.toArrowSchema(), runtime.getRootAllocator());
    collectorImp = new CollectorImp();
}

Example 3 with ArrowTableBuilder

use of edu.iu.dsc.tws.common.table.ArrowTableBuilder in project twister2 by DSC-SPIDAL.

the class RowSourceOp method execute.

@Override
public void execute() {
    if (source.hasNext()) {
        // todo:: change source function to accept a row, so we don't have to allocate new
        // row every time
        Row tuple = source.next();
        builder.add(tuple);
        if (builder.currentSize() > tableMaxSize) {
            multiEdgeOpAdapter.writeToEdges(builder.build());
            builder = new ArrowTableBuilder(schema.toArrowSchema(), runtime.getRootAllocator());
        }
    } else {
        multiEdgeOpAdapter.writeToEdges(builder.build());
        builder = null;
        multiEdgeOpAdapter.writeEndToEdges();
    }
}

Example 4 with ArrowTableBuilder

use of edu.iu.dsc.tws.common.table.ArrowTableBuilder in project twister2 by DSC-SPIDAL.

the class RowSourceOp method prepare.

@Override
public void prepare(Config cfg, TaskContext ctx) {
    super.prepare(cfg, ctx);
    runtime = WorkerEnvironment.getSharedValue(TableRuntime.TABLE_RUNTIME_CONF, TableRuntime.class);
    if (runtime == null) {
        throw new Twister2RuntimeException("Table runtime must be set");
    }
    schema = (RowSchema) ctx.getConfig(TSetConstants.OUTPUT_SCHEMA_KEY);
    tableMaxSize = cfg.getLongValue("twister2.table.max.size", tableMaxSize);
    builder = new ArrowTableBuilder(schema.toArrowSchema(), runtime.getRootAllocator());
}

Example 5 with ArrowTableBuilder

use of edu.iu.dsc.tws.common.table.ArrowTableBuilder in project twister2 by DSC-SPIDAL.

the class RowComupteCollectorOp method prepare.

@Override
public void prepare(Config cfg, TaskContext ctx) {
    super.prepare(cfg, ctx);
    runtime = WorkerEnvironment.getSharedValue(TableRuntime.TABLE_RUNTIME_CONF, TableRuntime.class);
    if (runtime == null) {
        throw new Twister2RuntimeException("Table runtime must be set");
    }
    schema = (RowSchema) ctx.getConfig(TSetConstants.OUTPUT_SCHEMA_KEY);
    tableMaxSize = cfg.getLongValue("twister2.table.max.size", tableMaxSize);
    builder = new ArrowTableBuilder(schema.toArrowSchema(), runtime.getRootAllocator());
    collectorImp = new CollectorImp();
}