Search in sources :

Example 1 with ArrowTable

use of edu.iu.dsc.tws.common.table.arrow.ArrowTable in project twister2 by DSC-SPIDAL.

the class BTAllToAll method execute.

@Override
public void execute(Config config, JobAPI.Job job, IWorkerController workerController, IPersistentVolume persistentVolume, IVolatileVolume volatileVolume) {
    this.jobParameters = JobParameters.build(config);
    // create a worker environment
    this.wEnv = WorkerEnvironment.init(config, job, workerController, persistentVolume, volatileVolume);
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), wEnv).withFairDistribution();
    RootAllocator rootAllocator = new RootAllocator();
    IntVector intVector = new IntVector("fist", rootAllocator);
    Float8Vector float8Vector = new Float8Vector("second", rootAllocator);
    for (int i = 0; i < 1000; i++) {
        intVector.setSafe(i, i);
        float8Vector.setSafe(i, i);
    }
    intVector.setValueCount(1000);
    float8Vector.setValueCount(1000);
    List<Field> fieldList = Arrays.asList(intVector.getField(), float8Vector.getField());
    Schema schema = new Schema(fieldList);
    Table t = new ArrowTable(schema, Arrays.asList(new FieldVector[] { intVector, float8Vector }));
    allToAll = new ArrowAllToAll(wEnv.getConfig(), wEnv.getWorkerController(), logicalPlanBuilder.getSources(), logicalPlanBuilder.getTargets(), logicalPlanBuilder.build(), wEnv.getCommunicator().nextEdge(), new ArrowReceiver(), schema, rootAllocator);
    for (int i : logicalPlanBuilder.getTargets()) {
        allToAll.insert(t, i);
    }
    for (int s : logicalPlanBuilder.getSourcesOnThisWorker()) {
        allToAll.finish(s);
    }
    while (!allToAll.isComplete()) {
    // wait
    }
}
Also used : Table(edu.iu.dsc.tws.common.table.Table) ArrowTable(edu.iu.dsc.tws.common.table.arrow.ArrowTable) IntVector(org.apache.arrow.vector.IntVector) ArrowAllToAll(edu.iu.dsc.tws.comms.table.ArrowAllToAll) Float8Vector(org.apache.arrow.vector.Float8Vector) Schema(org.apache.arrow.vector.types.pojo.Schema) LogicalPlanBuilder(edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder) FieldVector(org.apache.arrow.vector.FieldVector) Field(org.apache.arrow.vector.types.pojo.Field) RootAllocator(org.apache.arrow.memory.RootAllocator) ArrowTable(edu.iu.dsc.tws.common.table.arrow.ArrowTable)

Example 2 with ArrowTable

use of edu.iu.dsc.tws.common.table.arrow.ArrowTable in project twister2 by DSC-SPIDAL.

the class ArrowAllToAll method onReceive.

@Override
public void onReceive(int source, ChannelBuffer buffer, int length) {
    PendingReceiveTable table = receives.get(source);
    receivedBuffers++;
    ArrowBuf buf = ((ArrowChannelBuffer) buffer).getArrowBuf();
    table.buffers.add(buf);
    if (table.bufferIndex == 0) {
        table.fieldNodes.add(new ArrowFieldNode(table.noArray, 0));
    }
    VectorSchemaRoot schemaRoot = table.root;
    List<FieldVector> fieldVectors = schemaRoot.getFieldVectors();
    // we received everything for this array
    if (table.noBuffers == table.bufferIndex + 1) {
        FieldVector fieldVector = fieldVectors.get(table.columnIndex);
        loadBuffers(fieldVector, fieldVector.getField(), table.buffers.iterator(), table.fieldNodes.iterator());
        table.arrays.add(fieldVector);
        table.buffers.clear();
        if (table.arrays.size() == schemaRoot.getFieldVectors().size()) {
            List<ArrowColumn> columns = new ArrayList<>();
            // create the table
            for (FieldVector v : fieldVectors) {
                ArrowColumn c;
                if (v instanceof BaseFixedWidthVector) {
                    if (v instanceof IntVector) {
                        c = new Int4Column((IntVector) v);
                    } else if (v instanceof Float4Vector) {
                        c = new Float4Column((Float4Vector) v);
                    } else if (v instanceof Float8Vector) {
                        c = new Float8Column((Float8Vector) v);
                    } else if (v instanceof UInt8Vector) {
                        c = new Int8Column((UInt8Vector) v);
                    } else if (v instanceof UInt2Vector) {
                        c = new UInt2Column((UInt2Vector) v);
                    } else {
                        throw new RuntimeException("Un-supported type : " + v.getClass().getName());
                    }
                } else if (v instanceof BaseVariableWidthVector) {
                    if (v instanceof VarCharVector) {
                        c = new StringColumn((VarCharVector) v);
                    } else if (v instanceof VarBinaryVector) {
                        c = new BinaryColumn((VarBinaryVector) v);
                    } else {
                        throw new RuntimeException("Un-supported type : " + v.getClass().getName());
                    }
                } else {
                    throw new RuntimeException("Un-supported type : " + v.getClass().getName());
                }
                columns.add(c);
            }
            Table t = new ArrowTable(schemaRoot.getSchema(), table.noArray, columns);
            LOG.info("Received table from source " + source + " to " + table.target + " count" + t.rowCount());
            recvCallback.onReceive(source, table.target, t);
            table.clear();
        }
    }
}
Also used : BaseFixedWidthVector(org.apache.arrow.vector.BaseFixedWidthVector) VectorSchemaRoot(org.apache.arrow.vector.VectorSchemaRoot) ArrowBuf(io.netty.buffer.ArrowBuf) Float4Vector(org.apache.arrow.vector.Float4Vector) BinaryColumn(edu.iu.dsc.tws.common.table.arrow.BinaryColumn) ArrayList(java.util.ArrayList) VarBinaryVector(org.apache.arrow.vector.VarBinaryVector) ArrowColumn(edu.iu.dsc.tws.common.table.ArrowColumn) BaseVariableWidthVector(org.apache.arrow.vector.BaseVariableWidthVector) ArrowFieldNode(org.apache.arrow.vector.ipc.message.ArrowFieldNode) Int8Column(edu.iu.dsc.tws.common.table.arrow.Int8Column) StringColumn(edu.iu.dsc.tws.common.table.arrow.StringColumn) Table(edu.iu.dsc.tws.common.table.Table) ArrowTable(edu.iu.dsc.tws.common.table.arrow.ArrowTable) IntVector(org.apache.arrow.vector.IntVector) UInt2Column(edu.iu.dsc.tws.common.table.arrow.UInt2Column) Float8Vector(org.apache.arrow.vector.Float8Vector) VarCharVector(org.apache.arrow.vector.VarCharVector) FieldVector(org.apache.arrow.vector.FieldVector) Float4Column(edu.iu.dsc.tws.common.table.arrow.Float4Column) UInt8Vector(org.apache.arrow.vector.UInt8Vector) Float8Column(edu.iu.dsc.tws.common.table.arrow.Float8Column) Int4Column(edu.iu.dsc.tws.common.table.arrow.Int4Column) ArrowTable(edu.iu.dsc.tws.common.table.arrow.ArrowTable) UInt2Vector(org.apache.arrow.vector.UInt2Vector)

Aggregations

Table (edu.iu.dsc.tws.common.table.Table)2 ArrowTable (edu.iu.dsc.tws.common.table.arrow.ArrowTable)2 FieldVector (org.apache.arrow.vector.FieldVector)2 Float8Vector (org.apache.arrow.vector.Float8Vector)2 IntVector (org.apache.arrow.vector.IntVector)2 ArrowColumn (edu.iu.dsc.tws.common.table.ArrowColumn)1 BinaryColumn (edu.iu.dsc.tws.common.table.arrow.BinaryColumn)1 Float4Column (edu.iu.dsc.tws.common.table.arrow.Float4Column)1 Float8Column (edu.iu.dsc.tws.common.table.arrow.Float8Column)1 Int4Column (edu.iu.dsc.tws.common.table.arrow.Int4Column)1 Int8Column (edu.iu.dsc.tws.common.table.arrow.Int8Column)1 StringColumn (edu.iu.dsc.tws.common.table.arrow.StringColumn)1 UInt2Column (edu.iu.dsc.tws.common.table.arrow.UInt2Column)1 ArrowAllToAll (edu.iu.dsc.tws.comms.table.ArrowAllToAll)1 LogicalPlanBuilder (edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder)1 ArrowBuf (io.netty.buffer.ArrowBuf)1 ArrayList (java.util.ArrayList)1 RootAllocator (org.apache.arrow.memory.RootAllocator)1 BaseFixedWidthVector (org.apache.arrow.vector.BaseFixedWidthVector)1 BaseVariableWidthVector (org.apache.arrow.vector.BaseVariableWidthVector)1