use of edu.iu.dsc.tws.common.table.arrow.ArrowTable in project twister2 by DSC-SPIDAL.
the class BTAllToAll method execute.
@Override
public void execute(Config config, JobAPI.Job job, IWorkerController workerController, IPersistentVolume persistentVolume, IVolatileVolume volatileVolume) {
this.jobParameters = JobParameters.build(config);
// create a worker environment
this.wEnv = WorkerEnvironment.init(config, job, workerController, persistentVolume, volatileVolume);
LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), wEnv).withFairDistribution();
RootAllocator rootAllocator = new RootAllocator();
IntVector intVector = new IntVector("fist", rootAllocator);
Float8Vector float8Vector = new Float8Vector("second", rootAllocator);
for (int i = 0; i < 1000; i++) {
intVector.setSafe(i, i);
float8Vector.setSafe(i, i);
}
intVector.setValueCount(1000);
float8Vector.setValueCount(1000);
List<Field> fieldList = Arrays.asList(intVector.getField(), float8Vector.getField());
Schema schema = new Schema(fieldList);
Table t = new ArrowTable(schema, Arrays.asList(new FieldVector[] { intVector, float8Vector }));
allToAll = new ArrowAllToAll(wEnv.getConfig(), wEnv.getWorkerController(), logicalPlanBuilder.getSources(), logicalPlanBuilder.getTargets(), logicalPlanBuilder.build(), wEnv.getCommunicator().nextEdge(), new ArrowReceiver(), schema, rootAllocator);
for (int i : logicalPlanBuilder.getTargets()) {
allToAll.insert(t, i);
}
for (int s : logicalPlanBuilder.getSourcesOnThisWorker()) {
allToAll.finish(s);
}
while (!allToAll.isComplete()) {
// wait
}
}
use of edu.iu.dsc.tws.common.table.arrow.ArrowTable in project twister2 by DSC-SPIDAL.
the class ArrowAllToAll method onReceive.
@Override
public void onReceive(int source, ChannelBuffer buffer, int length) {
PendingReceiveTable table = receives.get(source);
receivedBuffers++;
ArrowBuf buf = ((ArrowChannelBuffer) buffer).getArrowBuf();
table.buffers.add(buf);
if (table.bufferIndex == 0) {
table.fieldNodes.add(new ArrowFieldNode(table.noArray, 0));
}
VectorSchemaRoot schemaRoot = table.root;
List<FieldVector> fieldVectors = schemaRoot.getFieldVectors();
// we received everything for this array
if (table.noBuffers == table.bufferIndex + 1) {
FieldVector fieldVector = fieldVectors.get(table.columnIndex);
loadBuffers(fieldVector, fieldVector.getField(), table.buffers.iterator(), table.fieldNodes.iterator());
table.arrays.add(fieldVector);
table.buffers.clear();
if (table.arrays.size() == schemaRoot.getFieldVectors().size()) {
List<ArrowColumn> columns = new ArrayList<>();
// create the table
for (FieldVector v : fieldVectors) {
ArrowColumn c;
if (v instanceof BaseFixedWidthVector) {
if (v instanceof IntVector) {
c = new Int4Column((IntVector) v);
} else if (v instanceof Float4Vector) {
c = new Float4Column((Float4Vector) v);
} else if (v instanceof Float8Vector) {
c = new Float8Column((Float8Vector) v);
} else if (v instanceof UInt8Vector) {
c = new Int8Column((UInt8Vector) v);
} else if (v instanceof UInt2Vector) {
c = new UInt2Column((UInt2Vector) v);
} else {
throw new RuntimeException("Un-supported type : " + v.getClass().getName());
}
} else if (v instanceof BaseVariableWidthVector) {
if (v instanceof VarCharVector) {
c = new StringColumn((VarCharVector) v);
} else if (v instanceof VarBinaryVector) {
c = new BinaryColumn((VarBinaryVector) v);
} else {
throw new RuntimeException("Un-supported type : " + v.getClass().getName());
}
} else {
throw new RuntimeException("Un-supported type : " + v.getClass().getName());
}
columns.add(c);
}
Table t = new ArrowTable(schemaRoot.getSchema(), table.noArray, columns);
LOG.info("Received table from source " + source + " to " + table.target + " count" + t.rowCount());
recvCallback.onReceive(source, table.target, t);
table.clear();
}
}
}
Aggregations