Examples with ArrowFileWriter org.apache.arrow.vector.ipc.ArrowFileWriter used on opensource projects

Example 1 with ArrowFileWriter

use of org.apache.arrow.vector.ipc.ArrowFileWriter in project twister2 by DSC-SPIDAL.

the class Twister2ArrowFileWriter method setUpTwister2ArrowWrite.

public boolean setUpTwister2ArrowWrite(int workerId) throws Exception {
    LOG.fine("%%%%%%%%% worker id details:" + workerId + "\t" + arrowFile);
    this.root = VectorSchemaRoot.create(Schema.fromJSON(arrowSchema), this.rootAllocator);
    Path path = new Path(arrowFile);
    this.fileSystem = FileSystemUtils.get(path);
    this.fsDataOutputStream = fileSystem.create(path);
    this.twister2ArrowOutputStream = new Twister2ArrowOutputStream(this.fsDataOutputStream);
    DictionaryProvider.MapDictionaryProvider provider = new DictionaryProvider.MapDictionaryProvider();
    if (!flag) {
        this.arrowFileWriter = new ArrowFileWriter(root, provider, this.fsDataOutputStream.getChannel());
    } else {
        this.arrowFileWriter = new ArrowFileWriter(root, provider, this.twister2ArrowOutputStream);
    }
    LOG.info("root schema fields:" + root.getSchema().getFields());
    for (Field field : root.getSchema().getFields()) {
        FieldVector vector = root.getVector(field.getName());
        if (vector.getMinorType().equals(Types.MinorType.INT)) {
            this.generatorMap.put(vector, new IntVectorGenerator());
        } else if (vector.getMinorType().equals(Types.MinorType.BIGINT)) {
            this.generatorMap.put(vector, new BigIntVectorGenerator());
        } else if (vector.getMinorType().equals(Types.MinorType.FLOAT4)) {
            this.generatorMap.put(vector, new FloatVectorGenerator());
        } else {
            throw new RuntimeException("unsupported arrow write type");
        }
    }
    return true;
}