Examples with Field org.apache.arrow.vector.types.pojo.Field used on opensource projects

Example 16 with Field

use of org.apache.arrow.vector.types.pojo.Field in project twister2 by DSC-SPIDAL.

the class Twister2ArrowFileWriter method setUpTwister2ArrowWrite.

public boolean setUpTwister2ArrowWrite(int workerId) throws Exception {
    LOG.fine("%%%%%%%%% worker id details:" + workerId + "\t" + arrowFile);
    this.root = VectorSchemaRoot.create(Schema.fromJSON(arrowSchema), this.rootAllocator);
    Path path = new Path(arrowFile);
    this.fileSystem = FileSystemUtils.get(path);
    this.fsDataOutputStream = fileSystem.create(path);
    this.twister2ArrowOutputStream = new Twister2ArrowOutputStream(this.fsDataOutputStream);
    DictionaryProvider.MapDictionaryProvider provider = new DictionaryProvider.MapDictionaryProvider();
    if (!flag) {
        this.arrowFileWriter = new ArrowFileWriter(root, provider, this.fsDataOutputStream.getChannel());
    } else {
        this.arrowFileWriter = new ArrowFileWriter(root, provider, this.twister2ArrowOutputStream);
    }
    LOG.info("root schema fields:" + root.getSchema().getFields());
    for (Field field : root.getSchema().getFields()) {
        FieldVector vector = root.getVector(field.getName());
        if (vector.getMinorType().equals(Types.MinorType.INT)) {
            this.generatorMap.put(vector, new IntVectorGenerator());
        } else if (vector.getMinorType().equals(Types.MinorType.BIGINT)) {
            this.generatorMap.put(vector, new BigIntVectorGenerator());
        } else if (vector.getMinorType().equals(Types.MinorType.FLOAT4)) {
            this.generatorMap.put(vector, new FloatVectorGenerator());
        } else {
            throw new RuntimeException("unsupported arrow write type");
        }
    }
    return true;
}

Example 17 with Field

use of org.apache.arrow.vector.types.pojo.Field in project parquet-mr by apache.

the class TestSchemaConverter method compareFields.

/**
 * for more pinpointed error on what is different
 * @param left
 * @param right
 */
private void compareFields(List<Field> left, List<Field> right) {
    Assert.assertEquals(left + "\n" + right, left.size(), right.size());
    int size = left.size();
    for (int i = 0; i < size; i++) {
        Field expectedField = left.get(i);
        Field field = right.get(i);
        compareFields(expectedField.getChildren(), field.getChildren());
        Assert.assertEquals(expectedField, field);
    }
}

Example 18 with Field

use of org.apache.arrow.vector.types.pojo.Field in project carbondata by apache.

the class ArrowWriter method createFieldWriter.

private static ArrowFieldWriter createFieldWriter(ValueVector valueVector) {
    if (valueVector instanceof BitVector) {
        return new BooleanWriter((BitVector) valueVector);
    } else if (valueVector instanceof TinyIntVector) {
        return new ByteWriter((TinyIntVector) valueVector);
    } else if (valueVector instanceof SmallIntVector) {
        return new ShortWriter((SmallIntVector) valueVector);
    } else if (valueVector instanceof IntVector) {
        return new IntWriter((IntVector) valueVector);
    } else if (valueVector instanceof BigIntVector) {
        return new LongWriter((BigIntVector) valueVector);
    } else if (valueVector instanceof DecimalVector) {
        DecimalVector decimalVector = (DecimalVector) valueVector;
        final Field field = decimalVector.getField();
        ArrowType.Decimal c = (ArrowType.Decimal) field.getType();
        return new DecimalWriter((DecimalVector) valueVector, c.getPrecision(), c.getScale());
    } else if (valueVector instanceof VarCharVector) {
        return new StringWriter((VarCharVector) valueVector);
    } else if (valueVector instanceof Float4Vector) {
        return new FloatWriter((Float4Vector) valueVector);
    } else if (valueVector instanceof Float8Vector) {
        return new DoubleWriter((Float8Vector) valueVector);
    } else if (valueVector instanceof ListVector) {
        ArrowFieldWriter elementVector = createFieldWriter(((ListVector) valueVector).getDataVector());
        return new ArrayWriter((ListVector) valueVector, elementVector);
    } else if (valueVector instanceof StructVector) {
        StructVector s = (StructVector) valueVector;
        List<ArrowFieldWriter> arrowFieldWriters = new ArrayList<>();
        for (int i = 0; i < s.size(); i++) {
            arrowFieldWriters.add(createFieldWriter(s.getChildByOrdinal(i)));
        }
        return new StructWriter(s, arrowFieldWriters.toArray(new ArrowFieldWriter[arrowFieldWriters.size()]));
    } else if (valueVector instanceof VarBinaryVector) {
        return new BinaryWriter((VarBinaryVector) valueVector);
    } else if (valueVector instanceof DateDayVector) {
        return new DateWriter((DateDayVector) valueVector);
    } else if (valueVector instanceof TimeStampMicroTZVector) {
        return new TimeStampWriter((TimeStampMicroTZVector) valueVector);
    } else {
        throw new UnsupportedOperationException("Invalid data type");
    }
}