Examples with Field org.apache.arrow.vector.types.pojo.Field used on opensource projects

Example 11 with Field

use of org.apache.arrow.vector.types.pojo.Field in project parquet-mr by apache.

the class SchemaConverter method fromParquetGroup.

/**
 * @param type parquet types
 * @param name overrides parquet.getName()
 * @return the mapping
 */
private TypeMapping fromParquetGroup(GroupType type, String name) {
    LogicalTypeAnnotation logicalType = type.getLogicalTypeAnnotation();
    if (logicalType == null) {
        List<TypeMapping> typeMappings = fromParquet(type.getFields());
        Field arrowField = new Field(name, type.isRepetition(OPTIONAL), new Struct(), fields(typeMappings));
        return new StructTypeMapping(arrowField, type, typeMappings);
    } else {
        return logicalType.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor<TypeMapping>() {

            @Override
            public Optional<TypeMapping> visit(LogicalTypeAnnotation.ListLogicalTypeAnnotation listLogicalType) {
                List3Levels list3Levels = new List3Levels(type);
                TypeMapping child = fromParquet(list3Levels.getElement(), null, list3Levels.getElement().getRepetition());
                Field arrowField = new Field(name, type.isRepetition(OPTIONAL), new ArrowType.List(), asList(child.getArrowField()));
                return of(new ListTypeMapping(arrowField, list3Levels, child));
            }
        }).orElseThrow(() -> new UnsupportedOperationException("Unsupported type " + type));
    }
}

Example 12 with Field

use of org.apache.arrow.vector.types.pojo.Field in project parquet-mr by apache.

the class SchemaConverter method fromParquet.

/**
 * Creates an Arrow Schema from an Parquet one and returns the mapping
 * @param parquetSchema the provided Parquet Schema
 * @return the mapping between the 2
 */
public SchemaMapping fromParquet(MessageType parquetSchema) {
    List<Type> fields = parquetSchema.getFields();
    List<TypeMapping> mappings = fromParquet(fields);
    List<Field> arrowFields = fields(mappings);
    return new SchemaMapping(new Schema(arrowFields), parquetSchema, mappings);
}

Example 13 with Field

use of org.apache.arrow.vector.types.pojo.Field in project parquet-mr by apache.

the class SchemaConverter method map.

private List<TypeMapping> map(List<Field> arrowFields, List<Type> parquetFields) {
    if (arrowFields.size() != parquetFields.size()) {
        throw new IllegalArgumentException("Can not map schemas as sizes differ: " + arrowFields + " != " + parquetFields);
    }
    List<TypeMapping> result = new ArrayList<>(arrowFields.size());
    for (int i = 0; i < arrowFields.size(); i++) {
        Field arrowField = arrowFields.get(i);
        Type parquetField = parquetFields.get(i);
        result.add(map(arrowField, parquetField));
    }
    return result;
}

Example 14 with Field

use of org.apache.arrow.vector.types.pojo.Field in project twister2 by DSC-SPIDAL.

the class RowSchema method toArrowSchema.

public org.apache.arrow.vector.types.pojo.Schema toArrowSchema() {
    List<Field> fields = new ArrayList<>();
    for (TField f : types) {
        Field field;
        if (f.getType().equals(MessageTypes.INTEGER)) {
            field = new Field(f.getName(), new FieldType(false, new ArrowType.Int(32, true), null), null);
        } else if (f.getType().equals(MessageTypes.LONG)) {
            field = new Field(f.getName(), new FieldType(false, new ArrowType.Int(64, true), null), null);
        } else if (f.getType().equals(MessageTypes.SHORT)) {
            field = new Field(f.getName(), new FieldType(false, new ArrowType.Int(16, true), null), null);
        } else if (f.getType().equals(MessageTypes.FLOAT)) {
            field = new Field(f.getName(), new FieldType(false, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null), null);
        } else if (f.getType().equals(MessageTypes.DOUBLE)) {
            field = new Field(f.getName(), new FieldType(false, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null), null);
        } else if (f.getType().equals(MessageTypes.STRING)) {
            field = new Field(f.getName(), new FieldType(false, new ArrowType.Binary(), null), null);
        } else if (f.getType().equals(MessageTypes.BYTE)) {
            field = new Field(f.getName(), new FieldType(false, new ArrowType.Binary(), null), null);
        } else {
            throw new Twister2RuntimeException("Un-known type");
        }
        fields.add(field);
    }
    return new org.apache.arrow.vector.types.pojo.Schema(fields);
}

Example 15 with Field

use of org.apache.arrow.vector.types.pojo.Field in project twister2 by DSC-SPIDAL.

the class ArrowAllToAll method loadBuffers.

private void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buffers, Iterator<ArrowFieldNode> nodes) {
    checkArgument(nodes.hasNext(), "no more field nodes for for field %s and vector %s", field, vector);
    ArrowFieldNode fieldNode = nodes.next();
    int bufferLayoutCount = TypeLayout.getTypeBufferCount(field.getType());
    List<ArrowBuf> ownBuffers = new ArrayList<>(bufferLayoutCount);
    for (int j = 0; j < bufferLayoutCount; j++) {
        ownBuffers.add(buffers.next());
    }
    try {
        vector.loadFieldBuffers(fieldNode, ownBuffers);
    } catch (RuntimeException e) {
        throw new IllegalArgumentException("Could not load buffers for field " + field + ". error message: " + e.getMessage(), e);
    }
    List<Field> children = field.getChildren();
    if (children.size() > 0) {
        List<FieldVector> childrenFromFields = vector.getChildrenFromFields();
        checkArgument(children.size() == childrenFromFields.size(), "should have as many children as in the schema: found %s expected %s", childrenFromFields.size(), children.size());
        for (int i = 0; i < childrenFromFields.size(); i++) {
            Field child = children.get(i);
            FieldVector fieldVector = childrenFromFields.get(i);
            loadBuffers(fieldVector, child, buffers, nodes);
        }
    }
}