Examples with FieldType org.apache.beam.sdk.schemas.Schema.FieldType used on opensource projects

Example 16 with FieldType

use of org.apache.beam.sdk.schemas.Schema.FieldType in project beam by apache.

the class ProtoSchemaTranslator method getSchema.

static Schema getSchema(Descriptors.Descriptor descriptor) {
    /* OneOfComponentFields refers to the field number in the protobuf where the component subfields
     * are. This is needed to prevent double inclusion of the component fields.*/
    Set<Integer> oneOfComponentFields = Sets.newHashSet();
    /* OneOfFieldLocation stores the field number of the first field in the OneOf. Using this, we can use the location
    of the first field in the OneOf as the location of the entire OneOf.*/
    Map<Integer, Field> oneOfFieldLocation = Maps.newHashMap();
    List<Field> fields = Lists.newArrayListWithCapacity(descriptor.getFields().size());
    for (OneofDescriptor oneofDescriptor : descriptor.getOneofs()) {
        List<Field> subFields = Lists.newArrayListWithCapacity(oneofDescriptor.getFieldCount());
        Map<String, Integer> enumIds = Maps.newHashMap();
        for (FieldDescriptor fieldDescriptor : oneofDescriptor.getFields()) {
            oneOfComponentFields.add(fieldDescriptor.getNumber());
            // Store proto field number in a field option.
            FieldType fieldType = beamFieldTypeFromProtoField(fieldDescriptor);
            subFields.add(withFieldNumber(Field.nullable(fieldDescriptor.getName(), fieldType), fieldDescriptor.getNumber()));
            checkArgument(enumIds.putIfAbsent(fieldDescriptor.getName(), fieldDescriptor.getNumber()) == null);
        }
        FieldType oneOfType = FieldType.logicalType(OneOfType.create(subFields, enumIds));
        oneOfFieldLocation.put(oneofDescriptor.getFields().get(0).getNumber(), Field.of(oneofDescriptor.getName(), oneOfType));
    }
    for (Descriptors.FieldDescriptor fieldDescriptor : descriptor.getFields()) {
        int fieldDescriptorNumber = fieldDescriptor.getNumber();
        if (!oneOfComponentFields.contains(fieldDescriptorNumber)) {
            // Store proto field number in metadata.
            FieldType fieldType = beamFieldTypeFromProtoField(fieldDescriptor);
            fields.add(withFieldNumber(Field.of(fieldDescriptor.getName(), fieldType), fieldDescriptorNumber).withOptions(getFieldOptions(fieldDescriptor)));
        /* Note that descriptor.getFields() returns an iterator in the order of the fields in the .proto file, not
         * in field number order. Therefore we can safely insert the OneOfField at the field of its first component.*/
        } else {
            Field oneOfField = oneOfFieldLocation.get(fieldDescriptorNumber);
            if (oneOfField != null) {
                fields.add(oneOfField);
            }
        }
    }
    return Schema.builder().addFields(fields).setOptions(getSchemaOptions(descriptor).setOption(SCHEMA_OPTION_META_TYPE_NAME, FieldType.STRING, descriptor.getFullName())).build();
}

Example 17 with FieldType

use of org.apache.beam.sdk.schemas.Schema.FieldType in project beam by apache.

the class RowWithGetters method getValue.

@Override
@SuppressWarnings({ "TypeParameterUnusedInFormals", "unchecked" })
@Nullable
public <T> T getValue(int fieldIdx) {
    Field field = getSchema().getField(fieldIdx);
    FieldType type = field.getType();
    Object fieldValue = getters.get(fieldIdx).get(getterTarget);
    if (fieldValue == null && !field.getType().getNullable()) {
        throw new RuntimeException("Null value set on non-nullable field " + field);
    }
    return fieldValue != null ? getValue(type, fieldValue, fieldIdx) : null;
}

Example 18 with FieldType

use of org.apache.beam.sdk.schemas.Schema.FieldType in project beam by apache.

the class FieldAccessDescriptor method fillInMissingQualifiers.

private FieldDescriptor fillInMissingQualifiers(FieldDescriptor fieldDescriptor, Schema schema) {
    // If there are nested arrays or maps, walk down them until we find the next row. If there
    // are missing qualifiers, fill them in. This allows users to omit the qualifiers in the
    // simple case where they are all wildcards. For example, if a is a list of a list of row,
    // the user could select a[*][*].b, however we allow them to simply type a.b for brevity.
    FieldType fieldType = schema.getField(fieldDescriptor.getFieldId()).getType();
    Iterator<Qualifier> qualifierIt = fieldDescriptor.getQualifiers().iterator();
    List<Qualifier> qualifiers = Lists.newArrayList();
    while (fieldType.getTypeName().isCollectionType() || fieldType.getTypeName().isMapType()) {
        Qualifier qualifier = qualifierIt.hasNext() ? qualifierIt.next() : null;
        if (fieldType.getTypeName().isCollectionType()) {
            qualifier = (qualifier == null) ? Qualifier.of(ListQualifier.ALL) : qualifier;
            checkArgument(qualifier.getKind().equals(Qualifier.Kind.LIST));
            checkArgument(qualifier.getList().equals(ListQualifier.ALL));
            qualifiers.add(qualifier);
            fieldType = fieldType.getCollectionElementType();
        } else if (fieldType.getTypeName().isMapType()) {
            qualifier = (qualifier == null) ? Qualifier.of(MapQualifier.ALL) : qualifier;
            checkArgument(qualifier.getKind().equals(Qualifier.Kind.MAP));
            checkArgument(qualifier.getMap().equals(MapQualifier.ALL));
            qualifiers.add(qualifier);
            fieldType = fieldType.getMapValueType();
        }
    }
    return fieldDescriptor.toBuilder().setQualifiers(qualifiers).build();
}

Example 19 with FieldType

use of org.apache.beam.sdk.schemas.Schema.FieldType in project beam by apache.

the class FieldTypeDescriptors method getIterableFieldType.

private static FieldType getIterableFieldType(TypeDescriptor typeDescriptor) {
    TypeDescriptor<Iterable<?>> iterable = typeDescriptor.getSupertype(Iterable.class);
    if (iterable.getType() instanceof ParameterizedType) {
        ParameterizedType ptype = (ParameterizedType) iterable.getType();
        java.lang.reflect.Type[] params = ptype.getActualTypeArguments();
        checkArgument(params.length == 1);
        return FieldType.iterable(fieldTypeForJavaType(TypeDescriptor.of(params[0])));
    }
    throw new RuntimeException("Could not determine array parameter type for field.");
}

Example 20 with FieldType

use of org.apache.beam.sdk.schemas.Schema.FieldType in project beam by apache.

the class FieldTypeDescriptors method getMapFieldType.

private static FieldType getMapFieldType(TypeDescriptor typeDescriptor) {
    TypeDescriptor<Collection<?>> map = typeDescriptor.getSupertype(Map.class);
    if (map.getType() instanceof ParameterizedType) {
        ParameterizedType ptype = (ParameterizedType) map.getType();
        java.lang.reflect.Type[] params = ptype.getActualTypeArguments();
        return FieldType.map(fieldTypeForJavaType(TypeDescriptor.of(params[0])), fieldTypeForJavaType(TypeDescriptor.of(params[1])));
    }
    throw new RuntimeException("Cound not determine array parameter type for field.");
}