Search in sources :

Example 1 with Multimap

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap in project beam by apache.

the class RenameFields method renameSchema.

// Apply the user-specified renames to the input schema.
@VisibleForTesting
static void renameSchema(Schema inputSchema, Collection<RenamePair> renames, Map<UUID, Schema> renamedSchemasMap, Map<UUID, BitSet> nestedFieldRenamedMap) {
    // The mapping of renames to apply at this level of the schema.
    Map<Integer, String> topLevelRenames = Maps.newHashMap();
    // For nested schemas, collect all applicable renames here.
    Multimap<Integer, RenamePair> nestedRenames = ArrayListMultimap.create();
    for (RenamePair rename : renames) {
        FieldAccessDescriptor access = rename.getFieldAccessDescriptor();
        if (!access.fieldIdsAccessed().isEmpty()) {
            // This references a field at this level of the schema.
            Integer fieldId = Iterables.getOnlyElement(access.fieldIdsAccessed());
            topLevelRenames.put(fieldId, rename.getNewName());
        } else {
            // This references a nested field.
            Map.Entry<Integer, FieldAccessDescriptor> nestedAccess = Iterables.getOnlyElement(access.nestedFieldsById().entrySet());
            nestedFieldRenamedMap.computeIfAbsent(inputSchema.getUUID(), s -> new BitSet(inputSchema.getFieldCount())).set(nestedAccess.getKey());
            nestedRenames.put(nestedAccess.getKey(), RenamePair.of(nestedAccess.getValue(), rename.getNewName()));
        }
    }
    Schema.Builder builder = Schema.builder();
    for (int i = 0; i < inputSchema.getFieldCount(); ++i) {
        Field field = inputSchema.getField(i);
        FieldType fieldType = field.getType();
        String newName = topLevelRenames.getOrDefault(i, field.getName());
        Collection<RenamePair> nestedFieldRenames = nestedRenames.asMap().getOrDefault(i, Collections.emptyList());
        builder.addField(newName, renameFieldType(fieldType, nestedFieldRenames, renamedSchemasMap, nestedFieldRenamedMap));
    }
    renamedSchemasMap.put(inputSchema.getUUID(), builder.build());
}
Also used : Experimental(org.apache.beam.sdk.annotations.Experimental) PTransform(org.apache.beam.sdk.transforms.PTransform) Multimap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap) Kind(org.apache.beam.sdk.annotations.Experimental.Kind) Map(java.util.Map) FieldAccessDescriptor(org.apache.beam.sdk.schemas.FieldAccessDescriptor) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) Maps(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps) ArrayListMultimap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap) Row(org.apache.beam.sdk.values.Row) Nullable(javax.annotation.Nullable) Field(org.apache.beam.sdk.schemas.Schema.Field) DoFn(org.apache.beam.sdk.transforms.DoFn) Collection(java.util.Collection) Lists(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists) FieldType(org.apache.beam.sdk.schemas.Schema.FieldType) UUID(java.util.UUID) PCollection(org.apache.beam.sdk.values.PCollection) Collectors(java.util.stream.Collectors) Schema(org.apache.beam.sdk.schemas.Schema) Serializable(java.io.Serializable) List(java.util.List) ParDo(org.apache.beam.sdk.transforms.ParDo) VisibleForTesting(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting) AutoValue(com.google.auto.value.AutoValue) ImmutableList(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList) BitSet(java.util.BitSet) Collections(java.util.Collections) FieldAccessDescriptor(org.apache.beam.sdk.schemas.FieldAccessDescriptor) Schema(org.apache.beam.sdk.schemas.Schema) BitSet(java.util.BitSet) FieldType(org.apache.beam.sdk.schemas.Schema.FieldType) Field(org.apache.beam.sdk.schemas.Schema.Field) Map(java.util.Map) VisibleForTesting(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting)

Example 2 with Multimap

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap in project beam by apache.

the class ProtoByteBuddyUtils method createGetter.

private static <ProtoT> FieldValueGetter createGetter(FieldValueTypeInformation fieldValueTypeInformation, TypeConversionsFactory typeConversionsFactory, Class clazz, Multimap<String, Method> methods, Field field, FieldValueTypeSupplier fieldValueTypeSupplier) {
    if (field.getType().isLogicalType(OneOfType.IDENTIFIER)) {
        OneOfType oneOfType = field.getType().getLogicalType(OneOfType.class);
        // The case accessor method in the proto is named getOneOfNameCase.
        Method caseMethod = getProtoGetter(methods, field.getName() + "_case", FieldType.logicalType(oneOfType.getCaseEnumType()));
        // Create a map of case enum value to getter. This must be sorted, so store in a TreeMap.
        TreeMap<Integer, FieldValueGetter<ProtoT, OneOfType.Value>> oneOfGetters = Maps.newTreeMap();
        Map<String, FieldValueTypeInformation> oneOfFieldTypes = fieldValueTypeSupplier.get(clazz, oneOfType.getOneOfSchema()).stream().collect(Collectors.toMap(FieldValueTypeInformation::getName, f -> f));
        for (Field oneOfField : oneOfType.getOneOfSchema().getFields()) {
            int protoFieldIndex = getFieldNumber(oneOfField);
            FieldValueGetter oneOfFieldGetter = createGetter(oneOfFieldTypes.get(oneOfField.getName()), typeConversionsFactory, clazz, methods, oneOfField, fieldValueTypeSupplier);
            oneOfGetters.put(protoFieldIndex, oneOfFieldGetter);
        }
        return createOneOfGetter(fieldValueTypeInformation, oneOfGetters, clazz, oneOfType, caseMethod);
    } else {
        return JavaBeanUtils.createGetter(fieldValueTypeInformation, typeConversionsFactory);
    }
}
Also used : Arrays(java.util.Arrays) ProtocolMessageEnum(com.google.protobuf.ProtocolMessageEnum) ArrayAccess(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.collection.ArrayAccess) OneOfType(org.apache.beam.sdk.schemas.logicaltypes.OneOfType) TypeCasting(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.assign.TypeCasting) StringValue(com.google.protobuf.StringValue) AsmVisitorWrapper(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.asm.AsmVisitorWrapper) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) ByteBuddyUtils(org.apache.beam.sdk.schemas.utils.ByteBuddyUtils) ByteCodeAppender(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.ByteCodeAppender) Label(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.jar.asm.Label) NullConstant(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.constant.NullConstant) Visibility(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.description.modifier.Visibility) Map(java.util.Map) EnumerationType(org.apache.beam.sdk.schemas.logicaltypes.EnumerationType) MethodReturn(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.member.MethodReturn) Method(java.lang.reflect.Method) ClassLoadingStrategy(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.dynamic.loading.ClassLoadingStrategy) FloatValue(com.google.protobuf.FloatValue) Size(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.ByteCodeAppender.Size) Set(java.util.Set) FieldType(org.apache.beam.sdk.schemas.Schema.FieldType) Collectors(java.util.stream.Collectors) TypeConversion(org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.TypeConversion) InvocationTargetException(java.lang.reflect.InvocationTargetException) FixedValue(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.FixedValue) ByteString(com.google.protobuf.ByteString) BytesValue(com.google.protobuf.BytesValue) FieldValueTypeSupplier(org.apache.beam.sdk.schemas.utils.FieldValueTypeSupplier) List(java.util.List) Type(java.lang.reflect.Type) Modifier(java.lang.reflect.Modifier) ReflectHelpers(org.apache.beam.sdk.util.common.ReflectHelpers) ProtoSchemaTranslator.getFieldNumber(org.apache.beam.sdk.extensions.protobuf.ProtoSchemaTranslator.getFieldNumber) Typing(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.assign.Assigner.Typing) Implementation(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.Implementation) ByteBuddy(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.ByteBuddy) TypeDescription(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.description.type.TypeDescription) TypeDescriptor(org.apache.beam.sdk.values.TypeDescriptor) Int64Value(com.google.protobuf.Int64Value) ConvertType(org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.ConvertType) MethodVariableAccess(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.member.MethodVariableAccess) BoolValue(com.google.protobuf.BoolValue) Experimental(org.apache.beam.sdk.annotations.Experimental) DynamicType(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.dynamic.DynamicType) UInt64Value(com.google.protobuf.UInt64Value) CaseFormat(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat) ClassWithSchema(org.apache.beam.sdk.schemas.utils.ReflectUtils.ClassWithSchema) Timestamp(com.google.protobuf.Timestamp) Supplier(java.util.function.Supplier) FieldAccess(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.member.FieldAccess) InstrumentedType(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.dynamic.scaffold.InstrumentedType) FieldManifestation(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.description.modifier.FieldManifestation) FieldValueSetter(org.apache.beam.sdk.schemas.FieldValueSetter) Multimap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap) Kind(org.apache.beam.sdk.annotations.Experimental.Kind) SchemaUserTypeCreator(org.apache.beam.sdk.schemas.SchemaUserTypeCreator) Maps(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps) MethodDescription(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.description.method.MethodDescription) FieldValueTypeInformation(org.apache.beam.sdk.schemas.FieldValueTypeInformation) ForLoadedType(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.description.type.TypeDescription.ForLoadedType) Row(org.apache.beam.sdk.values.Row) IntegerConstant(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.constant.IntegerConstant) JavaBeanUtils(org.apache.beam.sdk.schemas.utils.JavaBeanUtils) Nullable(org.checkerframework.checker.nullness.qual.Nullable) DoubleValue(com.google.protobuf.DoubleValue) Field(org.apache.beam.sdk.schemas.Schema.Field) Assigner(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.assign.Assigner) Int32Value(com.google.protobuf.Int32Value) Duplication(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.Duplication) Iterator(java.util.Iterator) ReflectUtils(org.apache.beam.sdk.schemas.utils.ReflectUtils) Compound(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.StackManipulation.Compound) ConvertValueForGetter(org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.ConvertValueForGetter) ElementMatchers(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.matcher.ElementMatchers) UInt32Value(com.google.protobuf.UInt32Value) Lists(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists) ClassWriter(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.jar.asm.ClassWriter) Schema(org.apache.beam.sdk.schemas.Schema) TypeName(org.apache.beam.sdk.schemas.Schema.TypeName) EnumLite(com.google.protobuf.Internal.EnumLite) MessageLite(com.google.protobuf.MessageLite) Duration(com.google.protobuf.Duration) StackManipulation(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.StackManipulation) MethodInvocation(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.implementation.bytecode.member.MethodInvocation) TreeMap(java.util.TreeMap) ConvertValueForSetter(org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.ConvertValueForSetter) InjectPackageStrategy(org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.InjectPackageStrategy) Preconditions(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions) TypeConversionsFactory(org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.TypeConversionsFactory) FieldValueGetter(org.apache.beam.sdk.schemas.FieldValueGetter) ForLoadedMethod(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.description.method.MethodDescription.ForLoadedMethod) Field(org.apache.beam.sdk.schemas.Schema.Field) FieldValueGetter(org.apache.beam.sdk.schemas.FieldValueGetter) FieldValueTypeInformation(org.apache.beam.sdk.schemas.FieldValueTypeInformation) Method(java.lang.reflect.Method) ForLoadedMethod(org.apache.beam.vendor.bytebuddy.v1_11_0.net.bytebuddy.description.method.MethodDescription.ForLoadedMethod) ByteString(com.google.protobuf.ByteString) OneOfType(org.apache.beam.sdk.schemas.logicaltypes.OneOfType)

Aggregations

List (java.util.List)2 Map (java.util.Map)2 Collectors (java.util.stream.Collectors)2 Experimental (org.apache.beam.sdk.annotations.Experimental)2 Kind (org.apache.beam.sdk.annotations.Experimental.Kind)2 Schema (org.apache.beam.sdk.schemas.Schema)2 Field (org.apache.beam.sdk.schemas.Schema.Field)2 FieldType (org.apache.beam.sdk.schemas.Schema.FieldType)2 Row (org.apache.beam.sdk.values.Row)2 Lists (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists)2 Maps (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps)2 Multimap (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap)2 AutoValue (com.google.auto.value.AutoValue)1 BoolValue (com.google.protobuf.BoolValue)1 ByteString (com.google.protobuf.ByteString)1 BytesValue (com.google.protobuf.BytesValue)1 DoubleValue (com.google.protobuf.DoubleValue)1 Duration (com.google.protobuf.Duration)1 FloatValue (com.google.protobuf.FloatValue)1 Int32Value (com.google.protobuf.Int32Value)1