Search in sources :

Example 1 with FieldDescriptorProto

use of com.google.protobuf.DescriptorProtos.FieldDescriptorProto in project beam by apache.

the class TableRowToStorageApiProtoTest method testDescriptorFromTableSchema.

@Test
public void testDescriptorFromTableSchema() {
    DescriptorProto descriptor = TableRowToStorageApiProto.descriptorSchemaFromTableSchema(BASE_TABLE_SCHEMA);
    Map<String, Type> types = descriptor.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    Map<String, Type> expectedTypes = BASE_TABLE_SCHEMA_PROTO.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    assertEquals(expectedTypes, types);
}
Also used : Type(com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type) FieldDescriptorProto(com.google.protobuf.DescriptorProtos.FieldDescriptorProto) DescriptorProto(com.google.protobuf.DescriptorProtos.DescriptorProto) ByteString(com.google.protobuf.ByteString) Test(org.junit.Test)

Example 2 with FieldDescriptorProto

use of com.google.protobuf.DescriptorProtos.FieldDescriptorProto in project beam by apache.

the class TableRowToStorageApiProtoTest method testNestedFromTableSchema.

@Test
public void testNestedFromTableSchema() {
    DescriptorProto descriptor = TableRowToStorageApiProto.descriptorSchemaFromTableSchema(NESTED_TABLE_SCHEMA);
    Map<String, Type> expectedBaseTypes = BASE_TABLE_SCHEMA_PROTO.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    Map<String, Type> expectedBaseTypesNoF = BASE_TABLE_SCHEMA_NO_F_PROTO.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    Map<String, Type> types = descriptor.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    Map<String, String> typeNames = descriptor.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getTypeName));
    assertEquals(4, types.size());
    Map<String, DescriptorProto> nestedTypes = descriptor.getNestedTypeList().stream().collect(Collectors.toMap(DescriptorProto::getName, Functions.identity()));
    assertEquals(4, nestedTypes.size());
    assertEquals(Type.TYPE_MESSAGE, types.get("nestedvalue1"));
    String nestedTypeName1 = typeNames.get("nestedvalue1");
    Map<String, Type> nestedTypes1 = nestedTypes.get(nestedTypeName1).getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    assertEquals(expectedBaseTypes, nestedTypes1);
    assertEquals(Type.TYPE_MESSAGE, types.get("nestedvalue2"));
    String nestedTypeName2 = typeNames.get("nestedvalue2");
    Map<String, Type> nestedTypes2 = nestedTypes.get(nestedTypeName2).getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    assertEquals(expectedBaseTypes, nestedTypes2);
    assertEquals(Type.TYPE_MESSAGE, types.get("nestedvaluenof1"));
    String nestedTypeNameNoF1 = typeNames.get("nestedvaluenof1");
    Map<String, Type> nestedTypesNoF1 = nestedTypes.get(nestedTypeNameNoF1).getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    assertEquals(expectedBaseTypesNoF, nestedTypesNoF1);
    assertEquals(Type.TYPE_MESSAGE, types.get("nestedvaluenof2"));
    String nestedTypeNameNoF2 = typeNames.get("nestedvaluenof2");
    Map<String, Type> nestedTypesNoF2 = nestedTypes.get(nestedTypeNameNoF2).getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    assertEquals(expectedBaseTypesNoF, nestedTypesNoF2);
}
Also used : Type(com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type) FieldDescriptorProto(com.google.protobuf.DescriptorProtos.FieldDescriptorProto) DescriptorProto(com.google.protobuf.DescriptorProtos.DescriptorProto) ByteString(com.google.protobuf.ByteString) Test(org.junit.Test)

Example 3 with FieldDescriptorProto

use of com.google.protobuf.DescriptorProtos.FieldDescriptorProto in project beam by apache.

the class BeamRowToStorageApiProtoTest method testDescriptorFromSchema.

@Test
public void testDescriptorFromSchema() {
    DescriptorProto descriptor = BeamRowToStorageApiProto.descriptorSchemaFromBeamSchema(BASE_SCHEMA);
    Map<String, Type> types = descriptor.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    Map<String, Type> expectedTypes = BASE_SCHEMA_PROTO.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    assertEquals(expectedTypes, types);
    Map<String, String> nameMapping = BASE_SCHEMA.getFields().stream().collect(Collectors.toMap(f -> f.getName().toLowerCase(), Field::getName));
    descriptor.getFieldList().forEach(p -> {
        FieldType schemaFieldType = BASE_SCHEMA.getField(nameMapping.get(p.getName())).getType();
        Label label = schemaFieldType.getTypeName().isCollectionType() ? Label.LABEL_REPEATED : schemaFieldType.getNullable() ? Label.LABEL_OPTIONAL : Label.LABEL_REQUIRED;
        assertEquals(label, p.getLabel());
    });
}
Also used : Type(com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type) Descriptor(com.google.protobuf.Descriptors.Descriptor) DynamicMessage(com.google.protobuf.DynamicMessage) LocalDateTime(java.time.LocalDateTime) RunWith(org.junit.runner.RunWith) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) BigDecimal(java.math.BigDecimal) SqlTypes(org.apache.beam.sdk.schemas.logicaltypes.SqlTypes) Map(java.util.Map) LocalTime(java.time.LocalTime) EnumerationType(org.apache.beam.sdk.schemas.logicaltypes.EnumerationType) Row(org.apache.beam.sdk.values.Row) Field(org.apache.beam.sdk.schemas.Schema.Field) FieldDescriptorProto(com.google.protobuf.DescriptorProtos.FieldDescriptorProto) FieldType(org.apache.beam.sdk.schemas.Schema.FieldType) Test(org.junit.Test) FieldDescriptor(com.google.protobuf.Descriptors.FieldDescriptor) JUnit4(org.junit.runners.JUnit4) Collectors(java.util.stream.Collectors) Schema(org.apache.beam.sdk.schemas.Schema) StandardCharsets(java.nio.charset.StandardCharsets) ByteString(com.google.protobuf.ByteString) LocalDate(java.time.LocalDate) Instant(org.joda.time.Instant) ImmutableList(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList) Label(com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Label) Functions(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Functions) DescriptorProto(com.google.protobuf.DescriptorProtos.DescriptorProto) Assert.assertEquals(org.junit.Assert.assertEquals) Type(com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type) EnumerationType(org.apache.beam.sdk.schemas.logicaltypes.EnumerationType) FieldType(org.apache.beam.sdk.schemas.Schema.FieldType) Label(com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Label) FieldDescriptorProto(com.google.protobuf.DescriptorProtos.FieldDescriptorProto) DescriptorProto(com.google.protobuf.DescriptorProtos.DescriptorProto) ByteString(com.google.protobuf.ByteString) FieldType(org.apache.beam.sdk.schemas.Schema.FieldType) Test(org.junit.Test)

Example 4 with FieldDescriptorProto

use of com.google.protobuf.DescriptorProtos.FieldDescriptorProto in project beam by apache.

the class BeamRowToStorageApiProtoTest method testNestedFromSchema.

@Test
public void testNestedFromSchema() {
    DescriptorProto descriptor = BeamRowToStorageApiProto.descriptorSchemaFromBeamSchema(NESTED_SCHEMA);
    Map<String, Type> expectedBaseTypes = BASE_SCHEMA_PROTO.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    Map<String, Type> types = descriptor.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    Map<String, String> typeNames = descriptor.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getTypeName));
    Map<String, Label> typeLabels = descriptor.getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getLabel));
    assertEquals(3, types.size());
    Map<String, DescriptorProto> nestedTypes = descriptor.getNestedTypeList().stream().collect(Collectors.toMap(DescriptorProto::getName, Functions.identity()));
    assertEquals(3, nestedTypes.size());
    assertEquals(Type.TYPE_MESSAGE, types.get("nested"));
    assertEquals(Label.LABEL_OPTIONAL, typeLabels.get("nested"));
    String nestedTypeName1 = typeNames.get("nested");
    Map<String, Type> nestedTypes1 = nestedTypes.get(nestedTypeName1).getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    assertEquals(expectedBaseTypes, nestedTypes1);
    assertEquals(Type.TYPE_MESSAGE, types.get("nestedarray"));
    assertEquals(Label.LABEL_REPEATED, typeLabels.get("nestedarray"));
    String nestedTypeName2 = typeNames.get("nestedarray");
    Map<String, Type> nestedTypes2 = nestedTypes.get(nestedTypeName2).getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    assertEquals(expectedBaseTypes, nestedTypes2);
    assertEquals(Type.TYPE_MESSAGE, types.get("nestediterable"));
    assertEquals(Label.LABEL_REPEATED, typeLabels.get("nestediterable"));
    String nestedTypeName3 = typeNames.get("nestediterable");
    Map<String, Type> nestedTypes3 = nestedTypes.get(nestedTypeName3).getFieldList().stream().collect(Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType));
    assertEquals(expectedBaseTypes, nestedTypes3);
}
Also used : Type(com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type) EnumerationType(org.apache.beam.sdk.schemas.logicaltypes.EnumerationType) FieldType(org.apache.beam.sdk.schemas.Schema.FieldType) Label(com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Label) FieldDescriptorProto(com.google.protobuf.DescriptorProtos.FieldDescriptorProto) DescriptorProto(com.google.protobuf.DescriptorProtos.DescriptorProto) ByteString(com.google.protobuf.ByteString) Test(org.junit.Test)

Aggregations

ByteString (com.google.protobuf.ByteString)4 DescriptorProto (com.google.protobuf.DescriptorProtos.DescriptorProto)4 FieldDescriptorProto (com.google.protobuf.DescriptorProtos.FieldDescriptorProto)4 Type (com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type)4 Test (org.junit.Test)4 Label (com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Label)2 FieldType (org.apache.beam.sdk.schemas.Schema.FieldType)2 EnumerationType (org.apache.beam.sdk.schemas.logicaltypes.EnumerationType)2 Descriptor (com.google.protobuf.Descriptors.Descriptor)1 FieldDescriptor (com.google.protobuf.Descriptors.FieldDescriptor)1 DynamicMessage (com.google.protobuf.DynamicMessage)1 BigDecimal (java.math.BigDecimal)1 StandardCharsets (java.nio.charset.StandardCharsets)1 LocalDate (java.time.LocalDate)1 LocalDateTime (java.time.LocalDateTime)1 LocalTime (java.time.LocalTime)1 Map (java.util.Map)1 Collectors (java.util.stream.Collectors)1 Schema (org.apache.beam.sdk.schemas.Schema)1 Field (org.apache.beam.sdk.schemas.Schema.Field)1