Examples with StructType org.apache.parquet.thrift.struct.ThriftType.StructType used on opensource projects

Example 6 with StructType

use of org.apache.parquet.thrift.struct.ThriftType.StructType in project parquet-mr by apache.

the class TestThriftRecordConverter method constructorDoesNotRequireStructOrUnionTypeMeta.

@Test
public void constructorDoesNotRequireStructOrUnionTypeMeta() throws Exception {
    String jsonWithNoStructOrUnionMeta = Strings.join(Files.readAllLines(new File("src/test/resources/org/apache/parquet/thrift/StructWithUnionV1NoStructOrUnionMeta.json"), Charset.forName("UTF-8")), "\n");
    StructType noStructOrUnionMeta = (StructType) ThriftType.fromJSON(jsonWithNoStructOrUnionMeta);
    // this used to throw, see PARQUET-346
    new ThriftRecordConverter<StructWithUnionV1>(new ThriftReader<StructWithUnionV1>() {

        @Override
        public StructWithUnionV1 readOneRecord(TProtocol protocol) throws TException {
            return null;
        }
    }, "name", new ThriftSchemaConverter().convert(StructWithUnionV1.class), noStructOrUnionMeta);
}

Example 7 with StructType

use of org.apache.parquet.thrift.struct.ThriftType.StructType in project parquet-mr by apache.

the class TestParquetWriteProtocol method validateThrift.

private void validateThrift(String[] expectations, TBase<?, ?> a) throws TException {
    final ThriftSchemaConverter thriftSchemaConverter = new ThriftSchemaConverter();
    // System.out.println(a);
    final Class<TBase<?, ?>> class1 = (Class<TBase<?, ?>>) a.getClass();
    final MessageType schema = thriftSchemaConverter.convert(class1);
    LOG.info("{}", schema);
    final StructType structType = thriftSchemaConverter.toStructType(class1);
    ExpectationValidatingRecordConsumer recordConsumer = new ExpectationValidatingRecordConsumer(new ArrayDeque<String>(Arrays.asList(expectations)));
    final MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
    ParquetWriteProtocol p = new ParquetWriteProtocol(new RecordConsumerLoggingWrapper(recordConsumer), columnIO, structType);
    a.write(p);
}

Example 8 with StructType

use of org.apache.parquet.thrift.struct.ThriftType.StructType in project parquet-mr by apache.

the class TestThriftMetaData method testToStringDoesNotThrow.

/**
 * Previously, ThriftMetaData.toString would try to instantiate thriftClassName,
 * but there is no guarantee that that class is on the classpath, and it is in fact
 * normal for that to be the case (for example, when a file was written with TBase objects
 * but is being read with scrooge objects).
 *
 * See PARQUET-345
 */
@Test
public void testToStringDoesNotThrow() {
    StructType descriptor = new StructType(new ArrayList<ThriftField>(), StructOrUnionType.STRUCT);
    ThriftMetaData tmd = new ThriftMetaData("non existent class!!!", descriptor);
    assertEquals("ThriftMetaData(thriftClassName: non existent class!!!, descriptor: {\n" + "  \"id\" : \"STRUCT\",\n" + "  \"children\" : [ ],\n" + "  \"structOrUnionType\" : \"STRUCT\"\n" + "})", tmd.toString());
    tmd = new ThriftMetaData("non existent class!!!", null);
    assertEquals("ThriftMetaData(thriftClassName: non existent class!!!, descriptor: null)", tmd.toString());
}

Example 9 with StructType

use of org.apache.parquet.thrift.struct.ThriftType.StructType in project parquet-mr by apache.

the class TestFieldsPath method testFieldsPath.

@Test
public void testFieldsPath() {
    StructType person = ThriftSchemaConverter.toStructType(Person.class);
    List<String> paths = PrimitivePathVisitor.visit(person, ".");
    assertEquals(Arrays.asList("name.first_name", "name.last_name", "id", "email", "phones.number", "phones.type"), paths);
    paths = PrimitivePathVisitor.visit(person, "/");
    assertEquals(Arrays.asList("name/first_name", "name/last_name", "id", "email", "phones/number", "phones/type"), paths);
    StructType structInMap = ThriftSchemaConverter.toStructType(TestStructInMap.class);
    paths = PrimitivePathVisitor.visit(structInMap, ".");
    assertEquals(Arrays.asList("name", "names.key", "names.value.name.first_name", "names.value.name.last_name", "names.value.phones.key", "names.value.phones.value", "name_to_id.key", "name_to_id.value"), paths);
    paths = PrimitivePathVisitor.visit(structInMap, "/");
    assertEquals(Arrays.asList("name", "names/key", "names/value/name/first_name", "names/value/name/last_name", "names/value/phones/key", "names/value/phones/value", "name_to_id/key", "name_to_id/value"), paths);
}

Example 10 with StructType

use of org.apache.parquet.thrift.struct.ThriftType.StructType in project parquet-mr by apache.

the class TestThriftType method testWriteUnionInfo.

@Test
public void testWriteUnionInfo() throws Exception {
    StructType st = new StructType(new LinkedList<ThriftField>(), null);
    assertEquals("{\n" + "  \"id\" : \"STRUCT\",\n" + "  \"children\" : [ ],\n" + "  \"structOrUnionType\" : \"UNKNOWN\"\n" + "}", st.toJSON());
    st = new StructType(new LinkedList<ThriftField>(), StructOrUnionType.UNION);
    assertEquals("{\n" + "  \"id\" : \"STRUCT\",\n" + "  \"children\" : [ ],\n" + "  \"structOrUnionType\" : \"UNION\"\n" + "}", st.toJSON());
    st = new StructType(new LinkedList<ThriftField>(), StructOrUnionType.STRUCT);
    assertEquals("{\n" + "  \"id\" : \"STRUCT\",\n" + "  \"children\" : [ ],\n" + "  \"structOrUnionType\" : \"STRUCT\"\n" + "}", st.toJSON());
}