Examples with MemPageStore org.apache.parquet.column.page.mem.MemPageStore used on opensource projects

Example 26 with MemPageStore

use of org.apache.parquet.column.page.mem.MemPageStore in project parquet-mr by apache.

the class TestParquetReadProtocol method validate.

private <T extends TBase<?, ?>> void validate(T expected) throws TException {
    @SuppressWarnings("unchecked") final Class<T> thriftClass = (Class<T>) expected.getClass();
    final MemPageStore memPageStore = new MemPageStore(1);
    final ThriftSchemaConverter schemaConverter = new ThriftSchemaConverter();
    final MessageType schema = schemaConverter.convert(thriftClass);
    LOG.info("{}", schema);
    final MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
    final ColumnWriteStoreV1 columns = new ColumnWriteStoreV1(memPageStore, ParquetProperties.builder().withPageSize(10000).withDictionaryEncoding(false).build());
    final RecordConsumer recordWriter = columnIO.getRecordWriter(columns);
    final StructType thriftType = schemaConverter.toStructType(thriftClass);
    ParquetWriteProtocol parquetWriteProtocol = new ParquetWriteProtocol(recordWriter, columnIO, thriftType);
    expected.write(parquetWriteProtocol);
    recordWriter.flush();
    columns.flush();
    ThriftRecordConverter<T> converter = new TBaseRecordConverter<T>(thriftClass, schema, thriftType);
    final RecordReader<T> recordReader = columnIO.getRecordReader(memPageStore, converter);
    final T result = recordReader.read();
    assertEquals(expected, result);
}