Examples with ParquetOutputFormat org.apache.parquet.hadoop.ParquetOutputFormat used on opensource projects

Example 1 with ParquetOutputFormat

use of org.apache.parquet.hadoop.ParquetOutputFormat in project parquet-mr by apache.

the class TestMapredParquetOutputFormat method testGetHiveRecordWriter.

@SuppressWarnings("unchecked")
@Test
public void testGetHiveRecordWriter() throws IOException {
    Properties tableProps = new Properties();
    tableProps.setProperty("columns", "foo,bar");
    tableProps.setProperty("columns.types", "int:int");
    final Progressable mockProgress = mock(Progressable.class);
    final ParquetOutputFormat<ArrayWritable> outputFormat = (ParquetOutputFormat<ArrayWritable>) mock(ParquetOutputFormat.class);
    JobConf jobConf = new JobConf();
    try {
        new MapredParquetOutputFormat(outputFormat) {

            @Override
            protected ParquetRecordWriterWrapper getParquerRecordWriterWrapper(ParquetOutputFormat<ArrayWritable> realOutputFormat, JobConf jobConf, String finalOutPath, Progressable progress) throws IOException {
                assertEquals(outputFormat, realOutputFormat);
                assertNotNull(jobConf.get(DataWritableWriteSupport.PARQUET_HIVE_SCHEMA));
                assertEquals("/foo", finalOutPath.toString());
                assertEquals(mockProgress, progress);
                throw new RuntimeException("passed tests");
            }
        }.getHiveRecordWriter(jobConf, new Path("/foo"), null, false, tableProps, mockProgress);
        fail("should throw runtime exception.");
    } catch (RuntimeException e) {
        assertEquals("passed tests", e.getMessage());
    }
}

Example 2 with ParquetOutputFormat

use of org.apache.parquet.hadoop.ParquetOutputFormat in project hive by apache.

the class TestMapredParquetOutputFormat method testGetHiveRecordWriter.

@SuppressWarnings("unchecked")
@Test
public void testGetHiveRecordWriter() throws IOException {
    Properties tableProps = new Properties();
    tableProps.setProperty("columns", "foo,bar");
    tableProps.setProperty("columns.types", "int:int");
    final Progressable mockProgress = mock(Progressable.class);
    final ParquetOutputFormat<ParquetHiveRecord> outputFormat = (ParquetOutputFormat<ParquetHiveRecord>) mock(ParquetOutputFormat.class);
    JobConf jobConf = new JobConf();
    try {
        new MapredParquetOutputFormat(outputFormat) {

            @Override
            protected ParquetRecordWriterWrapper getParquerRecordWriterWrapper(ParquetOutputFormat<ParquetHiveRecord> realOutputFormat, JobConf jobConf, String finalOutPath, Progressable progress, Properties tableProperties) throws IOException {
                assertEquals(outputFormat, realOutputFormat);
                assertNotNull(jobConf.get(DataWritableWriteSupport.PARQUET_HIVE_SCHEMA));
                assertEquals("/foo", finalOutPath.toString());
                assertEquals(mockProgress, progress);
                throw new RuntimeException("passed tests");
            }
        }.getHiveRecordWriter(jobConf, new Path("/foo"), null, false, tableProps, mockProgress);
        fail("should throw runtime exception.");
    } catch (RuntimeException e) {
        assertEquals("passed tests", e.getMessage());
    }
}