use of org.apache.orc.storage.ql.exec.vector.ListColumnVector in project incubator-gobblin by apache.
the class GobblinBaseOrcWriter method removeRefOfColumnVectorChild.
/**
* Set the child field of {@link ColumnVector} to null, assuming input {@link ColumnVector} is nonNull.
*/
private void removeRefOfColumnVectorChild(ColumnVector cv) {
if (cv instanceof StructColumnVector) {
StructColumnVector structCv = (StructColumnVector) cv;
for (ColumnVector childCv : structCv.fields) {
removeRefOfColumnVectorChild(childCv);
}
} else if (cv instanceof ListColumnVector) {
ListColumnVector listCv = (ListColumnVector) cv;
removeRefOfColumnVectorChild(listCv.child);
} else if (cv instanceof MapColumnVector) {
MapColumnVector mapCv = (MapColumnVector) cv;
removeRefOfColumnVectorChild(mapCv.keys);
removeRefOfColumnVectorChild(mapCv.values);
} else if (cv instanceof UnionColumnVector) {
UnionColumnVector unionCv = (UnionColumnVector) cv;
for (ColumnVector unionChildCv : unionCv.fields) {
removeRefOfColumnVectorChild(unionChildCv);
}
} else if (cv instanceof LongColumnVector) {
((LongColumnVector) cv).vector = null;
} else if (cv instanceof DoubleColumnVector) {
((DoubleColumnVector) cv).vector = null;
} else if (cv instanceof BytesColumnVector) {
((BytesColumnVector) cv).vector = null;
((BytesColumnVector) cv).start = null;
((BytesColumnVector) cv).length = null;
} else if (cv instanceof DecimalColumnVector) {
((DecimalColumnVector) cv).vector = null;
}
}
use of org.apache.orc.storage.ql.exec.vector.ListColumnVector in project incubator-gobblin by apache.
the class GobblinOrcWriterTest method testRowBatchDeepClean.
@Test
public void testRowBatchDeepClean() throws Exception {
Schema schema = new Schema.Parser().parse(this.getClass().getClassLoader().getResourceAsStream("orc_writer_list_test/schema.avsc"));
List<GenericRecord> recordList = deserializeAvroRecords(this.getClass(), schema, "orc_writer_list_test/data.json");
// Mock WriterBuilder, bunch of mocking behaviors to work-around precondition checks in writer builder
FsDataWriterBuilder<Schema, GenericRecord> mockBuilder = (FsDataWriterBuilder<Schema, GenericRecord>) Mockito.mock(FsDataWriterBuilder.class);
when(mockBuilder.getSchema()).thenReturn(schema);
State dummyState = new WorkUnit();
String stagingDir = Files.createTempDir().getAbsolutePath();
String outputDir = Files.createTempDir().getAbsolutePath();
dummyState.setProp(ConfigurationKeys.WRITER_STAGING_DIR, stagingDir);
dummyState.setProp(ConfigurationKeys.WRITER_FILE_PATH, "simple");
dummyState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, outputDir);
dummyState.setProp("orcWriter.deepCleanBatch", "true");
when(mockBuilder.getFileName(dummyState)).thenReturn("file");
Closer closer = Closer.create();
GobblinOrcWriter orcWriter = closer.register(new GobblinOrcWriter(mockBuilder, dummyState));
for (GenericRecord genericRecord : recordList) {
orcWriter.write(genericRecord);
}
// Manual trigger flush
orcWriter.flush();
Assert.assertNull(((BytesColumnVector) ((ListColumnVector) orcWriter.rowBatch.cols[0]).child).vector);
Assert.assertNull(((BytesColumnVector) orcWriter.rowBatch.cols[1]).vector);
}
Aggregations