use of io.prestosql.plugin.hive.util.MergingPageIterator in project hetu-core by openlookeng.
the class SortingFileWriter method mergeFiles.
private void mergeFiles(Iterable<TempFile> files, Consumer<Page> consumer) {
try (Closer closer = Closer.create()) {
Collection<Iterator<Page>> iterators = new ArrayList<>();
for (TempFile tempFile : files) {
Path file = tempFile.getPath();
FileStatus fileStatus = fileSystem.getFileStatus(file);
OrcDataSource dataSource = new HdfsOrcDataSource(new OrcDataSourceId(file.toString()), fileStatus.getLen(), new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(8, MEGABYTE), false, fileSystem.open(file), new FileFormatDataSourceStats(), fileStatus.getModificationTime());
TempFileReader reader = new TempFileReader(types, dataSource);
// Closing the reader also closes the data source
closer.register(reader);
iterators.add(reader);
}
new MergingPageIterator(iterators, types, sortFields, sortOrders).forEachRemaining(consumer);
for (TempFile tempFile : files) {
Path file = tempFile.getPath();
fileSystem.delete(file, false);
if (fileSystem.exists(file)) {
throw new IOException("Failed to delete temporary file: " + file);
}
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
Aggregations