Examples with MergingPageIterator com.facebook.presto.hive.util.MergingPageIterator used on opensource projects

Example 1 with MergingPageIterator

use of com.facebook.presto.hive.util.MergingPageIterator in project presto by prestodb.

the class SortingFileWriter method mergeFiles.

private void mergeFiles(Iterable<TempFile> files, Consumer<Page> consumer) {
    try (Closer closer = Closer.create()) {
        Collection<Iterator<Page>> iterators = new ArrayList<>();
        for (TempFile tempFile : files) {
            Path file = tempFile.getPath();
            OrcDataSource dataSource = new HdfsOrcDataSource(new OrcDataSourceId(file.toString()), fileSystem.getFileStatus(file).getLen(), new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(8, MEGABYTE), false, fileSystem.open(file), new FileFormatDataSourceStats());
            closer.register(dataSource);
            iterators.add(new TempFileReader(types, dataSource));
        }
        new MergingPageIterator(iterators, types, sortFields, sortOrders).forEachRemaining(consumer);
        if (!sortedWriteToTempPathEnabled) {
            for (TempFile tempFile : files) {
                Path file = tempFile.getPath();
                fileSystem.delete(file, false);
                if (fileSystem.exists(file)) {
                    throw new IOException("Failed to delete temporary file: " + file);
                }
            }
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}