Examples with PartitioningWriter org.apache.iceberg.io.PartitioningWriter used on opensource projects

Example 1 with PartitioningWriter

use of org.apache.iceberg.io.PartitioningWriter in project hive by apache.

the class HiveIcebergBufferedDeleteWriter method close.

@Override
public void close(boolean abort) throws IOException {
    long startTime = System.currentTimeMillis();
    Collection<DeleteFile> deleteFiles = new ConcurrentLinkedQueue<>();
    if (!abort) {
        LOG.info("Delete file flush is started");
        int size = Math.min(buffer.size(), poolSize);
        ExecutorService fileExecutor = fileExecutor(size);
        try {
            Tasks.foreach(buffer.keySet()).retry(3).executeWith(fileExecutor).onFailure((partition, exception) -> LOG.info("Failed to write delete file {}", partition, exception)).run(partition -> {
                PositionDelete<Record> positionDelete = PositionDelete.create();
                PartitioningWriter writerForFiles;
                try (PartitioningWriter writer = new ClusteredPositionDeleteWriter<>(writerFactory, fileFactory, io, format, targetFileSize)) {
                    Map<String, Roaring64Bitmap> deleteRows = buffer.get(partition);
                    for (String filePath : new TreeSet<>(deleteRows.keySet())) {
                        Roaring64Bitmap deletes = deleteRows.get(filePath);
                        PeekableLongIterator longIterator = deletes.getLongIterator();
                        while (longIterator.hasNext()) {
                            long position = longIterator.next();
                            positionDelete.set(filePath, position, null);
                            writer.write(positionDelete, keyToSpec.get(partition), partition);
                        }
                    }
                    // We need the writer object later to get the generated data files
                    writerForFiles = writer;
                }
                deleteFiles.addAll(((DeleteWriteResult) writerForFiles.result()).deleteFiles());
            }, IOException.class);
        } finally {
            fileExecutor.shutdown();
        }
    }
    LOG.info("HiveIcebergBufferedDeleteWriter is closed with abort={}. Created {} delete files and it took {} ns.", abort, deleteFiles.size(), System.currentTimeMillis() - startTime);
    LOG.debug("Delete files written {}", deleteFiles);
    this.filesForCommit = FilesForCommit.onlyDelete(deleteFiles);
}