Examples with CompactingFileWriter org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter used on opensource projects

Example 1 with CompactingFileWriter

use of org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter in project flink by apache.

the class CompactorOperatorTest method createTestBucketWriter.

private BucketWriter<?, String> createTestBucketWriter() {
    return new BucketWriter<Integer, String>() {

        @Override
        public InProgressFileWriter<Integer, String> openNewInProgressFile(String bucketId, Path path, long creationTime) throws IOException {
            return new InProgressFileWriter<Integer, String>() {

                BufferedWriter writer;

                long size = 0L;

                @Override
                public void write(Integer element, long currentTime) throws IOException {
                    if (writer == null) {
                        writer = new BufferedWriter(new FileWriter(path.toString()));
                    }
                    writer.write(element);
                    size += 1;
                }

                @Override
                public InProgressFileRecoverable persist() throws IOException {
                    return new TestInProgressFileRecoverable(path, size);
                }

                @Override
                public PendingFileRecoverable closeForCommit() throws IOException {
                    return new TestPendingFileRecoverable(path, size);
                }

                @Override
                public void dispose() {
                }

                @Override
                public String getBucketId() {
                    return bucketId;
                }

                @Override
                public long getCreationTime() {
                    return 0;
                }

                @Override
                public long getSize() throws IOException {
                    return size;
                }

                @Override
                public long getLastUpdateTime() {
                    return 0;
                }
            };
        }

        @Override
        public InProgressFileWriter<Integer, String> resumeInProgressFileFrom(String s, InProgressFileRecoverable inProgressFileSnapshot, long creationTime) throws IOException {
            return null;
        }

        @Override
        public WriterProperties getProperties() {
            return null;
        }

        @Override
        public PendingFile recoverPendingFile(PendingFileRecoverable pendingFileRecoverable) throws IOException {
            return new PendingFile() {

                @Override
                public void commit() throws IOException {
                    TestPendingFileRecoverable testRecoverable = (TestPendingFileRecoverable) pendingFileRecoverable;
                    if (testRecoverable.getPath() != null) {
                        if (!testRecoverable.getPath().equals(testRecoverable.getUncommittedPath())) {
                            testRecoverable.getPath().getFileSystem().rename(testRecoverable.getUncommittedPath(), testRecoverable.getPath());
                        }
                    }
                }

                @Override
                public void commitAfterRecovery() throws IOException {
                    commit();
                }
            };
        }

        @Override
        public boolean cleanupInProgressFileRecoverable(InProgressFileRecoverable inProgressFileRecoverable) throws IOException {
            return false;
        }

        @Override
        public CompactingFileWriter openNewCompactingFile(CompactingFileWriter.Type type, String bucketId, Path path, long creationTime) throws IOException {
            if (type == CompactingFileWriter.Type.RECORD_WISE) {
                return openNewInProgressFile(bucketId, path, creationTime);
            } else {
                FileOutputStream fileOutputStream = new FileOutputStream(path.toString());
                return new OutputStreamBasedCompactingFileWriter() {

                    @Override
                    public OutputStream asOutputStream() throws IOException {
                        return fileOutputStream;
                    }

                    @Override
                    public PendingFileRecoverable closeForCommit() throws IOException {
                        fileOutputStream.flush();
                        return new TestPendingFileRecoverable(path, fileOutputStream.getChannel().position());
                    }
                };
            }
        }
    };
}

Example 2 with CompactingFileWriter

use of org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter in project flink by apache.

the class CompactService method compact.

@SuppressWarnings({ "rawtypes", "unchecked" })
private Iterable<FileSinkCommittable> compact(CompactorRequest request) throws Exception {
    List<FileSinkCommittable> results = new ArrayList<>(request.getCommittableToPassthrough());
    List<Path> compactingFiles = getCompactingPath(request);
    if (compactingFiles.isEmpty()) {
        return results;
    }
    Path targetPath = assembleCompactedFilePath(compactingFiles.get(0));
    CompactingFileWriter compactingFileWriter = bucketWriter.openNewCompactingFile(compactingWriterType, request.getBucketId(), targetPath, System.currentTimeMillis());
    if (compactingWriterType == Type.RECORD_WISE) {
        ((RecordWiseFileCompactor) fileCompactor).compact(compactingFiles, ((RecordWiseCompactingFileWriter) compactingFileWriter)::write);
    } else if (compactingWriterType == CompactingFileWriter.Type.OUTPUT_STREAM) {
        ((OutputStreamBasedFileCompactor) fileCompactor).compact(compactingFiles, ((OutputStreamBasedCompactingFileWriter) compactingFileWriter).asOutputStream());
    }
    PendingFileRecoverable compactedPendingFile = compactingFileWriter.closeForCommit();
    FileSinkCommittable compacted = new FileSinkCommittable(request.getBucketId(), compactedPendingFile);
    results.add(compacted);
    for (Path f : compactingFiles) {
        // cleanup compacted files
        results.add(new FileSinkCommittable(request.getBucketId(), f));
    }
    return results;
}