Examples with RecoverableFsDataOutputStream org.apache.flink.core.fs.RecoverableFsDataOutputStream used on opensource projects

Example 11 with RecoverableFsDataOutputStream

use of org.apache.flink.core.fs.RecoverableFsDataOutputStream in project flink by apache.

the class HadoopS3RecoverableWriterITCase method testCommitAfterRecovery.

// ----------------------- Test Recovery -----------------------
@Test
public void testCommitAfterRecovery() throws Exception {
    final Path path = new Path(basePathForTest, "part-0");
    final RecoverableWriter initWriter = getRecoverableWriter();
    final RecoverableFsDataOutputStream stream = initWriter.open(path);
    stream.write(bytesOf(testData1));
    stream.persist();
    stream.persist();
    // and write some more data
    stream.write(bytesOf(testData2));
    final RecoverableWriter.CommitRecoverable recoverable = stream.closeForCommit().getRecoverable();
    final byte[] serializedRecoverable = initWriter.getCommitRecoverableSerializer().serialize(recoverable);
    // get a new serializer from a new writer to make sure that no pre-initialized state leaks
    // in.
    final RecoverableWriter newWriter = getRecoverableWriter();
    final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> deserializer = newWriter.getCommitRecoverableSerializer();
    final RecoverableWriter.CommitRecoverable recoveredRecoverable = deserializer.deserialize(deserializer.getVersion(), serializedRecoverable);
    final RecoverableFsDataOutputStream.Committer committer = newWriter.recoverForCommit(recoveredRecoverable);
    committer.commitAfterRecovery();
    Assert.assertEquals(testData1 + testData2, getContentsOfFile(path));
}

Example 12 with RecoverableFsDataOutputStream

use of org.apache.flink.core.fs.RecoverableFsDataOutputStream in project flink by apache.

the class HadoopS3RecoverableWriterITCase method testCallingDeleteObjectTwiceDoesNotThroughException.

@Test
public void testCallingDeleteObjectTwiceDoesNotThroughException() throws Exception {
    final RecoverableWriter writer = getRecoverableWriter();
    final Path path = new Path(basePathForTest, "part-0");
    final RecoverableFsDataOutputStream stream = writer.open(path);
    stream.write(bytesOf(testData1));
    S3Recoverable recoverable = (S3Recoverable) stream.persist();
    stream.closeForCommit().commit();
    // still the data is there as we have not deleted them from the tmp object
    final String content = getContentsOfFile(new Path('/' + recoverable.incompleteObjectName()));
    Assert.assertEquals(testData1, content);
    boolean successfullyDeletedState = writer.cleanupRecoverableState(recoverable);
    Assert.assertTrue(successfullyDeletedState);
    boolean unsuccessfulDeletion = writer.cleanupRecoverableState(recoverable);
    Assert.assertFalse(unsuccessfulDeletion);
}

Example 13 with RecoverableFsDataOutputStream

use of org.apache.flink.core.fs.RecoverableFsDataOutputStream in project flink by apache.

the class GSFileSystemScenarioTest method simpleWriteTest.

/* Test writing a single array of bytes to a stream. */
@Test
public void simpleWriteTest() throws IOException {
    // only run the test for valid chunk sizes
    assumeTrue(writeChunkSizeIsValid);
    // create the options and writer
    GSFileSystemOptions options = new GSFileSystemOptions(flinkConfig);
    RecoverableWriter writer = new GSRecoverableWriter(storage, options);
    // create a stream and write some random bytes to it
    RecoverableFsDataOutputStream stream = writer.open(path);
    byte[] data = new byte[128];
    random.nextBytes(data);
    stream.write(data);
    // close for commit
    RecoverableFsDataOutputStream.Committer committer = stream.closeForCommit();
    // there should be a single blob now, in the specified temporary bucket or, if no temporary
    // bucket
    // specified, in the final bucket
    assertEquals(1, storage.blobs.size());
    GSBlobIdentifier temporaryBlobIdentifier = (GSBlobIdentifier) storage.blobs.keySet().toArray()[0];
    String expectedTemporaryBucket = StringUtils.isNullOrWhitespaceOnly(temporaryBucketName) ? blobIdentifier.bucketName : temporaryBucketName;
    assertEquals(expectedTemporaryBucket, temporaryBlobIdentifier.bucketName);
    // commit
    committer.commit();
    // there should be exactly one blob after commit, with the expected contents.
    // all temporary blobs should be removed.
    assertEquals(1, storage.blobs.size());
    MockBlobStorage.BlobValue blobValue = storage.blobs.get(blobIdentifier);
    assertArrayEquals(data, blobValue.content);
}

Example 14 with RecoverableFsDataOutputStream

use of org.apache.flink.core.fs.RecoverableFsDataOutputStream in project flink by apache.

the class OutputStreamBasedPartFileRecoverableMigrationTest method prepareDeserializationPending.

@Test
@Ignore
public void prepareDeserializationPending() throws IOException {
    String scenario = "pending";
    java.nio.file.Path path = resolveVersionPath(CURRENT_VERSION, scenario);
    RecoverableWriter writer = FileSystem.getLocalFileSystem().createRecoverableWriter();
    OutputStreamBasedPendingFileRecoverableSerializer serializer = new OutputStreamBasedPendingFileRecoverableSerializer(writer.getCommitRecoverableSerializer());
    RecoverableFsDataOutputStream outputStream = writer.open(new Path(path.resolve("content").toString()));
    outputStream.write(PENDING_CONTENT.getBytes(StandardCharsets.UTF_8));
    CommitRecoverable commitRecoverable = outputStream.closeForCommit().getRecoverable();
    OutputStreamBasedPendingFileRecoverable recoverable = new OutputStreamBasedPendingFileRecoverable(commitRecoverable);
    byte[] bytes = serializer.serialize(recoverable);
    Files.write(path.resolve("recoverable"), bytes);
}

Example 15 with RecoverableFsDataOutputStream

use of org.apache.flink.core.fs.RecoverableFsDataOutputStream in project flink by apache.

the class GSFileSystemScenarioTest method compoundWriteTest.

/* Test writing multiple arrays of bytes to a stream. */
@Test
public void compoundWriteTest() throws IOException {
    // only run the test for valid chunk sizes
    assumeTrue(writeChunkSizeIsValid);
    // create the options and writer
    GSFileSystemOptions options = new GSFileSystemOptions(flinkConfig);
    RecoverableWriter writer = new GSRecoverableWriter(storage, options);
    // create a stream
    RecoverableFsDataOutputStream stream = writer.open(path);
    // write 10 arrays of bytes
    final int writeCount = 10;
    // write multiple arrays of bytes to it
    try (ByteArrayOutputStream expectedData = new ByteArrayOutputStream()) {
        for (int i = 0; i < writeCount; i++) {
            byte[] data = new byte[128];
            random.nextBytes(data);
            stream.write(data);
            expectedData.write(data);
        }
        // close for commit and commit
        RecoverableFsDataOutputStream.Committer committer = stream.closeForCommit();
        committer.commit();
        // there should be exactly one blob after commit, with the expected contents.
        // all temporary blobs should be removed.
        assertEquals(1, storage.blobs.size());
        MockBlobStorage.BlobValue blobValue = storage.blobs.get(blobIdentifier);
        assertArrayEquals(expectedData.toByteArray(), blobValue.content);
    }
}