Examples with GSBlobIdentifier org.apache.flink.fs.gs.storage.GSBlobIdentifier used on opensource projects

Example 16 with GSBlobIdentifier

use of org.apache.flink.fs.gs.storage.GSBlobIdentifier in project flink by apache.

the class GSRecoverableFsDataOutputStreamTest method writeContent.

private void writeContent(ThrowingRunnable<IOException> write, byte[] expectedContent) throws IOException {
    // write the byte, confirm position change and existence of write channel
    assertEquals(position, fsDataOutputStream.getPos());
    write.run();
    assertEquals(position + expectedContent.length, fsDataOutputStream.getPos());
    // close and persist. there should be exactly zero blobs before and one after, with this
    // byte value in it
    assertEquals(0, blobStorage.blobs.size());
    fsDataOutputStream.closeForCommit();
    assertEquals(1, blobStorage.blobs.size());
    GSBlobIdentifier blobIdentifier = blobStorage.blobs.keySet().toArray(new GSBlobIdentifier[0])[0];
    MockBlobStorage.BlobValue blobValue = blobStorage.blobs.get(blobIdentifier);
    assertNotNull(blobValue);
    assertArrayEquals(expectedContent, blobValue.content);
}

Example 17 with GSBlobIdentifier

use of org.apache.flink.fs.gs.storage.GSBlobIdentifier in project flink by apache.

the class GSCommitRecoverableTest method shouldGetComponentBlobIds.

@Test
public void shouldGetComponentBlobIds() {
    // configure options, if this test configuration has a temporary bucket name, set it
    Configuration flinkConfig = new Configuration();
    if (temporaryBucketName != null) {
        flinkConfig.set(GSFileSystemOptions.WRITER_TEMPORARY_BUCKET_NAME, temporaryBucketName);
    }
    GSFileSystemOptions options = new GSFileSystemOptions(flinkConfig);
    GSCommitRecoverable commitRecoverable = new GSCommitRecoverable(blobIdentifier, componentObjectIds);
    List<GSBlobIdentifier> componentBlobIdentifiers = commitRecoverable.getComponentBlobIds(options);
    for (int i = 0; i < componentObjectIds.size(); i++) {
        UUID componentObjectId = componentObjectIds.get(i);
        GSBlobIdentifier componentBlobIdentifier = componentBlobIdentifiers.get(i);
        // if a temporary bucket is specified in options, the component blob identifier
        // should be in this bucket; otherwise, it should be in the bucket with the final blob
        assertEquals(temporaryBucketName == null ? blobIdentifier.bucketName : temporaryBucketName, componentBlobIdentifier.bucketName);
        // make sure the name is what is expected
        String expectedObjectName = String.format(".inprogress/%s/%s/%s", blobIdentifier.bucketName, blobIdentifier.objectName, componentObjectId);
        assertEquals(expectedObjectName, componentBlobIdentifier.objectName);
    }
}

Example 18 with GSBlobIdentifier

use of org.apache.flink.fs.gs.storage.GSBlobIdentifier in project flink by apache.

the class GSRecoverableWriterCommitterTest method before.

@Before
public void before() {
    Configuration flinkConfig = new Configuration();
    if (temporaryBucketName != null) {
        flinkConfig.set(GSFileSystemOptions.WRITER_TEMPORARY_BUCKET_NAME, temporaryBucketName);
    }
    options = new GSFileSystemOptions(flinkConfig);
    random = new Random();
    random.setSeed(TestUtils.RANDOM_SEED);
    blobStorage = new MockBlobStorage();
    blobIdentifier = new GSBlobIdentifier("foo", "bar");
    expectedBytes = new ByteArrayOutputStream();
}

Example 19 with GSBlobIdentifier

use of org.apache.flink.fs.gs.storage.GSBlobIdentifier in project flink by apache.

the class GSResumeRecoverableSerializerTest method shouldSerdeState.

@Test
public void shouldSerdeState() throws IOException {
    // create the state
    GSBlobIdentifier finalBlobIdentifier = new GSBlobIdentifier(bucketName, objectName);
    ArrayList<UUID> componentObjectIds = new ArrayList<>();
    for (int i = 0; i < componentCount; i++) {
        componentObjectIds.add(UUID.randomUUID());
    }
    GSResumeRecoverable state = new GSResumeRecoverable(finalBlobIdentifier, componentObjectIds, position, closed);
    // serialize and deserialize
    GSResumeRecoverableSerializer serializer = GSResumeRecoverableSerializer.INSTANCE;
    byte[] serialized = serializer.serialize(state);
    GSResumeRecoverable deserializedState = (GSResumeRecoverable) serializer.deserialize(serializer.getVersion(), serialized);
    // check that states match
    assertEquals(bucketName, deserializedState.finalBlobIdentifier.bucketName);
    assertEquals(objectName, deserializedState.finalBlobIdentifier.objectName);
    assertEquals(position, deserializedState.position);
    assertEquals(closed, deserializedState.closed);
    assertEquals(componentCount, deserializedState.componentObjectIds.size());
    for (int i = 0; i < componentCount; i++) {
        assertEquals(state.componentObjectIds.get(i), deserializedState.componentObjectIds.get(i));
    }
}

Example 20 with GSBlobIdentifier

use of org.apache.flink.fs.gs.storage.GSBlobIdentifier in project flink by apache.

the class GSRecoverableWriterTest method before.

@Before
public void before() {
    MockBlobStorage storage = new MockBlobStorage();
    blobIdentifier = new GSBlobIdentifier("foo", "bar");
    Configuration flinkConfig = new Configuration();
    options = new GSFileSystemOptions(flinkConfig);
    writer = new GSRecoverableWriter(storage, options);
    componentObjectIds = new ArrayList<UUID>();
    for (int i = 0; i < componentCount; i++) {
        componentObjectIds.add(UUID.randomUUID());
    }
    resumeRecoverable = new GSResumeRecoverable(blobIdentifier, componentObjectIds, position, closed);
    commitRecoverable = new GSCommitRecoverable(blobIdentifier, componentObjectIds);
}