Examples with GSBlobIdentifier org.apache.flink.fs.gs.storage.GSBlobIdentifier used on opensource projects

Example 11 with GSBlobIdentifier

use of org.apache.flink.fs.gs.storage.GSBlobIdentifier in project flink by apache.

the class BlobUtilsTest method shouldProperlyConstructTemporaryObjectPartialName.

@Test
public void shouldProperlyConstructTemporaryObjectPartialName() {
    GSBlobIdentifier identifier = new GSBlobIdentifier("foo", "bar");
    String partialName = BlobUtils.getTemporaryObjectPartialName(identifier);
    assertEquals(".inprogress/foo/bar/", partialName);
}

Example 12 with GSBlobIdentifier

use of org.apache.flink.fs.gs.storage.GSBlobIdentifier in project flink by apache.

the class BlobUtilsTest method shouldProperlyConstructTemporaryBlobIdentifierWithDefaultBucket.

@Test
public void shouldProperlyConstructTemporaryBlobIdentifierWithDefaultBucket() {
    Configuration flinkConfig = new Configuration();
    GSFileSystemOptions options = new GSFileSystemOptions(flinkConfig);
    GSBlobIdentifier identifier = new GSBlobIdentifier("foo", "bar");
    UUID temporaryObjectId = UUID.fromString("f09c43e5-ea49-4537-a406-0586f8f09d47");
    GSBlobIdentifier temporaryBlobIdentifier = BlobUtils.getTemporaryBlobIdentifier(identifier, temporaryObjectId, options);
    assertEquals("foo", temporaryBlobIdentifier.bucketName);
    assertEquals(".inprogress/foo/bar/f09c43e5-ea49-4537-a406-0586f8f09d47", temporaryBlobIdentifier.objectName);
}

Example 13 with GSBlobIdentifier

use of org.apache.flink.fs.gs.storage.GSBlobIdentifier in project flink by apache.

the class BlobUtilsTest method shouldUseTemporaryBucketNameIfSpecified.

@Test
public void shouldUseTemporaryBucketNameIfSpecified() {
    Configuration flinkConfig = new Configuration();
    flinkConfig.set(GSFileSystemOptions.WRITER_TEMPORARY_BUCKET_NAME, "temp");
    GSFileSystemOptions options = new GSFileSystemOptions(flinkConfig);
    GSBlobIdentifier identifier = new GSBlobIdentifier("foo", "bar");
    String bucketName = BlobUtils.getTemporaryBucketName(identifier, options);
    assertEquals("temp", bucketName);
}

Example 14 with GSBlobIdentifier

use of org.apache.flink.fs.gs.storage.GSBlobIdentifier in project flink by apache.

the class GSCommitRecoverableSerializerTest method shouldSerdeState.

@Test
public void shouldSerdeState() throws IOException {
    // create the state
    GSBlobIdentifier finalBlobIdentifier = new GSBlobIdentifier(bucketName, objectName);
    ArrayList<UUID> componentObjectIds = new ArrayList<>();
    for (int i = 0; i < componentCount; i++) {
        componentObjectIds.add(UUID.randomUUID());
    }
    GSCommitRecoverable state = new GSCommitRecoverable(finalBlobIdentifier, componentObjectIds);
    // serialize and deserialize
    GSCommitRecoverableSerializer serializer = GSCommitRecoverableSerializer.INSTANCE;
    byte[] serialized = serializer.serialize(state);
    GSCommitRecoverable deserializedState = (GSCommitRecoverable) serializer.deserialize(serializer.getVersion(), serialized);
    // check that states match
    assertEquals(bucketName, deserializedState.finalBlobIdentifier.bucketName);
    assertEquals(objectName, deserializedState.finalBlobIdentifier.objectName);
    assertEquals(componentCount, deserializedState.componentObjectIds.size());
    for (int i = 0; i < componentCount; i++) {
        assertEquals(state.componentObjectIds.get(i), deserializedState.componentObjectIds.get(i));
    }
}

Example 15 with GSBlobIdentifier

use of org.apache.flink.fs.gs.storage.GSBlobIdentifier in project flink by apache.

the class GSRecoverableFsDataOutputStreamTest method before.

@Before
public void before() {
    random = new Random(TestUtils.RANDOM_SEED);
    blobIdentifier = new GSBlobIdentifier("foo", "bar");
    byteValue = (byte) 167;
    Configuration flinkConfig = new Configuration();
    if (temporaryBucketName != null) {
        flinkConfig.set(GSFileSystemOptions.WRITER_TEMPORARY_BUCKET_NAME, temporaryBucketName);
    }
    componentObjectIds = new ArrayList<>();
    for (int i = 0; i < componentObjectCount; i++) {
        componentObjectIds.add(UUID.randomUUID());
    }
    options = new GSFileSystemOptions(flinkConfig);
    blobStorage = new MockBlobStorage();
    if (empty) {
        fsDataOutputStream = new GSRecoverableFsDataOutputStream(blobStorage, options, blobIdentifier);
    } else {
        GSResumeRecoverable resumeRecoverable = new GSResumeRecoverable(blobIdentifier, componentObjectIds, position, closed);
        fsDataOutputStream = new GSRecoverableFsDataOutputStream(blobStorage, options, resumeRecoverable);
    }
}