Examples with GSFileSystemOptions org.apache.flink.fs.gs.GSFileSystemOptions used on opensource projects

Example 1 with GSFileSystemOptions

use of org.apache.flink.fs.gs.GSFileSystemOptions in project flink by apache.

the class BlobUtilsTest method shouldUseIdentifierBucketNameNameIfTemporaryBucketNotSpecified.

@Test
public void shouldUseIdentifierBucketNameNameIfTemporaryBucketNotSpecified() {
    Configuration flinkConfig = new Configuration();
    GSFileSystemOptions options = new GSFileSystemOptions(flinkConfig);
    GSBlobIdentifier identifier = new GSBlobIdentifier("foo", "bar");
    String bucketName = BlobUtils.getTemporaryBucketName(identifier, options);
    assertEquals("foo", bucketName);
}

Example 2 with GSFileSystemOptions

use of org.apache.flink.fs.gs.GSFileSystemOptions in project flink by apache.

the class BlobUtilsTest method shouldProperlyConstructTemporaryBlobIdentifierWithDefaultBucket.

@Test
public void shouldProperlyConstructTemporaryBlobIdentifierWithDefaultBucket() {
    Configuration flinkConfig = new Configuration();
    GSFileSystemOptions options = new GSFileSystemOptions(flinkConfig);
    GSBlobIdentifier identifier = new GSBlobIdentifier("foo", "bar");
    UUID temporaryObjectId = UUID.fromString("f09c43e5-ea49-4537-a406-0586f8f09d47");
    GSBlobIdentifier temporaryBlobIdentifier = BlobUtils.getTemporaryBlobIdentifier(identifier, temporaryObjectId, options);
    assertEquals("foo", temporaryBlobIdentifier.bucketName);
    assertEquals(".inprogress/foo/bar/f09c43e5-ea49-4537-a406-0586f8f09d47", temporaryBlobIdentifier.objectName);
}

Example 3 with GSFileSystemOptions

use of org.apache.flink.fs.gs.GSFileSystemOptions in project flink by apache.

the class BlobUtilsTest method shouldUseTemporaryBucketNameIfSpecified.

@Test
public void shouldUseTemporaryBucketNameIfSpecified() {
    Configuration flinkConfig = new Configuration();
    flinkConfig.set(GSFileSystemOptions.WRITER_TEMPORARY_BUCKET_NAME, "temp");
    GSFileSystemOptions options = new GSFileSystemOptions(flinkConfig);
    GSBlobIdentifier identifier = new GSBlobIdentifier("foo", "bar");
    String bucketName = BlobUtils.getTemporaryBucketName(identifier, options);
    assertEquals("temp", bucketName);
}

Example 4 with GSFileSystemOptions

use of org.apache.flink.fs.gs.GSFileSystemOptions in project flink by apache.

the class GSRecoverableFsDataOutputStreamTest method before.

@Before
public void before() {
    random = new Random(TestUtils.RANDOM_SEED);
    blobIdentifier = new GSBlobIdentifier("foo", "bar");
    byteValue = (byte) 167;
    Configuration flinkConfig = new Configuration();
    if (temporaryBucketName != null) {
        flinkConfig.set(GSFileSystemOptions.WRITER_TEMPORARY_BUCKET_NAME, temporaryBucketName);
    }
    componentObjectIds = new ArrayList<>();
    for (int i = 0; i < componentObjectCount; i++) {
        componentObjectIds.add(UUID.randomUUID());
    }
    options = new GSFileSystemOptions(flinkConfig);
    blobStorage = new MockBlobStorage();
    if (empty) {
        fsDataOutputStream = new GSRecoverableFsDataOutputStream(blobStorage, options, blobIdentifier);
    } else {
        GSResumeRecoverable resumeRecoverable = new GSResumeRecoverable(blobIdentifier, componentObjectIds, position, closed);
        fsDataOutputStream = new GSRecoverableFsDataOutputStream(blobStorage, options, resumeRecoverable);
    }
}

Example 5 with GSFileSystemOptions

use of org.apache.flink.fs.gs.GSFileSystemOptions in project flink by apache.

the class GSCommitRecoverableTest method shouldGetComponentBlobIds.

@Test
public void shouldGetComponentBlobIds() {
    // configure options, if this test configuration has a temporary bucket name, set it
    Configuration flinkConfig = new Configuration();
    if (temporaryBucketName != null) {
        flinkConfig.set(GSFileSystemOptions.WRITER_TEMPORARY_BUCKET_NAME, temporaryBucketName);
    }
    GSFileSystemOptions options = new GSFileSystemOptions(flinkConfig);
    GSCommitRecoverable commitRecoverable = new GSCommitRecoverable(blobIdentifier, componentObjectIds);
    List<GSBlobIdentifier> componentBlobIdentifiers = commitRecoverable.getComponentBlobIds(options);
    for (int i = 0; i < componentObjectIds.size(); i++) {
        UUID componentObjectId = componentObjectIds.get(i);
        GSBlobIdentifier componentBlobIdentifier = componentBlobIdentifiers.get(i);
        // if a temporary bucket is specified in options, the component blob identifier
        // should be in this bucket; otherwise, it should be in the bucket with the final blob
        assertEquals(temporaryBucketName == null ? blobIdentifier.bucketName : temporaryBucketName, componentBlobIdentifier.bucketName);
        // make sure the name is what is expected
        String expectedObjectName = String.format(".inprogress/%s/%s/%s", blobIdentifier.bucketName, blobIdentifier.objectName, componentObjectId);
        assertEquals(expectedObjectName, componentBlobIdentifier.objectName);
    }
}