Examples with MockBlobStorage org.apache.flink.fs.gs.storage.MockBlobStorage used on opensource projects

Example 1 with MockBlobStorage

use of org.apache.flink.fs.gs.storage.MockBlobStorage in project flink by apache.

the class GSFileSystemScenarioTest method before.

@Before
public void before() {
    random = new Random(TestUtils.RANDOM_SEED);
    // construct the flink configuration
    flinkConfig = new Configuration();
    if (!StringUtils.isNullOrWhitespaceOnly(temporaryBucketName)) {
        flinkConfig.set(GSFileSystemOptions.WRITER_TEMPORARY_BUCKET_NAME, temporaryBucketName);
    }
    if (writeChunkSize != null) {
        flinkConfig.set(GSFileSystemOptions.WRITER_CHUNK_SIZE, writeChunkSize);
    }
    if (writeChunkSize == null) {
        // unspecified chunk size is valid
        writeChunkSizeIsValid = true;
    } else {
        // chunk size that is > 0 and multiple of 256KB is valid
        long byteCount = writeChunkSize.getBytes();
        writeChunkSizeIsValid = (byteCount > 0) && (byteCount % (256 * 1024) == 0);
    }
    storage = new MockBlobStorage();
    blobIdentifier = new GSBlobIdentifier("foo", "bar");
    path = new Path(String.format("gs://%s/%s", blobIdentifier.bucketName, blobIdentifier.objectName));
}

Example 2 with MockBlobStorage

use of org.apache.flink.fs.gs.storage.MockBlobStorage in project flink by apache.

the class GSRecoverableFsDataOutputStreamTest method before.

@Before
public void before() {
    random = new Random(TestUtils.RANDOM_SEED);
    blobIdentifier = new GSBlobIdentifier("foo", "bar");
    byteValue = (byte) 167;
    Configuration flinkConfig = new Configuration();
    if (temporaryBucketName != null) {
        flinkConfig.set(GSFileSystemOptions.WRITER_TEMPORARY_BUCKET_NAME, temporaryBucketName);
    }
    componentObjectIds = new ArrayList<>();
    for (int i = 0; i < componentObjectCount; i++) {
        componentObjectIds.add(UUID.randomUUID());
    }
    options = new GSFileSystemOptions(flinkConfig);
    blobStorage = new MockBlobStorage();
    if (empty) {
        fsDataOutputStream = new GSRecoverableFsDataOutputStream(blobStorage, options, blobIdentifier);
    } else {
        GSResumeRecoverable resumeRecoverable = new GSResumeRecoverable(blobIdentifier, componentObjectIds, position, closed);
        fsDataOutputStream = new GSRecoverableFsDataOutputStream(blobStorage, options, resumeRecoverable);
    }
}

Example 3 with MockBlobStorage

use of org.apache.flink.fs.gs.storage.MockBlobStorage in project flink by apache.

the class GSChecksumWriteChannelTest method shouldThrowOnChecksumMismatch.

/**
 * Simulate a checksum failure and confirm an exception is thrown.
 *
 * @throws IOException On checksum failure.
 */
@Test(expected = IOException.class)
public void shouldThrowOnChecksumMismatch() throws IOException {
    MockBlobStorage blobStorage = new MockBlobStorage();
    blobStorage.forcedChecksum = "";
    GSBlobStorage.WriteChannel writeChannel = blobStorage.writeBlob(blobIdentifier);
    GSChecksumWriteChannel checksumWriteChannel = new GSChecksumWriteChannel(blobStorage, writeChannel, blobIdentifier);
    // write each partial buffer and validate the written count
    for (int i = 0; i < byteBuffers.length; i++) {
        int writtenCount = checksumWriteChannel.write(byteBuffers[i], writeStarts[i], writeLengths[i]);
        assertEquals(writeLengths[i], writtenCount);
    }
    // close the write, this also validates the checksum
    checksumWriteChannel.close();
}

Example 4 with MockBlobStorage

use of org.apache.flink.fs.gs.storage.MockBlobStorage in project flink by apache.

the class GSChecksumWriteChannelTest method shouldWriteProperly.

/**
 * Write each of the partial byte buffers and confirm we get the expected results, including a
 * valid checksum and the expected data in the storage.
 *
 * @throws IOException On storage failure.
 */
@Test
public void shouldWriteProperly() throws IOException {
    MockBlobStorage blobStorage = new MockBlobStorage();
    GSBlobStorage.WriteChannel writeChannel = blobStorage.writeBlob(blobIdentifier);
    GSChecksumWriteChannel checksumWriteChannel = new GSChecksumWriteChannel(blobStorage, writeChannel, blobIdentifier);
    // write each partial buffer and validate the written count
    for (int i = 0; i < byteBuffers.length; i++) {
        int writtenCount = checksumWriteChannel.write(byteBuffers[i], writeStarts[i], writeLengths[i]);
        assertEquals(writeLengths[i], writtenCount);
    }
    // close the write, this also validates the checksum
    checksumWriteChannel.close();
    // read the value out of storage, the bytes should match
    MockBlobStorage.BlobValue blobValue = blobStorage.blobs.get(blobIdentifier);
    assertArrayEquals(expectedWrittenBytes, blobValue.content);
}

Example 5 with MockBlobStorage

use of org.apache.flink.fs.gs.storage.MockBlobStorage in project flink by apache.

the class GSRecoverableWriterCommitterTest method before.

@Before
public void before() {
    Configuration flinkConfig = new Configuration();
    if (temporaryBucketName != null) {
        flinkConfig.set(GSFileSystemOptions.WRITER_TEMPORARY_BUCKET_NAME, temporaryBucketName);
    }
    options = new GSFileSystemOptions(flinkConfig);
    random = new Random();
    random.setSeed(TestUtils.RANDOM_SEED);
    blobStorage = new MockBlobStorage();
    blobIdentifier = new GSBlobIdentifier("foo", "bar");
    expectedBytes = new ByteArrayOutputStream();
}