Examples with PERMANENT_BLOB org.apache.flink.runtime.blob.BlobKey.BlobType.PERMANENT_BLOB used on opensource projects

Example 1 with PERMANENT_BLOB

use of org.apache.flink.runtime.blob.BlobKey.BlobType.PERMANENT_BLOB in project flink by apache.

the class BlobServerPutTest method testConcurrentPutOperations.

/**
 * [FLINK-6020] Tests that concurrent put operations will only upload the file once to the
 * {@link BlobStore} and that the files are not corrupt at any time.
 *
 * @param jobId job ID to use (or <tt>null</tt> if job-unrelated)
 * @param blobType whether the BLOB should become permanent or transient
 */
private void testConcurrentPutOperations(@Nullable final JobID jobId, final BlobKey.BlobType blobType) throws IOException, InterruptedException, ExecutionException {
    final Configuration config = new Configuration();
    final int concurrentPutOperations = 2;
    final int dataSize = 1024;
    Collection<BlobKey> persistedBlobs = ConcurrentHashMap.newKeySet();
    TestingBlobStore blobStore = new TestingBlobStoreBuilder().setPutFunction((file, jobID, blobKey) -> {
        persistedBlobs.add(blobKey);
        return true;
    }).createTestingBlobStore();
    final CountDownLatch countDownLatch = new CountDownLatch(concurrentPutOperations);
    final byte[] data = new byte[dataSize];
    ArrayList<CompletableFuture<BlobKey>> allFutures = new ArrayList<>(concurrentPutOperations);
    ExecutorService executor = Executors.newFixedThreadPool(concurrentPutOperations);
    try (final BlobServer server = new BlobServer(config, temporaryFolder.newFolder(), blobStore)) {
        server.start();
        for (int i = 0; i < concurrentPutOperations; i++) {
            CompletableFuture<BlobKey> putFuture = CompletableFuture.supplyAsync(() -> {
                try {
                    BlockingInputStream inputStream = new BlockingInputStream(countDownLatch, data);
                    BlobKey uploadedKey = put(server, jobId, inputStream, blobType);
                    // check the uploaded file's contents (concurrently)
                    verifyContents(server, jobId, uploadedKey, data);
                    return uploadedKey;
                } catch (IOException e) {
                    throw new CompletionException(new FlinkException("Could not upload blob.", e));
                }
            }, executor);
            allFutures.add(putFuture);
        }
        FutureUtils.ConjunctFuture<Collection<BlobKey>> conjunctFuture = FutureUtils.combineAll(allFutures);
        // wait until all operations have completed and check that no exception was thrown
        Collection<BlobKey> blobKeys = conjunctFuture.get();
        Iterator<BlobKey> blobKeyIterator = blobKeys.iterator();
        assertTrue(blobKeyIterator.hasNext());
        BlobKey blobKey = blobKeyIterator.next();
        // make sure that all blob keys are the same
        while (blobKeyIterator.hasNext()) {
            verifyKeyDifferentHashEquals(blobKey, blobKeyIterator.next());
        }
        // check the uploaded file's contents
        verifyContents(server, jobId, blobKey, data);
        // check that we only uploaded the file once to the blob store
        if (blobType == PERMANENT_BLOB) {
            assertThat(persistedBlobs).hasSameElementsAs(blobKeys);
        } else {
            // can't really verify much in the other cases other than that the put operations
            // should
            // work and not corrupt files
            assertThat(persistedBlobs).isEmpty();
        }
    } finally {
        executor.shutdownNow();
    }
}

Example 2 with PERMANENT_BLOB

use of org.apache.flink.runtime.blob.BlobKey.BlobType.PERMANENT_BLOB in project flink by apache.

the class BlobServerPutTest method testFailedBlobStorePutsDeletesLocalBlob.

@Test
public void testFailedBlobStorePutsDeletesLocalBlob() throws IOException {
    final BlobKey.BlobType blobType = PERMANENT_BLOB;
    final JobID jobId = JobID.generate();
    final byte[] data = new byte[] { 1, 2, 3 };
    final File storageDir = temporaryFolder.newFolder();
    final TestingBlobStore blobStore = new TestingBlobStoreBuilder().setPutFunction((file, jobID, blobKey) -> {
        throw new IOException("Could not persist the file.");
    }).createTestingBlobStore();
    try (final BlobServer blobServer = new BlobServer(new Configuration(), storageDir, blobStore)) {
        try {
            put(blobServer, jobId, data, blobType);
            fail("Expected that the put operation fails with an IOException.");
        } catch (IOException expected) {
        // expected :-)
        }
        final File jobSpecificStorageDirectory = new File(BlobUtils.getStorageLocationPath(storageDir.getAbsolutePath(), jobId));
        assertThat(jobSpecificStorageDirectory).isEmptyDirectory();
    }
}

Example 3 with PERMANENT_BLOB

use of org.apache.flink.runtime.blob.BlobKey.BlobType.PERMANENT_BLOB in project flink by apache.

the class BlobServerGetTest method testGetReDownloadsCorruptedPermanentBlobFromBlobStoreInCaseOfRestart.

@Test
public void testGetReDownloadsCorruptedPermanentBlobFromBlobStoreInCaseOfRestart() throws IOException {
    final JobID jobId = JobID.generate();
    final byte[] data = new byte[] { 1, 2, 3 };
    final byte[] corruptedData = new byte[] { 3, 2, 1 };
    final File storageDir = temporaryFolder.newFolder();
    final OneShotLatch getCalled = new OneShotLatch();
    final BlobStore blobStore = new TestingBlobStoreBuilder().setGetFunction((jobID, blobKey, file) -> {
        getCalled.trigger();
        FileUtils.writeByteArrayToFile(file, data);
        return true;
    }).createTestingBlobStore();
    try (final BlobServer blobServer = new BlobServer(new Configuration(), Reference.borrowed(storageDir), blobStore)) {
        final BlobKey blobKey = put(blobServer, jobId, data, PERMANENT_BLOB);
        blobServer.close();
        final File blob = blobServer.getStorageLocation(jobId, blobKey);
        // corrupt the file
        FileUtils.writeByteArrayToFile(blob, corruptedData);
        try (final BlobServer restartedBlobServer = new BlobServer(new Configuration(), Reference.borrowed(storageDir), blobStore)) {
            // we should re-download the file from the BlobStore
            final File file = get(restartedBlobServer, jobId, blobKey);
            validateGetAndClose(new FileInputStream(file), data);
            assertThat(getCalled.isTriggered()).isTrue();
        }
    }
}

Example 4 with PERMANENT_BLOB

use of org.apache.flink.runtime.blob.BlobKey.BlobType.PERMANENT_BLOB in project flink by apache.

the class BlobServerGetTest method testConcurrentGetOperations.

/**
 * [FLINK-6020] Tests that concurrent get operations don't concurrently access the BlobStore to
 * download a blob.
 *
 * @param jobId job ID to use (or <tt>null</tt> if job-unrelated)
 * @param blobType whether the BLOB should become permanent or transient
 */
private void testConcurrentGetOperations(@Nullable final JobID jobId, final BlobKey.BlobType blobType) throws IOException, InterruptedException, ExecutionException {
    final byte[] data = { 1, 2, 3, 4, 99, 42 };
    final BlobStore blobStore = new TestingBlobStoreBuilder().setGetFunction((jobID, blobKey, file) -> {
        FileUtils.writeByteArrayToFile(file, data);
        return true;
    }).createTestingBlobStore();
    final int numberConcurrentGetOperations = 3;
    final List<CompletableFuture<File>> getOperations = new ArrayList<>(numberConcurrentGetOperations);
    final ExecutorService executor = Executors.newFixedThreadPool(numberConcurrentGetOperations);
    try (final BlobServer server = new BlobServer(new Configuration(), temporaryFolder.newFolder(), blobStore)) {
        server.start();
        // upload data first
        final BlobKey blobKey = put(server, jobId, data, blobType);
        // store!)
        if (blobType == PERMANENT_BLOB) {
            // remove local copy so that a transfer from HA store takes place
            assertTrue(server.getStorageLocation(jobId, blobKey).delete());
        }
        for (int i = 0; i < numberConcurrentGetOperations; i++) {
            CompletableFuture<File> getOperation = CompletableFuture.supplyAsync(() -> {
                try {
                    File file = get(server, jobId, blobKey);
                    // check that we have read the right data
                    validateGetAndClose(new FileInputStream(file), data);
                    return file;
                } catch (IOException e) {
                    throw new CompletionException(new FlinkException("Could not read blob for key " + blobKey + '.', e));
                }
            }, executor);
            getOperations.add(getOperation);
        }
        CompletableFuture<Collection<File>> filesFuture = FutureUtils.combineAll(getOperations);
        filesFuture.get();
    } finally {
        executor.shutdownNow();
    }
}