Examples with BlobStoreService org.apache.flink.runtime.blob.BlobStoreService used on opensource projects

Example 1 with BlobStoreService

use of org.apache.flink.runtime.blob.BlobStoreService in project flink by apache.

the class HighAvailabilityServicesUtils method createZooKeeperHaServices.

private static HighAvailabilityServices createZooKeeperHaServices(Configuration configuration, Executor executor, FatalErrorHandler fatalErrorHandler) throws Exception {
    final boolean useOldHaServices = configuration.get(HighAvailabilityOptions.USE_OLD_HA_SERVICES);
    BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);
    final CuratorFrameworkWithUnhandledErrorListener curatorFrameworkWrapper = ZooKeeperUtils.startCuratorFramework(configuration, fatalErrorHandler);
    if (useOldHaServices) {
        return new ZooKeeperHaServices(curatorFrameworkWrapper, executor, configuration, blobStoreService);
    } else {
        return new ZooKeeperMultipleComponentLeaderElectionHaServices(curatorFrameworkWrapper, configuration, executor, blobStoreService, fatalErrorHandler);
    }
}

Example 2 with BlobStoreService

use of org.apache.flink.runtime.blob.BlobStoreService in project flink by apache.

the class HDFSTest method testBlobCacheCorruptedFile.

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed corrupted JARs are
 * recognised during the download via a BLOB cache.
 */
@Test
public void testBlobCacheCorruptedFile() throws Exception {
    org.apache.flink.configuration.Configuration config = new org.apache.flink.configuration.Configuration();
    config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
    config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
    config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);
    BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);
    try {
        BlobCacheCorruptionTest.testGetFailsFromCorruptFile(new JobID(), config, blobStoreService, temporaryFolder.newFolder());
    } finally {
        blobStoreService.closeAndCleanupAllData();
    }
}

Example 3 with BlobStoreService

use of org.apache.flink.runtime.blob.BlobStoreService in project flink by apache.

the class HDFSTest method testBlobServerRecovery.

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from
 * any participating BlobServer when talking to the {@link
 * org.apache.flink.runtime.blob.BlobServer} directly.
 */
@Test
public void testBlobServerRecovery() throws Exception {
    org.apache.flink.configuration.Configuration config = new org.apache.flink.configuration.Configuration();
    config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
    config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
    config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);
    BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);
    try {
        BlobServerRecoveryTest.testBlobServerRecovery(config, blobStoreService, temporaryFolder.newFolder());
    } finally {
        blobStoreService.closeAndCleanupAllData();
    }
}

Example 4 with BlobStoreService

use of org.apache.flink.runtime.blob.BlobStoreService in project flink by apache.

the class HDFSTest method testBlobCacheRecovery.

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from
 * any participating BlobServer when uploaded via a BLOB cache.
 */
@Test
public void testBlobCacheRecovery() throws Exception {
    org.apache.flink.configuration.Configuration config = new org.apache.flink.configuration.Configuration();
    config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
    config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
    config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);
    BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);
    try {
        BlobCacheRecoveryTest.testBlobCacheRecovery(config, blobStoreService, temporaryFolder.newFolder());
    } finally {
        blobStoreService.closeAndCleanupAllData();
    }
}

Example 5 with BlobStoreService

use of org.apache.flink.runtime.blob.BlobStoreService in project flink by apache.

the class HDFSTest method testBlobServerCorruptedFile.

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed corrupted JARs are
 * recognised during the download via a {@link org.apache.flink.runtime.blob.BlobServer}.
 */
@Test
public void testBlobServerCorruptedFile() throws Exception {
    org.apache.flink.configuration.Configuration config = new org.apache.flink.configuration.Configuration();
    config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
    config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
    config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);
    BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);
    try {
        BlobServerCorruptionTest.testGetFailsFromCorruptFile(config, blobStoreService, temporaryFolder.newFolder());
    } finally {
        blobStoreService.closeAndCleanupAllData();
    }
}