Examples with BlobServer org.apache.flink.runtime.blob.BlobServer used on opensource projects

Example 6 with BlobServer

use of org.apache.flink.runtime.blob.BlobServer in project flink by apache.

the class ZooKeeperDefaultDispatcherRunnerTest method setup.

@Before
public void setup() throws IOException {
    fatalErrorHandler = new TestingFatalErrorHandler();
    configuration = new Configuration();
    configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
    configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());
    configuration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getAbsolutePath());
    clusterHaStorageDir = new File(HighAvailabilityServicesUtils.getClusterHighAvailableStoragePath(configuration).toString());
    blobServer = new BlobServer(configuration, temporaryFolder.newFolder(), BlobUtils.createBlobStoreFromConfig(configuration));
}

Example 7 with BlobServer

use of org.apache.flink.runtime.blob.BlobServer in project flink by apache.

the class DefaultExecutionGraphDeploymentWithBlobCacheTest method setupBlobServer.

@Before
@Override
public void setupBlobServer() throws IOException {
    Configuration config = new Configuration();
    // always offload the serialized job and task information
    config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
    blobServer = new BlobServer(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore());
    blobServer.start();
    blobWriter = blobServer;
    InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort());
    blobCache = new PermanentBlobCache(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore(), serverAddress);
}

Example 8 with BlobServer

use of org.apache.flink.runtime.blob.BlobServer in project flink by apache.

the class BlobServerExtension method before.

@Override
public void before(ExtensionContext context) throws Exception {
    temporaryFolder.create();
    Configuration config = new Configuration();
    blobServer = new BlobServer(config, temporaryFolder.newFolder(), new VoidBlobStore());
    blobServer.start();
}

Example 9 with BlobServer

use of org.apache.flink.runtime.blob.BlobServer in project flink by apache.

the class JobManagerServices method fromConfiguration.

// ------------------------------------------------------------------------
//  Creating the components from a configuration 
// ------------------------------------------------------------------------
public static JobManagerServices fromConfiguration(Configuration config, HighAvailabilityServices haServices) throws Exception {
    final BlobServer blobServer = new BlobServer(config, haServices);
    final long cleanupInterval = config.getLong(ConfigConstants.LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL, ConfigConstants.DEFAULT_LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL) * 1000;
    final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(blobServer, cleanupInterval);
    final FiniteDuration timeout;
    try {
        timeout = AkkaUtils.getTimeout(config);
    } catch (NumberFormatException e) {
        throw new IllegalConfigurationException(AkkaUtils.formatDurationParingErrorMessage());
    }
    final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool(Hardware.getNumberCPUCores(), new ExecutorThreadFactory("jobmanager-future"));
    return new JobManagerServices(futureExecutor, libraryCacheManager, RestartStrategyFactory.createRestartStrategyFactory(config), Time.of(timeout.length(), timeout.unit()));
}

Example 10 with BlobServer

use of org.apache.flink.runtime.blob.BlobServer in project flink by apache.

the class DefaultExecutionGraphDeploymentWithSmallBlobCacheSizeLimitTest method setupBlobServer.

@Before
@Override
public void setupBlobServer() throws IOException {
    Configuration config = new Configuration();
    // Always offload the serialized JobInformation, TaskInformation and cached
    // ShuffleDescriptors
    config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
    blobServer = new BlobServer(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore());
    blobServer.start();
    blobWriter = blobServer;
    InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort());
    // Set the size limit of the blob cache to 1
    BlobCacheSizeTracker blobCacheSizeTracker = new BlobCacheSizeTracker(1L);
    blobCache = new PermanentBlobCache(config, TEMPORARY_FOLDER.newFolder(), new VoidBlobStore(), serverAddress, blobCacheSizeTracker);
}