Search in sources :

Example 1 with ShuffleMasterContext

use of org.apache.flink.runtime.shuffle.ShuffleMasterContext in project flink by apache.

the class JobManagerSharedServices method fromConfiguration.

// ------------------------------------------------------------------------
// Creating the components from a configuration
// ------------------------------------------------------------------------
public static JobManagerSharedServices fromConfiguration(Configuration config, BlobServer blobServer, FatalErrorHandler fatalErrorHandler) throws Exception {
    checkNotNull(config);
    checkNotNull(blobServer);
    final String classLoaderResolveOrder = config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);
    final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(config);
    final boolean failOnJvmMetaspaceOomError = config.getBoolean(CoreOptions.FAIL_ON_USER_CLASS_LOADING_METASPACE_OOM);
    final boolean checkClassLoaderLeak = config.getBoolean(CoreOptions.CHECK_LEAKED_CLASSLOADER);
    final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(blobServer, BlobLibraryCacheManager.defaultClassLoaderFactory(FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder), alwaysParentFirstLoaderPatterns, failOnJvmMetaspaceOomError ? fatalErrorHandler : null, checkClassLoaderLeak));
    final int numberCPUCores = Hardware.getNumberCPUCores();
    final int jobManagerFuturePoolSize = config.getInteger(JobManagerOptions.JOB_MANAGER_FUTURE_POOL_SIZE, numberCPUCores);
    final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool(jobManagerFuturePoolSize, new ExecutorThreadFactory("jobmanager-future"));
    final int jobManagerIoPoolSize = config.getInteger(JobManagerOptions.JOB_MANAGER_IO_POOL_SIZE, numberCPUCores);
    final ExecutorService ioExecutor = Executors.newFixedThreadPool(jobManagerIoPoolSize, new ExecutorThreadFactory("jobmanager-io"));
    final ShuffleMasterContext shuffleMasterContext = new ShuffleMasterContextImpl(config, fatalErrorHandler);
    final ShuffleMaster<?> shuffleMaster = ShuffleServiceLoader.loadShuffleServiceFactory(config).createShuffleMaster(shuffleMasterContext);
    shuffleMaster.start();
    return new JobManagerSharedServices(futureExecutor, ioExecutor, libraryCacheManager, shuffleMaster, blobServer);
}
Also used : ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) BlobLibraryCacheManager(org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExecutorService(java.util.concurrent.ExecutorService) ShuffleMasterContext(org.apache.flink.runtime.shuffle.ShuffleMasterContext) ShuffleMasterContextImpl(org.apache.flink.runtime.shuffle.ShuffleMasterContextImpl)

Aggregations

ExecutorService (java.util.concurrent.ExecutorService)1 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)1 BlobLibraryCacheManager (org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager)1 ShuffleMasterContext (org.apache.flink.runtime.shuffle.ShuffleMasterContext)1 ShuffleMasterContextImpl (org.apache.flink.runtime.shuffle.ShuffleMasterContextImpl)1 ExecutorThreadFactory (org.apache.flink.util.concurrent.ExecutorThreadFactory)1