Examples with PluginManager org.apache.flink.core.plugin.PluginManager used on opensource projects

Example 1 with PluginManager

use of org.apache.flink.core.plugin.PluginManager in project flink by apache.

the class ClusterEntrypoint method startCluster.

public void startCluster() throws ClusterEntrypointException {
    LOG.info("Starting {}.", getClass().getSimpleName());
    try {
        FlinkSecurityManager.setFromConfiguration(configuration);
        PluginManager pluginManager = PluginUtils.createPluginManagerFromRootFolder(configuration);
        configureFileSystems(configuration, pluginManager);
        SecurityContext securityContext = installSecurityContext(configuration);
        ClusterEntrypointUtils.configureUncaughtExceptionHandler(configuration);
        securityContext.runSecured((Callable<Void>) () -> {
            runCluster(configuration, pluginManager);
            return null;
        });
    } catch (Throwable t) {
        final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
        try {
            // clean up any partial state
            shutDownAsync(ApplicationStatus.FAILED, ShutdownBehaviour.GRACEFUL_SHUTDOWN, ExceptionUtils.stringifyException(strippedThrowable), false).get(INITIALIZATION_SHUTDOWN_TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS);
        } catch (InterruptedException | ExecutionException | TimeoutException e) {
            strippedThrowable.addSuppressed(e);
        }
        throw new ClusterEntrypointException(String.format("Failed to initialize the cluster entrypoint %s.", getClass().getSimpleName()), strippedThrowable);
    }
}

Example 2 with PluginManager

use of org.apache.flink.core.plugin.PluginManager in project flink by apache.

the class TaskManagerRunner method runTaskManagerProcessSecurely.

public static void runTaskManagerProcessSecurely(Configuration configuration) {
    FlinkSecurityManager.setFromConfiguration(configuration);
    final PluginManager pluginManager = PluginUtils.createPluginManagerFromRootFolder(configuration);
    FileSystem.initialize(configuration, pluginManager);
    StateChangelogStorageLoader.initialize(pluginManager);
    int exitCode;
    Throwable throwable = null;
    ClusterEntrypointUtils.configureUncaughtExceptionHandler(configuration);
    try {
        SecurityUtils.install(new SecurityConfiguration(configuration));
        exitCode = SecurityUtils.getInstalledContext().runSecured(() -> runTaskManager(configuration, pluginManager));
    } catch (Throwable t) {
        throwable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
        exitCode = FAILURE_EXIT_CODE;
    }
    if (throwable != null) {
        LOG.error("Terminating TaskManagerRunner with exit code {}.", exitCode, throwable);
    } else {
        LOG.info("Terminating TaskManagerRunner with exit code {}.", exitCode);
    }
    System.exit(exitCode);
}

Example 3 with PluginManager

use of org.apache.flink.core.plugin.PluginManager in project flink by apache.

the class TaskManagerRunner method startTaskManagerRunnerServices.

private void startTaskManagerRunnerServices() throws Exception {
    synchronized (lock) {
        rpcSystem = RpcSystem.load(configuration);
        this.executor = Executors.newScheduledThreadPool(Hardware.getNumberCPUCores(), new ExecutorThreadFactory("taskmanager-future"));
        highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(configuration, executor, AddressResolution.NO_ADDRESS_RESOLUTION, rpcSystem, this);
        JMXService.startInstance(configuration.getString(JMXServerOptions.JMX_SERVER_PORT));
        rpcService = createRpcService(configuration, highAvailabilityServices, rpcSystem);
        this.resourceId = getTaskManagerResourceID(configuration, rpcService.getAddress(), rpcService.getPort());
        this.workingDirectory = ClusterEntrypointUtils.createTaskManagerWorkingDirectory(configuration, resourceId);
        LOG.info("Using working directory: {}", workingDirectory);
        HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);
        metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(configuration, rpcSystem.getMaximumMessageSizeInBytes(configuration)), ReporterSetup.fromConfiguration(configuration, pluginManager));
        final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, rpcService.getAddress(), configuration.getString(TaskManagerOptions.BIND_HOST), rpcSystem);
        metricRegistry.startQueryService(metricQueryServiceRpcService, resourceId.unwrap());
        blobCacheService = BlobUtils.createBlobCacheService(configuration, Reference.borrowed(workingDirectory.unwrap().getBlobStorageDirectory()), highAvailabilityServices.createBlobStore(), null);
        final ExternalResourceInfoProvider externalResourceInfoProvider = ExternalResourceUtils.createStaticExternalResourceInfoProviderFromConfig(configuration, pluginManager);
        taskExecutorService = taskExecutorServiceFactory.createTaskExecutor(this.configuration, this.resourceId.unwrap(), rpcService, highAvailabilityServices, heartbeatServices, metricRegistry, blobCacheService, false, externalResourceInfoProvider, workingDirectory.unwrap(), this);
        handleUnexpectedTaskExecutorServiceTermination();
        MemoryLogger.startIfConfigured(LOG, configuration, terminationFuture.thenAccept(ignored -> {
        }));
    }
}

Example 4 with PluginManager

use of org.apache.flink.core.plugin.PluginManager in project flink by apache.

the class TaskExecutorProcessEntryPoint method main.

public static void main(String[] args) {
    try {
        final ParameterTool parameterTool = ParameterTool.fromArgs(args);
        Configuration cfg = parameterTool.getConfiguration();
        final PluginManager pluginManager = PluginUtils.createPluginManagerFromRootFolder(cfg);
        TaskExecutorResourceUtils.adjustForLocalExecution(cfg);
        TaskManagerRunner.runTaskManager(cfg, pluginManager);
    } catch (Throwable t) {
        LOG.error("Failed to run the TaskManager process", t);
        System.exit(1);
    }
}

Example 5 with PluginManager

use of org.apache.flink.core.plugin.PluginManager in project flink by apache.

the class ExternalResourceUtilsTest method testConstructExternalResourceDriversFromConfig.

@Test
public void testConstructExternalResourceDriversFromConfig() {
    final Configuration config = new Configuration();
    final String driverFactoryClassName = TestingExternalResourceDriverFactory.class.getName();
    final Map<Class<?>, Iterator<?>> plugins = new HashMap<>();
    plugins.put(ExternalResourceDriverFactory.class, IteratorUtils.singletonIterator(new TestingExternalResourceDriverFactory()));
    final PluginManager testingPluginManager = new TestingPluginManager(plugins);
    config.set(ExternalResourceOptions.EXTERNAL_RESOURCE_LIST, Collections.singletonList(RESOURCE_NAME_1));
    config.setString(ExternalResourceOptions.getExternalResourceDriverFactoryConfigOptionForResource(RESOURCE_NAME_1), driverFactoryClassName);
    final Map<String, ExternalResourceDriver> externalResourceDrivers = ExternalResourceUtils.externalResourceDriversFromConfig(config, testingPluginManager);
    assertThat(externalResourceDrivers.size(), is(1));
    assertThat(externalResourceDrivers.get(RESOURCE_NAME_1), instanceOf(TestingExternalResourceDriver.class));
}