Search in sources :

Example 1 with RpcSystem

use of org.apache.flink.runtime.rpc.RpcSystem in project flink by apache.

the class TaskManagerRunner method startTaskManagerRunnerServices.

private void startTaskManagerRunnerServices() throws Exception {
    synchronized (lock) {
        rpcSystem = RpcSystem.load(configuration);
        this.executor = Executors.newScheduledThreadPool(Hardware.getNumberCPUCores(), new ExecutorThreadFactory("taskmanager-future"));
        highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(configuration, executor, AddressResolution.NO_ADDRESS_RESOLUTION, rpcSystem, this);
        JMXService.startInstance(configuration.getString(JMXServerOptions.JMX_SERVER_PORT));
        rpcService = createRpcService(configuration, highAvailabilityServices, rpcSystem);
        this.resourceId = getTaskManagerResourceID(configuration, rpcService.getAddress(), rpcService.getPort());
        this.workingDirectory = ClusterEntrypointUtils.createTaskManagerWorkingDirectory(configuration, resourceId);
        LOG.info("Using working directory: {}", workingDirectory);
        HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);
        metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(configuration, rpcSystem.getMaximumMessageSizeInBytes(configuration)), ReporterSetup.fromConfiguration(configuration, pluginManager));
        final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, rpcService.getAddress(), configuration.getString(TaskManagerOptions.BIND_HOST), rpcSystem);
        metricRegistry.startQueryService(metricQueryServiceRpcService, resourceId.unwrap());
        blobCacheService = BlobUtils.createBlobCacheService(configuration, Reference.borrowed(workingDirectory.unwrap().getBlobStorageDirectory()), highAvailabilityServices.createBlobStore(), null);
        final ExternalResourceInfoProvider externalResourceInfoProvider = ExternalResourceUtils.createStaticExternalResourceInfoProviderFromConfig(configuration, pluginManager);
        taskExecutorService = taskExecutorServiceFactory.createTaskExecutor(this.configuration, this.resourceId.unwrap(), rpcService, highAvailabilityServices, heartbeatServices, metricRegistry, blobCacheService, false, externalResourceInfoProvider, workingDirectory.unwrap(), this);
        handleUnexpectedTaskExecutorServiceTermination();
        MemoryLogger.startIfConfigured(LOG, configuration, terminationFuture.thenAccept(ignored -> {
        }));
    }
}
Also used : ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) HeartbeatServices(org.apache.flink.runtime.heartbeat.HeartbeatServices) TaskExecutorBlobService(org.apache.flink.runtime.blob.TaskExecutorBlobService) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Hardware(org.apache.flink.runtime.util.Hardware) RpcSystemUtils(org.apache.flink.runtime.rpc.RpcSystemUtils) LoggerFactory(org.slf4j.LoggerFactory) ExternalResourceUtils(org.apache.flink.runtime.externalresource.ExternalResourceUtils) LeaderRetrievalException(org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException) ExceptionUtils(org.apache.flink.util.ExceptionUtils) LeaderRetrievalUtils(org.apache.flink.runtime.util.LeaderRetrievalUtils) ReporterSetup(org.apache.flink.runtime.metrics.ReporterSetup) InetAddress(java.net.InetAddress) MemoryLogger(org.apache.flink.runtime.taskmanager.MemoryLogger) StateChangelogStorageLoader(org.apache.flink.runtime.state.changelog.StateChangelogStorageLoader) MetricRegistry(org.apache.flink.runtime.metrics.MetricRegistry) FunctionUtils(org.apache.flink.util.function.FunctionUtils) Duration(java.time.Duration) SecurityUtils(org.apache.flink.runtime.security.SecurityUtils) TaskManagerOptionsInternal(org.apache.flink.configuration.TaskManagerOptionsInternal) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) AkkaOptions(org.apache.flink.configuration.AkkaOptions) HighAvailabilityServices(org.apache.flink.runtime.highavailability.HighAvailabilityServices) ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) AbstractID(org.apache.flink.util.AbstractID) Collection(java.util.Collection) JvmShutdownSafeguard(org.apache.flink.runtime.util.JvmShutdownSafeguard) HighAvailabilityServicesUtils(org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils) MetricUtils(org.apache.flink.runtime.metrics.util.MetricUtils) GuardedBy(javax.annotation.concurrent.GuardedBy) SecurityConfiguration(org.apache.flink.runtime.security.SecurityConfiguration) EnvironmentInformation(org.apache.flink.runtime.util.EnvironmentInformation) StringUtils(org.apache.flink.util.StringUtils) Executors(java.util.concurrent.Executors) HeartbeatServices(org.apache.flink.runtime.heartbeat.HeartbeatServices) ExecutorUtils(org.apache.flink.util.ExecutorUtils) JMXService(org.apache.flink.management.jmx.JMXService) MetricGroup(org.apache.flink.metrics.MetricGroup) FileSystem(org.apache.flink.core.fs.FileSystem) PluginManager(org.apache.flink.core.plugin.PluginManager) ConfigurationParserUtils(org.apache.flink.runtime.util.ConfigurationParserUtils) ClusterEntrypointUtils(org.apache.flink.runtime.entrypoint.ClusterEntrypointUtils) Time(org.apache.flink.api.common.time.Time) TaskExecutorPartitionTrackerImpl(org.apache.flink.runtime.io.network.partition.TaskExecutorPartitionTrackerImpl) FlinkException(org.apache.flink.util.FlinkException) MetricRegistryImpl(org.apache.flink.runtime.metrics.MetricRegistryImpl) CompletableFuture(java.util.concurrent.CompletableFuture) ShutdownHookUtil(org.apache.flink.util.ShutdownHookUtil) ArrayList(java.util.ArrayList) MetricRegistryConfiguration(org.apache.flink.runtime.metrics.MetricRegistryConfiguration) TaskManagerOptions(org.apache.flink.configuration.TaskManagerOptions) RpcService(org.apache.flink.runtime.rpc.RpcService) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) ExternalResourceInfoProvider(org.apache.flink.runtime.externalresource.ExternalResourceInfoProvider) FatalErrorHandler(org.apache.flink.runtime.rpc.FatalErrorHandler) FlinkSecurityManager(org.apache.flink.core.security.FlinkSecurityManager) ResourceID(org.apache.flink.runtime.clusterframework.types.ResourceID) ExecutorService(java.util.concurrent.ExecutorService) TaskManagerExceptionUtils(org.apache.flink.util.TaskManagerExceptionUtils) Logger(org.slf4j.Logger) AutoCloseableAsync(org.apache.flink.util.AutoCloseableAsync) Configuration(org.apache.flink.configuration.Configuration) TaskManagerMetricGroup(org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup) IOException(java.io.IOException) Reference(org.apache.flink.util.Reference) RpcUtils(org.apache.flink.runtime.rpc.RpcUtils) SignalHandler(org.apache.flink.runtime.util.SignalHandler) UndeclaredThrowableException(java.lang.reflect.UndeclaredThrowableException) VisibleForTesting(org.apache.flink.annotation.VisibleForTesting) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) DeterminismEnvelope(org.apache.flink.runtime.entrypoint.DeterminismEnvelope) RpcSystem(org.apache.flink.runtime.rpc.RpcSystem) PluginUtils(org.apache.flink.core.plugin.PluginUtils) WorkingDirectory(org.apache.flink.runtime.entrypoint.WorkingDirectory) JMXServerOptions(org.apache.flink.configuration.JMXServerOptions) BlobCacheService(org.apache.flink.runtime.blob.BlobCacheService) AddressResolution(org.apache.flink.runtime.rpc.AddressResolution) BlobUtils(org.apache.flink.runtime.blob.BlobUtils) FlinkParseException(org.apache.flink.runtime.entrypoint.FlinkParseException) RpcService(org.apache.flink.runtime.rpc.RpcService) MetricRegistryImpl(org.apache.flink.runtime.metrics.MetricRegistryImpl) ExternalResourceInfoProvider(org.apache.flink.runtime.externalresource.ExternalResourceInfoProvider)

Example 2 with RpcSystem

use of org.apache.flink.runtime.rpc.RpcSystem in project flink by apache.

the class AkkaRpcSystemLoader method loadRpcSystem.

@Override
public RpcSystem loadRpcSystem(Configuration config) {
    try {
        final ClassLoader flinkClassLoader = RpcSystem.class.getClassLoader();
        final Path tmpDirectory = Paths.get(ConfigurationUtils.parseTempDirectories(config)[0]);
        Files.createDirectories(tmpDirectory);
        final Path tempFile = Files.createFile(tmpDirectory.resolve("flink-rpc-akka_" + UUID.randomUUID() + ".jar"));
        final InputStream resourceStream = flinkClassLoader.getResourceAsStream(FLINK_RPC_AKKA_FAT_JAR);
        if (resourceStream == null) {
            throw new RpcLoaderException(String.format("Akka RPC system could not be found. If this happened while running a test in the IDE, " + "run '%s' on the command-line, " + "or add a test dependency on the flink-rpc-akka-loader test-jar.", HINT_USAGE));
        }
        IOUtils.copyBytes(resourceStream, Files.newOutputStream(tempFile));
        final SubmoduleClassLoader submoduleClassLoader = new SubmoduleClassLoader(new URL[] { tempFile.toUri().toURL() }, flinkClassLoader);
        return new CleanupOnCloseRpcSystem(ServiceLoader.load(RpcSystem.class, submoduleClassLoader).iterator().next(), submoduleClassLoader, tempFile);
    } catch (IOException e) {
        throw new RuntimeException("Could not initialize RPC system.", e);
    }
}
Also used : Path(java.nio.file.Path) InputStream(java.io.InputStream) SubmoduleClassLoader(org.apache.flink.core.classloading.SubmoduleClassLoader) RpcLoaderException(org.apache.flink.runtime.rpc.exceptions.RpcLoaderException) SubmoduleClassLoader(org.apache.flink.core.classloading.SubmoduleClassLoader) IOException(java.io.IOException) RpcSystem(org.apache.flink.runtime.rpc.RpcSystem)

Example 3 with RpcSystem

use of org.apache.flink.runtime.rpc.RpcSystem in project flink by apache.

the class FallbackAkkaRpcSystemLoader method loadRpcSystem.

@Override
public RpcSystem loadRpcSystem(Configuration config) {
    try {
        LOG.debug("Using Fallback AkkaRpcSystemLoader; this loader will invoke maven to retrieve the dependencies of flink-rpc-akka.");
        final ClassLoader flinkClassLoader = RpcSystem.class.getClassLoader();
        // flink-rpc/flink-rpc-akka
        final Path akkaRpcModuleDirectory = findAkkaRpcModuleDirectory(getCurrentWorkingDirectory());
        // flink-rpc/flink-rpc-akka/target/classes
        final Path akkaRpcModuleClassesDirectory = akkaRpcModuleDirectory.resolve(Paths.get("target", "classes"));
        // flink-rpc/flink-rpc-akka/target/dependencies
        final Path akkaRpcModuleDependenciesDirectory = akkaRpcModuleDirectory.resolve(Paths.get("target", "dependencies"));
        if (!Files.exists(akkaRpcModuleDependenciesDirectory)) {
            int exitCode = downloadDependencies(akkaRpcModuleDirectory, akkaRpcModuleDependenciesDirectory);
            if (exitCode != 0) {
                throw new RpcLoaderException("Could not download dependencies of flink-rpc-akka, please see the log output for details.");
            }
        } else {
            LOG.debug("Re-using previously downloaded flink-rpc-akka dependencies. If you are experiencing strange issues, try clearing '{}'.", akkaRpcModuleDependenciesDirectory);
        }
        // assemble URL collection containing target/classes and each jar
        final List<URL> urls = new ArrayList<>();
        urls.add(akkaRpcModuleClassesDirectory.toUri().toURL());
        try (final Stream<Path> files = Files.list(akkaRpcModuleDependenciesDirectory)) {
            final List<Path> collect = files.filter(path -> path.getFileName().toString().endsWith(".jar")).collect(Collectors.toList());
            for (Path path : collect) {
                urls.add(path.toUri().toURL());
            }
        }
        final SubmoduleClassLoader submoduleClassLoader = new SubmoduleClassLoader(urls.toArray(new URL[0]), flinkClassLoader);
        return new CleanupOnCloseRpcSystem(ServiceLoader.load(RpcSystem.class, submoduleClassLoader).iterator().next(), submoduleClassLoader, null);
    } catch (Exception e) {
        throw new RpcLoaderException(String.format("Could not initialize RPC system. Run '%s' on the command-line instead.", AkkaRpcSystemLoader.HINT_USAGE), e);
    }
}
Also used : Path(java.nio.file.Path) SubmoduleClassLoader(org.apache.flink.core.classloading.SubmoduleClassLoader) Logger(org.slf4j.Logger) Files(java.nio.file.Files) URL(java.net.URL) Configuration(org.apache.flink.configuration.Configuration) LoggerFactory(org.slf4j.LoggerFactory) IOException(java.io.IOException) ServiceLoader(java.util.ServiceLoader) RpcLoaderException(org.apache.flink.runtime.rpc.exceptions.RpcLoaderException) OperatingSystem(org.apache.flink.util.OperatingSystem) Collectors(java.util.stream.Collectors) RpcSystemLoader(org.apache.flink.runtime.rpc.RpcSystemLoader) ArrayList(java.util.ArrayList) List(java.util.List) RpcSystem(org.apache.flink.runtime.rpc.RpcSystem) Stream(java.util.stream.Stream) Paths(java.nio.file.Paths) Optional(java.util.Optional) Path(java.nio.file.Path) ArrayList(java.util.ArrayList) RpcLoaderException(org.apache.flink.runtime.rpc.exceptions.RpcLoaderException) RpcSystem(org.apache.flink.runtime.rpc.RpcSystem) URL(java.net.URL) IOException(java.io.IOException) RpcLoaderException(org.apache.flink.runtime.rpc.exceptions.RpcLoaderException) SubmoduleClassLoader(org.apache.flink.core.classloading.SubmoduleClassLoader) SubmoduleClassLoader(org.apache.flink.core.classloading.SubmoduleClassLoader)

Example 4 with RpcSystem

use of org.apache.flink.runtime.rpc.RpcSystem in project flink by apache.

the class AkkaRpcSystemLoaderITCase method testServiceLoadingWithNonExistingPath.

@Test
public void testServiceLoadingWithNonExistingPath() {
    final Configuration config = new Configuration();
    config.set(CoreOptions.TMP_DIRS, TMP_DIR.getRoot().toPath().resolve(Paths.get("some", "directory")).toString());
    try (final RpcSystem rpcSystem = LOADER.loadRpcSystem(config)) {
        assertThat(rpcSystem, not(nullValue()));
    }
}
Also used : Configuration(org.apache.flink.configuration.Configuration) RpcSystem(org.apache.flink.runtime.rpc.RpcSystem) Test(org.junit.Test)

Aggregations

RpcSystem (org.apache.flink.runtime.rpc.RpcSystem)4 IOException (java.io.IOException)3 Configuration (org.apache.flink.configuration.Configuration)3 Path (java.nio.file.Path)2 ArrayList (java.util.ArrayList)2 SubmoduleClassLoader (org.apache.flink.core.classloading.SubmoduleClassLoader)2 InputStream (java.io.InputStream)1 UndeclaredThrowableException (java.lang.reflect.UndeclaredThrowableException)1 InetAddress (java.net.InetAddress)1 URL (java.net.URL)1 Files (java.nio.file.Files)1 Paths (java.nio.file.Paths)1 Duration (java.time.Duration)1 Collection (java.util.Collection)1 List (java.util.List)1 Optional (java.util.Optional)1 ServiceLoader (java.util.ServiceLoader)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 ExecutionException (java.util.concurrent.ExecutionException)1 ExecutorService (java.util.concurrent.ExecutorService)1