use of org.apache.flink.runtime.rpc.RpcSystem in project flink by apache.
the class TaskManagerRunner method startTaskManagerRunnerServices.
private void startTaskManagerRunnerServices() throws Exception {
synchronized (lock) {
rpcSystem = RpcSystem.load(configuration);
this.executor = Executors.newScheduledThreadPool(Hardware.getNumberCPUCores(), new ExecutorThreadFactory("taskmanager-future"));
highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(configuration, executor, AddressResolution.NO_ADDRESS_RESOLUTION, rpcSystem, this);
JMXService.startInstance(configuration.getString(JMXServerOptions.JMX_SERVER_PORT));
rpcService = createRpcService(configuration, highAvailabilityServices, rpcSystem);
this.resourceId = getTaskManagerResourceID(configuration, rpcService.getAddress(), rpcService.getPort());
this.workingDirectory = ClusterEntrypointUtils.createTaskManagerWorkingDirectory(configuration, resourceId);
LOG.info("Using working directory: {}", workingDirectory);
HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);
metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(configuration, rpcSystem.getMaximumMessageSizeInBytes(configuration)), ReporterSetup.fromConfiguration(configuration, pluginManager));
final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, rpcService.getAddress(), configuration.getString(TaskManagerOptions.BIND_HOST), rpcSystem);
metricRegistry.startQueryService(metricQueryServiceRpcService, resourceId.unwrap());
blobCacheService = BlobUtils.createBlobCacheService(configuration, Reference.borrowed(workingDirectory.unwrap().getBlobStorageDirectory()), highAvailabilityServices.createBlobStore(), null);
final ExternalResourceInfoProvider externalResourceInfoProvider = ExternalResourceUtils.createStaticExternalResourceInfoProviderFromConfig(configuration, pluginManager);
taskExecutorService = taskExecutorServiceFactory.createTaskExecutor(this.configuration, this.resourceId.unwrap(), rpcService, highAvailabilityServices, heartbeatServices, metricRegistry, blobCacheService, false, externalResourceInfoProvider, workingDirectory.unwrap(), this);
handleUnexpectedTaskExecutorServiceTermination();
MemoryLogger.startIfConfigured(LOG, configuration, terminationFuture.thenAccept(ignored -> {
}));
}
}
use of org.apache.flink.runtime.rpc.RpcSystem in project flink by apache.
the class AkkaRpcSystemLoader method loadRpcSystem.
@Override
public RpcSystem loadRpcSystem(Configuration config) {
try {
final ClassLoader flinkClassLoader = RpcSystem.class.getClassLoader();
final Path tmpDirectory = Paths.get(ConfigurationUtils.parseTempDirectories(config)[0]);
Files.createDirectories(tmpDirectory);
final Path tempFile = Files.createFile(tmpDirectory.resolve("flink-rpc-akka_" + UUID.randomUUID() + ".jar"));
final InputStream resourceStream = flinkClassLoader.getResourceAsStream(FLINK_RPC_AKKA_FAT_JAR);
if (resourceStream == null) {
throw new RpcLoaderException(String.format("Akka RPC system could not be found. If this happened while running a test in the IDE, " + "run '%s' on the command-line, " + "or add a test dependency on the flink-rpc-akka-loader test-jar.", HINT_USAGE));
}
IOUtils.copyBytes(resourceStream, Files.newOutputStream(tempFile));
final SubmoduleClassLoader submoduleClassLoader = new SubmoduleClassLoader(new URL[] { tempFile.toUri().toURL() }, flinkClassLoader);
return new CleanupOnCloseRpcSystem(ServiceLoader.load(RpcSystem.class, submoduleClassLoader).iterator().next(), submoduleClassLoader, tempFile);
} catch (IOException e) {
throw new RuntimeException("Could not initialize RPC system.", e);
}
}
use of org.apache.flink.runtime.rpc.RpcSystem in project flink by apache.
the class FallbackAkkaRpcSystemLoader method loadRpcSystem.
@Override
public RpcSystem loadRpcSystem(Configuration config) {
try {
LOG.debug("Using Fallback AkkaRpcSystemLoader; this loader will invoke maven to retrieve the dependencies of flink-rpc-akka.");
final ClassLoader flinkClassLoader = RpcSystem.class.getClassLoader();
// flink-rpc/flink-rpc-akka
final Path akkaRpcModuleDirectory = findAkkaRpcModuleDirectory(getCurrentWorkingDirectory());
// flink-rpc/flink-rpc-akka/target/classes
final Path akkaRpcModuleClassesDirectory = akkaRpcModuleDirectory.resolve(Paths.get("target", "classes"));
// flink-rpc/flink-rpc-akka/target/dependencies
final Path akkaRpcModuleDependenciesDirectory = akkaRpcModuleDirectory.resolve(Paths.get("target", "dependencies"));
if (!Files.exists(akkaRpcModuleDependenciesDirectory)) {
int exitCode = downloadDependencies(akkaRpcModuleDirectory, akkaRpcModuleDependenciesDirectory);
if (exitCode != 0) {
throw new RpcLoaderException("Could not download dependencies of flink-rpc-akka, please see the log output for details.");
}
} else {
LOG.debug("Re-using previously downloaded flink-rpc-akka dependencies. If you are experiencing strange issues, try clearing '{}'.", akkaRpcModuleDependenciesDirectory);
}
// assemble URL collection containing target/classes and each jar
final List<URL> urls = new ArrayList<>();
urls.add(akkaRpcModuleClassesDirectory.toUri().toURL());
try (final Stream<Path> files = Files.list(akkaRpcModuleDependenciesDirectory)) {
final List<Path> collect = files.filter(path -> path.getFileName().toString().endsWith(".jar")).collect(Collectors.toList());
for (Path path : collect) {
urls.add(path.toUri().toURL());
}
}
final SubmoduleClassLoader submoduleClassLoader = new SubmoduleClassLoader(urls.toArray(new URL[0]), flinkClassLoader);
return new CleanupOnCloseRpcSystem(ServiceLoader.load(RpcSystem.class, submoduleClassLoader).iterator().next(), submoduleClassLoader, null);
} catch (Exception e) {
throw new RpcLoaderException(String.format("Could not initialize RPC system. Run '%s' on the command-line instead.", AkkaRpcSystemLoader.HINT_USAGE), e);
}
}
use of org.apache.flink.runtime.rpc.RpcSystem in project flink by apache.
the class AkkaRpcSystemLoaderITCase method testServiceLoadingWithNonExistingPath.
@Test
public void testServiceLoadingWithNonExistingPath() {
final Configuration config = new Configuration();
config.set(CoreOptions.TMP_DIRS, TMP_DIR.getRoot().toPath().resolve(Paths.get("some", "directory")).toString());
try (final RpcSystem rpcSystem = LOADER.loadRpcSystem(config)) {
assertThat(rpcSystem, not(nullValue()));
}
}
Aggregations