Examples with RpcService org.apache.flink.runtime.rpc.RpcService used on opensource projects

Example 6 with RpcService

use of org.apache.flink.runtime.rpc.RpcService in project flink by apache.

the class TaskManagerRunner method startTaskManagerRunnerServices.

private void startTaskManagerRunnerServices() throws Exception {
    synchronized (lock) {
        rpcSystem = RpcSystem.load(configuration);
        this.executor = Executors.newScheduledThreadPool(Hardware.getNumberCPUCores(), new ExecutorThreadFactory("taskmanager-future"));
        highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(configuration, executor, AddressResolution.NO_ADDRESS_RESOLUTION, rpcSystem, this);
        JMXService.startInstance(configuration.getString(JMXServerOptions.JMX_SERVER_PORT));
        rpcService = createRpcService(configuration, highAvailabilityServices, rpcSystem);
        this.resourceId = getTaskManagerResourceID(configuration, rpcService.getAddress(), rpcService.getPort());
        this.workingDirectory = ClusterEntrypointUtils.createTaskManagerWorkingDirectory(configuration, resourceId);
        LOG.info("Using working directory: {}", workingDirectory);
        HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);
        metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(configuration, rpcSystem.getMaximumMessageSizeInBytes(configuration)), ReporterSetup.fromConfiguration(configuration, pluginManager));
        final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, rpcService.getAddress(), configuration.getString(TaskManagerOptions.BIND_HOST), rpcSystem);
        metricRegistry.startQueryService(metricQueryServiceRpcService, resourceId.unwrap());
        blobCacheService = BlobUtils.createBlobCacheService(configuration, Reference.borrowed(workingDirectory.unwrap().getBlobStorageDirectory()), highAvailabilityServices.createBlobStore(), null);
        final ExternalResourceInfoProvider externalResourceInfoProvider = ExternalResourceUtils.createStaticExternalResourceInfoProviderFromConfig(configuration, pluginManager);
        taskExecutorService = taskExecutorServiceFactory.createTaskExecutor(this.configuration, this.resourceId.unwrap(), rpcService, highAvailabilityServices, heartbeatServices, metricRegistry, blobCacheService, false, externalResourceInfoProvider, workingDirectory.unwrap(), this);
        handleUnexpectedTaskExecutorServiceTermination();
        MemoryLogger.startIfConfigured(LOG, configuration, terminationFuture.thenAccept(ignored -> {
        }));
    }
}

Example 7 with RpcService

use of org.apache.flink.runtime.rpc.RpcService in project flink by apache.

the class RetryingRegistrationTest method testCancellation.

@Test
public void testCancellation() throws Exception {
    final String testEndpointAddress = "my-test-address";
    final UUID leaderId = UUID.randomUUID();
    CompletableFuture<RegistrationResponse> result = new CompletableFuture<>();
    AtomicInteger registrationCallCounter = new AtomicInteger(0);
    TestRegistrationGateway testGateway = DefaultTestRegistrationGateway.newBuilder().setRegistrationFunction((uuid, aLong) -> {
        registrationCallCounter.incrementAndGet();
        return result;
    }).build();
    rpcService.registerGateway(testEndpointAddress, testGateway);
    TestRetryingRegistration registration = new TestRetryingRegistration(rpcService, testEndpointAddress, leaderId);
    registration.startRegistration();
    // cancel and fail the current registration attempt
    registration.cancel();
    result.completeExceptionally(new TimeoutException());
    // there should not be a second registration attempt
    assertThat(registrationCallCounter.get(), is(lessThanOrEqualTo(1)));
}

Example 8 with RpcService

use of org.apache.flink.runtime.rpc.RpcService in project flink by apache.

the class RetryingRegistrationTest method testPropagateFailures.

@Test
public void testPropagateFailures() throws Exception {
    final String testExceptionMessage = "testExceptionMessage";
    // RPC service that fails with exception upon the connection
    RpcService rpc = mock(RpcService.class);
    when(rpc.connect(anyString(), any(Class.class))).thenThrow(new RuntimeException(testExceptionMessage));
    TestRetryingRegistration registration = new TestRetryingRegistration(rpc, "testaddress", UUID.randomUUID());
    registration.startRegistration();
    CompletableFuture<?> future = registration.getFuture();
    assertTrue(future.isDone());
    try {
        future.get();
        fail("We expected an ExecutionException.");
    } catch (ExecutionException e) {
        assertEquals(testExceptionMessage, e.getCause().getMessage());
    }
}

Example 9 with RpcService

use of org.apache.flink.runtime.rpc.RpcService in project flink by apache.

the class TaskManagerRunnerStartupTest method testMetricInitialization.

/**
 * Checks that all expected metrics are initialized.
 */
@Test
public void testMetricInitialization() throws Exception {
    Configuration cfg = createFlinkConfiguration();
    List<String> registeredMetrics = new ArrayList<>();
    startTaskManager(cfg, rpcService, highAvailabilityServices, WORKING_DIRECTORY_RESOURCE.createNewWorkingDirectory(), TestingMetricRegistry.builder().setRegisterConsumer((metric, metricName, group) -> registeredMetrics.add(group.getMetricIdentifier(metricName))).setScopeFormats(ScopeFormats.fromConfig(cfg)).build());
    // GC-related metrics are not checked since their existence depends on the JVM used
    Set<String> expectedTaskManagerMetricsWithoutTaskManagerId = Sets.newHashSet(".taskmanager..Status.JVM.ClassLoader.ClassesLoaded", ".taskmanager..Status.JVM.ClassLoader.ClassesUnloaded", ".taskmanager..Status.JVM.Memory.Heap.Used", ".taskmanager..Status.JVM.Memory.Heap.Committed", ".taskmanager..Status.JVM.Memory.Heap.Max", ".taskmanager..Status.JVM.Memory.NonHeap.Used", ".taskmanager..Status.JVM.Memory.NonHeap.Committed", ".taskmanager..Status.JVM.Memory.NonHeap.Max", ".taskmanager..Status.JVM.Memory.Direct.Count", ".taskmanager..Status.JVM.Memory.Direct.MemoryUsed", ".taskmanager..Status.JVM.Memory.Direct.TotalCapacity", ".taskmanager..Status.JVM.Memory.Mapped.Count", ".taskmanager..Status.JVM.Memory.Mapped.MemoryUsed", ".taskmanager..Status.JVM.Memory.Mapped.TotalCapacity", ".taskmanager..Status.Flink.Memory.Managed.Used", ".taskmanager..Status.Flink.Memory.Managed.Total", ".taskmanager..Status.JVM.Threads.Count", ".taskmanager..Status.JVM.CPU.Load", ".taskmanager..Status.JVM.CPU.Time", ".taskmanager..Status.Network.TotalMemorySegments", ".taskmanager..Status.Network.AvailableMemorySegments", ".taskmanager..Status.Shuffle.Netty.TotalMemorySegments", ".taskmanager..Status.Shuffle.Netty.TotalMemory", ".taskmanager..Status.Shuffle.Netty.AvailableMemorySegments", ".taskmanager..Status.Shuffle.Netty.AvailableMemory", ".taskmanager..Status.Shuffle.Netty.UsedMemorySegments", ".taskmanager..Status.Shuffle.Netty.UsedMemory");
    Pattern pattern = Pattern.compile("\\.taskmanager\\.([^.]+)\\..*");
    Set<String> registeredMetricsWithoutTaskManagerId = registeredMetrics.stream().map(pattern::matcher).flatMap(matcher -> matcher.find() ? Stream.of(matcher.group(0).replaceAll(matcher.group(1), "")) : Stream.empty()).collect(Collectors.toSet());
    assertThat(expectedTaskManagerMetricsWithoutTaskManagerId, everyItem(isIn(registeredMetricsWithoutTaskManagerId)));
}

Example 10 with RpcService

use of org.apache.flink.runtime.rpc.RpcService in project flink by apache.

the class IPv6HostnamesITCase method getLocalIPv6Address.

private Inet6Address getLocalIPv6Address() {
    try {
        Enumeration<NetworkInterface> e = NetworkInterface.getNetworkInterfaces();
        while (e.hasMoreElements()) {
            NetworkInterface netInterface = e.nextElement();
            // for each address of the network interface
            Enumeration<InetAddress> ee = netInterface.getInetAddresses();
            while (ee.hasMoreElements()) {
                InetAddress addr = ee.nextElement();
                if (addr instanceof Inet6Address && (!addr.isLoopbackAddress()) && (!addr.isAnyLocalAddress())) {
                    // see if it is possible to bind to the address
                    InetSocketAddress socketAddress = new InetSocketAddress(addr, 0);
                    try {
                        log.info("Considering address " + addr);
                        // test whether we can bind a socket to that address
                        log.info("Testing whether sockets can bind to " + addr);
                        ServerSocket sock = new ServerSocket();
                        sock.bind(socketAddress);
                        sock.close();
                        // test whether Akka's netty can bind to the address
                        log.info("Testing whether Akka can use " + addr);
                        try (NetUtils.Port port = NetUtils.getAvailablePort()) {
                            final RpcService rpcService = RpcSystem.load().localServiceBuilder(new Configuration()).withBindAddress(addr.getHostAddress()).withBindPort(port.getPort()).createAndStart();
                            rpcService.stopService().get();
                        }
                        log.info("Using address " + addr);
                        return (Inet6Address) addr;
                    } catch (IOException ignored) {
                    // fall through the loop
                    }
                }
            }
        }
        return null;
    } catch (Exception e) {
        return null;
    }
}