Examples with TaskManagerInfo org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerInfo used on opensource projects

Example 1 with TaskManagerInfo

use of org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerInfo in project flink by apache.

the class ResourceManager method requestTaskManagerInfo.

@Override
public CompletableFuture<Collection<TaskManagerInfo>> requestTaskManagerInfo(Time timeout) {
    final ArrayList<TaskManagerInfo> taskManagerInfos = new ArrayList<>(taskExecutors.size());
    for (Map.Entry<ResourceID, WorkerRegistration<WorkerType>> taskExecutorEntry : taskExecutors.entrySet()) {
        final ResourceID resourceId = taskExecutorEntry.getKey();
        final WorkerRegistration<WorkerType> taskExecutor = taskExecutorEntry.getValue();
        taskManagerInfos.add(new TaskManagerInfo(resourceId, taskExecutor.getTaskExecutorGateway().getAddress(), taskExecutor.getDataPort(), taskExecutor.getJmxPort(), taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId), slotManager.getNumberRegisteredSlotsOf(taskExecutor.getInstanceID()), slotManager.getNumberFreeSlotsOf(taskExecutor.getInstanceID()), slotManager.getRegisteredResourceOf(taskExecutor.getInstanceID()), slotManager.getFreeResourceOf(taskExecutor.getInstanceID()), taskExecutor.getHardwareDescription(), taskExecutor.getMemoryConfiguration()));
    }
    return CompletableFuture.completedFuture(taskManagerInfos);
}

Example 2 with TaskManagerInfo

use of org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerInfo in project flink by apache.

the class ResourceManager method requestTaskManagerDetailsInfo.

@Override
public CompletableFuture<TaskManagerInfoWithSlots> requestTaskManagerDetailsInfo(ResourceID resourceId, Time timeout) {
    final WorkerRegistration<WorkerType> taskExecutor = taskExecutors.get(resourceId);
    if (taskExecutor == null) {
        return FutureUtils.completedExceptionally(new UnknownTaskExecutorException(resourceId));
    } else {
        final InstanceID instanceId = taskExecutor.getInstanceID();
        final TaskManagerInfoWithSlots taskManagerInfoWithSlots = new TaskManagerInfoWithSlots(new TaskManagerInfo(resourceId, taskExecutor.getTaskExecutorGateway().getAddress(), taskExecutor.getDataPort(), taskExecutor.getJmxPort(), taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId), slotManager.getNumberRegisteredSlotsOf(instanceId), slotManager.getNumberFreeSlotsOf(instanceId), slotManager.getRegisteredResourceOf(instanceId), slotManager.getFreeResourceOf(instanceId), taskExecutor.getHardwareDescription(), taskExecutor.getMemoryConfiguration()), slotManager.getAllocatedSlotsOf(instanceId));
        return CompletableFuture.completedFuture(taskManagerInfoWithSlots);
    }
}

Example 3 with TaskManagerInfo

use of org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerInfo in project flink by apache.

the class ResourceManagerTest method testRequestTaskManagerInfo.

/**
 * Tests that we can retrieve the correct {@link TaskManagerInfo} from the {@link
 * ResourceManager}.
 */
@Test
public void testRequestTaskManagerInfo() throws Exception {
    final ResourceID taskManagerId = ResourceID.generate();
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setAddress(UUID.randomUUID().toString()).createTestingTaskExecutorGateway();
    rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);
    resourceManager = new ResourceManagerBuilder().buildAndStart();
    final ResourceManagerGateway resourceManagerGateway = resourceManager.getSelfGateway(ResourceManagerGateway.class);
    registerTaskExecutor(resourceManagerGateway, taskManagerId, taskExecutorGateway.getAddress());
    CompletableFuture<TaskManagerInfoWithSlots> taskManagerInfoFuture = resourceManagerGateway.requestTaskManagerDetailsInfo(taskManagerId, TestingUtils.TIMEOUT);
    TaskManagerInfoWithSlots taskManagerInfoWithSlots = taskManagerInfoFuture.get();
    TaskManagerInfo taskManagerInfo = taskManagerInfoWithSlots.getTaskManagerInfo();
    assertEquals(taskManagerId, taskManagerInfo.getResourceId());
    assertEquals(hardwareDescription, taskManagerInfo.getHardwareDescription());
    assertEquals(taskExecutorGateway.getAddress(), taskManagerInfo.getAddress());
    assertEquals(dataPort, taskManagerInfo.getDataPort());
    assertEquals(jmxPort, taskManagerInfo.getJmxPort());
    assertEquals(0, taskManagerInfo.getNumberSlots());
    assertEquals(0, taskManagerInfo.getNumberAvailableSlots());
    assertThat(taskManagerInfoWithSlots.getAllocatedSlots(), is(empty()));
}

Example 4 with TaskManagerInfo

use of org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerInfo in project flink by apache.

the class YarnConfigurationITCase method testFlinkContainerMemory.

/**
 * Tests that the Flink components are started with the correct memory settings.
 */
@Test(timeout = 60000)
public void testFlinkContainerMemory() throws Exception {
    runTest(() -> {
        final YarnClient yarnClient = getYarnClient();
        final Configuration configuration = new Configuration(flinkConfiguration);
        final int slotsPerTaskManager = 3;
        configuration.set(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);
        final int masterMemory = 768;
        configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(masterMemory));
        final TaskExecutorProcessSpec tmResourceSpec = TaskExecutorProcessUtils.processSpecFromConfig(configuration);
        final int taskManagerMemory = tmResourceSpec.getTotalProcessMemorySize().getMebiBytes();
        final YarnConfiguration yarnConfiguration = getYarnConfiguration();
        final YarnClusterDescriptor clusterDescriptor = YarnTestUtils.createClusterDescriptorWithLogging(CliFrontend.getConfigurationDirectoryFromEnv(), configuration, yarnConfiguration, yarnClient, true);
        clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
        clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
        final File streamingWordCountFile = getTestJarPath("WindowJoin.jar");
        final PackagedProgram packagedProgram = PackagedProgram.newBuilder().setJarFile(streamingWordCountFile).build();
        final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, configuration, 1, false);
        try {
            final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder().setMasterMemoryMB(masterMemory).setTaskManagerMemoryMB(taskManagerMemory).setSlotsPerTaskManager(slotsPerTaskManager).createClusterSpecification();
            final ClusterClient<ApplicationId> clusterClient = clusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true).getClusterClient();
            final ApplicationId clusterId = clusterClient.getClusterId();
            final RestClient restClient = new RestClient(configuration, TestingUtils.defaultExecutor());
            try {
                final ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterId);
                final ApplicationAttemptId currentApplicationAttemptId = applicationReport.getCurrentApplicationAttemptId();
                // wait until we have second container allocated
                List<ContainerReport> containers = yarnClient.getContainers(currentApplicationAttemptId);
                while (containers.size() < 2) {
                    // this is nasty but Yarn does not offer a better way to wait
                    Thread.sleep(50L);
                    containers = yarnClient.getContainers(currentApplicationAttemptId);
                }
                for (ContainerReport container : containers) {
                    if (container.getContainerId().getId() == 1) {
                        // this should be the application master
                        assertThat(container.getAllocatedResource().getMemory(), is(masterMemory));
                    } else {
                        assertThat(container.getAllocatedResource().getMemory(), is(taskManagerMemory));
                    }
                }
                final URI webURI = new URI(clusterClient.getWebInterfaceURL());
                CompletableFuture<TaskManagersInfo> taskManagersInfoCompletableFuture;
                Collection<TaskManagerInfo> taskManagerInfos;
                while (true) {
                    taskManagersInfoCompletableFuture = restClient.sendRequest(webURI.getHost(), webURI.getPort(), TaskManagersHeaders.getInstance(), EmptyMessageParameters.getInstance(), EmptyRequestBody.getInstance());
                    final TaskManagersInfo taskManagersInfo = taskManagersInfoCompletableFuture.get();
                    taskManagerInfos = taskManagersInfo.getTaskManagerInfos();
                    // wait until the task manager has registered and reported its slots
                    if (hasTaskManagerConnectedAndReportedSlots(taskManagerInfos)) {
                        break;
                    } else {
                        Thread.sleep(100L);
                    }
                }
                // there should be at least one TaskManagerInfo
                final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next();
                assertThat(taskManagerInfo.getNumberSlots(), is(slotsPerTaskManager));
                final long expectedHeapSizeBytes = tmResourceSpec.getJvmHeapMemorySize().getBytes();
                // We compare here physical memory assigned to a container with the heap
                // memory that we should pass to
                // jvm as Xmx parameter. Those value might differ significantly due to
                // system page size or jvm
                // implementation therefore we use 15% threshold here.
                assertThat((double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap() / (double) expectedHeapSizeBytes, is(closeTo(1.0, 0.15)));
                final int expectedManagedMemoryMB = tmResourceSpec.getManagedMemorySize().getMebiBytes();
                assertThat((int) (taskManagerInfo.getHardwareDescription().getSizeOfManagedMemory() >> 20), is(expectedManagedMemoryMB));
            } finally {
                restClient.shutdown(TIMEOUT);
                clusterClient.close();
            }
            clusterDescriptor.killCluster(clusterId);
        } finally {
            clusterDescriptor.close();
        }
    });
}