Examples with TaskManagerServices org.apache.flink.runtime.taskexecutor.TaskManagerServices used on opensource projects

Example 1 with TaskManagerServices

use of org.apache.flink.runtime.taskexecutor.TaskManagerServices in project flink by apache.

the class TaskManagerMetricsTest method testMetricRegistryLifeCycle.

/**
	 * Tests the metric registry life cycle on JobManager re-connects.
	 */
@Test
public void testMetricRegistryLifeCycle() throws Exception {
    ActorSystem actorSystem = null;
    try {
        actorSystem = AkkaUtils.createLocalActorSystem(new Configuration());
        // ================================================================
        // Start JobManager
        // ================================================================
        final ActorRef jobManager = JobManager.startJobManagerActors(new Configuration(), actorSystem, TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), JobManager.class, MemoryArchivist.class)._1();
        LeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(jobManager.path().toString());
        // ================================================================
        // Start TaskManager
        // ================================================================
        final Configuration config = new Configuration();
        final ResourceID tmResourceID = ResourceID.generate();
        TaskManagerServicesConfiguration taskManagerServicesConfiguration = TaskManagerServicesConfiguration.fromConfiguration(config, InetAddress.getLocalHost(), false);
        TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(config);
        TaskManagerServices taskManagerServices = TaskManagerServices.fromConfiguration(taskManagerServicesConfiguration, tmResourceID);
        final MetricRegistry tmRegistry = taskManagerServices.getMetricRegistry();
        // create the task manager
        final Props tmProps = TaskManager.getTaskManagerProps(TaskManager.class, taskManagerConfiguration, tmResourceID, taskManagerServices.getTaskManagerLocation(), taskManagerServices.getMemoryManager(), taskManagerServices.getIOManager(), taskManagerServices.getNetworkEnvironment(), leaderRetrievalService, tmRegistry);
        final ActorRef taskManager = actorSystem.actorOf(tmProps);
        new JavaTestKit(actorSystem) {

            {
                new Within(new FiniteDuration(5000, TimeUnit.SECONDS)) {

                    @Override
                    protected void run() {
                        taskManager.tell(TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(), getTestActor());
                        // wait for the TM to be registered
                        expectMsgEquals(TaskManagerMessages.getRegisteredAtJobManagerMessage());
                        // trigger re-registration of TM; this should include a disconnect from the current JM
                        taskManager.tell(new TaskManagerMessages.JobManagerLeaderAddress(jobManager.path().toString(), null), jobManager);
                        // wait for re-registration to be completed
                        taskManager.tell(TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(), getTestActor());
                        expectMsgEquals(TaskManagerMessages.getRegisteredAtJobManagerMessage());
                    }
                };
            }
        };
        // verify that the registry was not shutdown due to the disconnect
        Assert.assertFalse(tmRegistry.isShutdown());
        // shut down the actors and the actor system
        actorSystem.shutdown();
        actorSystem.awaitTermination();
    } finally {
        if (actorSystem != null) {
            actorSystem.shutdown();
        }
    }
}

Example 2 with TaskManagerServices

use of org.apache.flink.runtime.taskexecutor.TaskManagerServices in project flink by apache.

the class MetricUtilsTest method testManagedMemoryMetricsInitialization.

@Test
public void testManagedMemoryMetricsInitialization() throws MemoryAllocationException {
    final int maxMemorySize = 16284;
    final int numberOfAllocatedPages = 2;
    final int pageSize = 4096;
    final Object owner = new Object();
    final MemoryManager memoryManager = MemoryManager.create(maxMemorySize, pageSize);
    memoryManager.allocatePages(owner, numberOfAllocatedPages);
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(new TestingTaskSlotTable.TestingTaskSlotTableBuilder<Task>().memoryManagerGetterReturns(memoryManager).allActiveSlotAllocationIds(() -> Sets.newHashSet(new AllocationID())).build()).setManagedMemorySize(maxMemorySize).build();
    List<String> actualSubGroupPath = new ArrayList<>();
    final InterceptingOperatorMetricGroup metricGroup = new InterceptingOperatorMetricGroup() {

        @Override
        public MetricGroup addGroup(String name) {
            actualSubGroupPath.add(name);
            return this;
        }
    };
    MetricUtils.instantiateFlinkMemoryMetricGroup(metricGroup, taskManagerServices.getTaskSlotTable(), taskManagerServices::getManagedMemorySize);
    Gauge<Number> usedMetric = (Gauge<Number>) metricGroup.get("Used");
    Gauge<Number> maxMetric = (Gauge<Number>) metricGroup.get("Total");
    assertThat(usedMetric.getValue().intValue(), is(numberOfAllocatedPages * pageSize));
    assertThat(maxMetric.getValue().intValue(), is(maxMemorySize));
    assertThat(actualSubGroupPath, is(Arrays.asList(METRIC_GROUP_FLINK, METRIC_GROUP_MEMORY, METRIC_GROUP_MANAGED_MEMORY)));
}

Example 3 with TaskManagerServices

use of org.apache.flink.runtime.taskexecutor.TaskManagerServices in project flink by apache.

the class TaskExecutorLocalStateStoresManagerTest method testCreationFromConfigDefault.

/**
 * This tests that the creation of {@link TaskManagerServices} correctly falls back to the first
 * tmp directory of the IOManager as default for the local state root directory.
 */
@Test
public void testCreationFromConfigDefault() throws Exception {
    final Configuration config = new Configuration();
    final WorkingDirectory workingDirectory = WORKING_DIRECTORY_RESOURCE.createNewWorkingDirectory();
    TaskManagerServicesConfiguration taskManagerServicesConfiguration = createTaskManagerServiceConfiguration(config, workingDirectory);
    TaskManagerServices taskManagerServices = createTaskManagerServices(taskManagerServicesConfiguration, workingDirectory);
    try {
        TaskExecutorLocalStateStoresManager taskStateManager = taskManagerServices.getTaskManagerStateStore();
        File[] localStateRootDirectories = taskStateManager.getLocalStateRootDirectories();
        for (int i = 0; i < localStateRootDirectories.length; ++i) {
            Assert.assertEquals(workingDirectory.getLocalStateDirectory(), localStateRootDirectories[i]);
        }
        Assert.assertFalse(taskStateManager.isLocalRecoveryEnabled());
    } finally {
        taskManagerServices.shutDown();
    }
}

Example 4 with TaskManagerServices

use of org.apache.flink.runtime.taskexecutor.TaskManagerServices in project flink by apache.

the class TaskExecutorLocalStateStoresManagerTest method testCreationFromConfig.

/**
 * This tests that the creation of {@link TaskManagerServices} correctly creates the local state
 * root directory for the {@link TaskExecutorLocalStateStoresManager} with the configured root
 * directory.
 */
@Test
public void testCreationFromConfig() throws Exception {
    final Configuration config = new Configuration();
    File newFolder = temporaryFolder.newFolder();
    String tmpDir = newFolder.getAbsolutePath() + File.separator;
    final String rootDirString = "__localStateRoot1,__localStateRoot2,__localStateRoot3".replaceAll("__", tmpDir);
    // test configuration of the local state directories
    config.setString(CheckpointingOptions.LOCAL_RECOVERY_TASK_MANAGER_STATE_ROOT_DIRS, rootDirString);
    // test configuration of the local state mode
    config.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);
    final WorkingDirectory workingDirectory = WORKING_DIRECTORY_RESOURCE.createNewWorkingDirectory();
    TaskManagerServices taskManagerServices = createTaskManagerServices(createTaskManagerServiceConfiguration(config, workingDirectory), workingDirectory);
    try {
        TaskExecutorLocalStateStoresManager taskStateManager = taskManagerServices.getTaskManagerStateStore();
        // verify configured directories for local state
        String[] split = rootDirString.split(",");
        File[] rootDirectories = taskStateManager.getLocalStateRootDirectories();
        for (int i = 0; i < split.length; ++i) {
            assertThat(rootDirectories[i].toPath()).startsWith(Paths.get(split[i]));
        }
        // verify local recovery mode
        Assert.assertTrue(taskStateManager.isLocalRecoveryEnabled());
        for (File rootDirectory : rootDirectories) {
            FileUtils.deleteFileOrDirectory(rootDirectory);
        }
    } finally {
        taskManagerServices.shutDown();
    }
}