Examples with LocalUnresolvedTaskManagerLocation org.apache.flink.runtime.taskmanager.LocalUnresolvedTaskManagerLocation used on opensource projects

Example 1 with LocalUnresolvedTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalUnresolvedTaskManagerLocation in project flink by apache.

the class JobMasterTest method testJobMasterRejectsTaskExecutorRegistrationIfJobIdsAreNotEqual.

/**
 * Tests that the JobMaster rejects a TaskExecutor registration attempt if the expected and
 * actual JobID are not equal. See FLINK-21606.
 */
@Test
public void testJobMasterRejectsTaskExecutorRegistrationIfJobIdsAreNotEqual() throws Exception {
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).createJobMaster();
    try {
        jobMaster.start();
        final CompletableFuture<RegistrationResponse> registrationResponse = jobMaster.registerTaskManager(new JobID(), TaskManagerRegistrationInformation.create("foobar", new LocalUnresolvedTaskManagerLocation(), TestingUtils.zeroUUID()), testingTimeout);
        assertThat(registrationResponse.get(), instanceOf(JMTMRegistrationRejection.class));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 2 with LocalUnresolvedTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalUnresolvedTaskManagerLocation in project flink by apache.

the class JobMasterTest method testJobMasterAcceptsSlotsWhenJobIsRestarting.

@Test
public void testJobMasterAcceptsSlotsWhenJobIsRestarting() throws Exception {
    configuration.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay");
    configuration.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY, Duration.ofDays(1));
    final int numberSlots = 1;
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withConfiguration(configuration).createJobMaster();
    try {
        jobMaster.start();
        final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
        final LocalUnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
        registerSlotsAtJobMaster(numberSlots, jobMasterGateway, jobGraph.getJobID(), new TestingTaskExecutorGatewayBuilder().setAddress("firstTaskManager").createTestingTaskExecutorGateway(), unresolvedTaskManagerLocation);
        CommonTestUtils.waitUntilCondition(() -> jobMasterGateway.requestJobStatus(testingTimeout).get() == JobStatus.RUNNING, Deadline.fromNow(TimeUtils.toDuration(testingTimeout)));
        jobMasterGateway.disconnectTaskManager(unresolvedTaskManagerLocation.getResourceID(), new FlinkException("Test exception."));
        CommonTestUtils.waitUntilCondition(() -> jobMasterGateway.requestJobStatus(testingTimeout).get() == JobStatus.RESTARTING, Deadline.fromNow(TimeUtils.toDuration(testingTimeout)));
        assertThat(registerSlotsAtJobMaster(numberSlots, jobMasterGateway, jobGraph.getJobID(), new TestingTaskExecutorGatewayBuilder().setAddress("secondTaskManager").createTestingTaskExecutorGateway(), new LocalUnresolvedTaskManagerLocation()), hasSize(numberSlots));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 3 with LocalUnresolvedTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalUnresolvedTaskManagerLocation in project flink by apache.

the class JobMasterTest method runHeartbeatTest.

private void runHeartbeatTest(TestingTaskExecutorGatewayBuilder testingTaskExecutorGatewayBuilder, HeartbeatServices heartbeatServices) throws Exception {
    final CompletableFuture<JobID> disconnectedJobManagerFuture = new CompletableFuture<>();
    final UnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
    final TestingTaskExecutorGateway taskExecutorGateway = testingTaskExecutorGatewayBuilder.setDisconnectJobManagerConsumer((jobId, throwable) -> disconnectedJobManagerFuture.complete(jobId)).createTestingTaskExecutorGateway();
    rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withResourceId(jmResourceId).withConfiguration(configuration).withHighAvailabilityServices(haServices).withHeartbeatServices(heartbeatServices).createJobMaster();
    jobMaster.start();
    try {
        final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);
        // register task manager will trigger monitor heartbeat target, schedule heartbeat
        // request at interval time
        CompletableFuture<RegistrationResponse> registrationResponse = jobMasterGateway.registerTaskManager(jobGraph.getJobID(), TaskManagerRegistrationInformation.create(taskExecutorGateway.getAddress(), unresolvedTaskManagerLocation, TestingUtils.zeroUUID()), testingTimeout);
        // wait for the completion of the registration
        registrationResponse.get();
        final JobID disconnectedJobManager = disconnectedJobManagerFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);
        assertThat(disconnectedJobManager, equalTo(jobGraph.getJobID()));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 4 with LocalUnresolvedTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalUnresolvedTaskManagerLocation in project flink by apache.

the class DefaultJobLeaderServiceTest method createAndStartJobLeaderService.

private JobLeaderService createAndStartJobLeaderService(HighAvailabilityServices haServices, JobLeaderListener testingJobLeaderListener) {
    final JobLeaderService jobLeaderService = new DefaultJobLeaderService(new LocalUnresolvedTaskManagerLocation(), RetryingRegistrationConfiguration.defaultConfiguration());
    jobLeaderService.start("foobar", rpcServiceResource.getTestingRpcService(), haServices, testingJobLeaderListener);
    return jobLeaderService;
}

Example 5 with LocalUnresolvedTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalUnresolvedTaskManagerLocation in project flink by apache.

the class TaskExecutorSlotLifetimeTest method testUserCodeClassLoaderIsBoundToSlot.

/**
 * Tests that the user code class loader is bound to the lifetime of the slot. This means that
 * it is being reused across a failover, for example. See FLINK-16408.
 */
@Test
public void testUserCodeClassLoaderIsBoundToSlot() throws Exception {
    final Configuration configuration = new Configuration();
    final TestingRpcService rpcService = TESTING_RPC_SERVICE_RESOURCE.getTestingRpcService();
    final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
    final CompletableFuture<SlotReport> firstSlotReportFuture = new CompletableFuture<>();
    resourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
        firstSlotReportFuture.complete(resourceIDInstanceIDSlotReportTuple3.f2);
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    final BlockingQueue<TaskExecutionState> taskExecutionStates = new ArrayBlockingQueue<>(3);
    final OneShotLatch slotsOfferedLatch = new OneShotLatch();
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setOfferSlotsFunction((resourceID, slotOffers) -> {
        slotsOfferedLatch.trigger();
        return CompletableFuture.completedFuture(slotOffers);
    }).setUpdateTaskExecutionStateFunction(FunctionUtils.uncheckedFunction(taskExecutionState -> {
        taskExecutionStates.put(taskExecutionState);
        return CompletableFuture.completedFuture(Acknowledge.get());
    })).build();
    final LeaderRetrievalService resourceManagerLeaderRetriever = new SettableLeaderRetrievalService(resourceManagerGateway.getAddress(), resourceManagerGateway.getFencingToken().toUUID());
    final LeaderRetrievalService jobMasterLeaderRetriever = new SettableLeaderRetrievalService(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
    final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServicesBuilder().setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever).setJobMasterLeaderRetrieverFunction(ignored -> jobMasterLeaderRetriever).build();
    rpcService.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
    rpcService.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final LocalUnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
    try (final TaskExecutor taskExecutor = createTaskExecutor(configuration, rpcService, haServices, unresolvedTaskManagerLocation)) {
        taskExecutor.start();
        final SlotReport slotReport = firstSlotReportFuture.join();
        final SlotID firstSlotId = slotReport.iterator().next().getSlotID();
        final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        final JobID jobId = new JobID();
        final AllocationID allocationId = new AllocationID();
        taskExecutorGateway.requestSlot(firstSlotId, jobId, allocationId, ResourceProfile.ZERO, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken(), RpcUtils.INF_TIMEOUT).join();
        final TaskDeploymentDescriptor tdd = TaskDeploymentDescriptorBuilder.newBuilder(jobId, UserClassLoaderExtractingInvokable.class).setAllocationId(allocationId).build();
        slotsOfferedLatch.await();
        taskExecutorGateway.submitTask(tdd, jobMasterGateway.getFencingToken(), RpcUtils.INF_TIMEOUT).join();
        final ClassLoader firstClassLoader = UserClassLoaderExtractingInvokable.take();
        // wait for the first task to finish
        TaskExecutionState taskExecutionState;
        do {
            taskExecutionState = taskExecutionStates.take();
        } while (!taskExecutionState.getExecutionState().isTerminal());
        // check that a second task will re-use the same class loader
        taskExecutorGateway.submitTask(tdd, jobMasterGateway.getFencingToken(), RpcUtils.INF_TIMEOUT).join();
        final ClassLoader secondClassLoader = UserClassLoaderExtractingInvokable.take();
        assertThat(firstClassLoader, sameInstance(secondClassLoader));
    }
}