Examples with TaskManagerLocation org.apache.flink.runtime.taskmanager.TaskManagerLocation used on opensource projects

Example 66 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class ExecutionVertexLocalityTest method testLocalityInputBasedForward.

/**
	 * This test validates that vertices that have only one input stream try to
	 * co-locate their tasks with the producer.
	 */
@Test
public void testLocalityInputBasedForward() throws Exception {
    final int parallelism = 10;
    final TaskManagerLocation[] locations = new TaskManagerLocation[parallelism];
    final ExecutionGraph graph = createTestGraph(parallelism, false);
    // set the location for all sources to a distinct location
    for (int i = 0; i < parallelism; i++) {
        ExecutionVertex source = graph.getAllVertices().get(sourceVertexId).getTaskVertices()[i];
        TaskManagerLocation location = new TaskManagerLocation(ResourceID.generate(), InetAddress.getLoopbackAddress(), 10000 + i);
        locations[i] = location;
        initializeLocation(source, location);
    }
    // validate that the target vertices have no location preference
    for (int i = 0; i < parallelism; i++) {
        ExecutionVertex target = graph.getAllVertices().get(targetVertexId).getTaskVertices()[i];
        Iterator<TaskManagerLocation> preference = target.getPreferredLocations().iterator();
        assertTrue(preference.hasNext());
        assertEquals(locations[i], preference.next());
        assertFalse(preference.hasNext());
    }
}

Example 67 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class InputChannelDeploymentDescriptorTest method mockSlot.

// ------------------------------------------------------------------------
private static SimpleSlot mockSlot(ResourceID resourceId) {
    SimpleSlot slot = mock(SimpleSlot.class);
    when(slot.getTaskManagerLocation()).thenReturn(new TaskManagerLocation(resourceId, InetAddress.getLoopbackAddress(), 5000));
    when(slot.getTaskManagerID()).thenReturn(resourceId);
    return slot;
}

Example 68 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class SchedulerTestUtils method getRandomInstance.

// --------------------------------------------------------------------------------------------
public static Instance getRandomInstance(int numSlots) {
    if (numSlots <= 0) {
        throw new IllegalArgumentException();
    }
    final ResourceID resourceID = ResourceID.generate();
    final InetAddress address;
    try {
        address = InetAddress.getByName("127.0.0.1");
    } catch (UnknownHostException e) {
        throw new RuntimeException("Test could not create IP address for localhost loopback.");
    }
    int dataPort = port.getAndIncrement();
    TaskManagerLocation ci = new TaskManagerLocation(resourceID, address, dataPort);
    final long GB = 1024L * 1024 * 1024;
    HardwareDescription resources = new HardwareDescription(4, 4 * GB, 3 * GB, 2 * GB);
    return new Instance(new ActorTaskManagerGateway(DummyActorGateway.INSTANCE), ci, new InstanceID(), resources, numSlots);
}

Example 69 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class JobMasterTest method testHeartbeatTimeoutWithTaskManager.

@Test
public void testHeartbeatTimeoutWithTaskManager() throws Exception {
    final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServices();
    final TestingLeaderRetrievalService rmLeaderRetrievalService = new TestingLeaderRetrievalService();
    haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);
    haServices.setCheckpointRecoveryFactory(mock(CheckpointRecoveryFactory.class));
    final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
    final String jobManagerAddress = "jm";
    final UUID jmLeaderId = UUID.randomUUID();
    final ResourceID jmResourceId = new ResourceID(jobManagerAddress);
    final String taskManagerAddress = "tm";
    final ResourceID tmResourceId = new ResourceID(taskManagerAddress);
    final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(tmResourceId, InetAddress.getLoopbackAddress(), 1234);
    final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
    final TestingSerialRpcService rpc = new TestingSerialRpcService();
    rpc.registerGateway(taskManagerAddress, taskExecutorGateway);
    final long heartbeatInterval = 1L;
    final long heartbeatTimeout = 5L;
    final ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
    final HeartbeatServices heartbeatServices = new TestingHeartbeatServices(heartbeatInterval, heartbeatTimeout, scheduledExecutor);
    final JobGraph jobGraph = new JobGraph();
    try {
        final JobMaster jobMaster = new JobMaster(jmResourceId, jobGraph, new Configuration(), rpc, haServices, heartbeatServices, Executors.newScheduledThreadPool(1), mock(BlobLibraryCacheManager.class), mock(RestartStrategyFactory.class), Time.of(10, TimeUnit.SECONDS), null, mock(OnCompletionActions.class), testingFatalErrorHandler, new FlinkUserCodeClassLoader(new URL[0]));
        // also start the heartbeat manager in job manager
        jobMaster.start(jmLeaderId);
        // register task manager will trigger monitoring heartbeat target, schedule heartbeat request in interval time
        jobMaster.registerTaskManager(taskManagerAddress, taskManagerLocation, jmLeaderId);
        ArgumentCaptor<Runnable> heartbeatRunnableCaptor = ArgumentCaptor.forClass(Runnable.class);
        verify(scheduledExecutor, times(1)).scheduleAtFixedRate(heartbeatRunnableCaptor.capture(), eq(0L), eq(heartbeatInterval), eq(TimeUnit.MILLISECONDS));
        Runnable heartbeatRunnable = heartbeatRunnableCaptor.getValue();
        ArgumentCaptor<Runnable> timeoutRunnableCaptor = ArgumentCaptor.forClass(Runnable.class);
        verify(scheduledExecutor).schedule(timeoutRunnableCaptor.capture(), eq(heartbeatTimeout), eq(TimeUnit.MILLISECONDS));
        Runnable timeoutRunnable = timeoutRunnableCaptor.getValue();
        // run the first heartbeat request
        heartbeatRunnable.run();
        verify(taskExecutorGateway, times(1)).heartbeatFromJobManager(eq(jmResourceId));
        // run the timeout runnable to simulate a heartbeat timeout
        timeoutRunnable.run();
        verify(taskExecutorGateway).disconnectJobManager(eq(jobGraph.getJobID()), any(TimeoutException.class));
        // check if a concurrent error occurred
        testingFatalErrorHandler.rethrowError();
    } finally {
        rpc.stopService();
    }
}

Example 70 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class ScheduleWithCoLocationHintTest method testSlotReleasedInBetweenAndNoNewLocal.

@Test
public void testSlotReleasedInBetweenAndNoNewLocal() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jidx = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i1);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
        CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup, cc1), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup, cc2), false).get();
        s1.releaseSlot();
        s2.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        SimpleSlot sa = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jidx, 0, 2)), false).get();
        SimpleSlot sb = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jidx, 1, 2)), false).get();
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc2), sharingGroup, cc1), false).get();
            fail("should not be able to find a resource");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("wrong exception");
        }
        sa.releaseSlot();
        sb.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(2, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(2, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}