Examples with TaskManagerLocation org.apache.flink.runtime.taskmanager.TaskManagerLocation used on opensource projects

Example 21 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class ExecutionGraphTestUtils method getInstance.

public static Instance getInstance(final TaskManagerGateway gateway, final int numberOfSlots) throws Exception {
    ResourceID resourceID = ResourceID.generate();
    HardwareDescription hardwareDescription = new HardwareDescription(4, 2L * 1024 * 1024 * 1024, 1024 * 1024 * 1024, 512 * 1024 * 1024);
    InetAddress address = InetAddress.getByName("127.0.0.1");
    TaskManagerLocation connection = new TaskManagerLocation(resourceID, address, 10001);
    return new Instance(gateway, connection, new InstanceID(), hardwareDescription, numberOfSlots);
}

Example 22 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class InstanceManagerTest method testReportHeartbeat.

@Test
public void testReportHeartbeat() {
    try {
        InstanceManager cm = new InstanceManager();
        final int dataPort = 20000;
        ResourceID resID1 = ResourceID.generate();
        ResourceID resID2 = ResourceID.generate();
        ResourceID resID3 = ResourceID.generate();
        HardwareDescription hardwareDescription = HardwareDescription.extractFromSystem(4096);
        InetAddress address = InetAddress.getByName("127.0.0.1");
        // register three instances
        TaskManagerLocation ici1 = new TaskManagerLocation(resID1, address, dataPort);
        TaskManagerLocation ici2 = new TaskManagerLocation(resID2, address, dataPort + 1);
        TaskManagerLocation ici3 = new TaskManagerLocation(resID3, address, dataPort + 2);
        JavaTestKit probe1 = new JavaTestKit(system);
        JavaTestKit probe2 = new JavaTestKit(system);
        JavaTestKit probe3 = new JavaTestKit(system);
        InstanceID instanceID1 = cm.registerTaskManager(new ActorTaskManagerGateway(new AkkaActorGateway(probe1.getRef(), leaderSessionID)), ici1, hardwareDescription, 1);
        InstanceID instanceID2 = cm.registerTaskManager(new ActorTaskManagerGateway(new AkkaActorGateway(probe2.getRef(), leaderSessionID)), ici2, hardwareDescription, 1);
        InstanceID instanceID3 = cm.registerTaskManager(new ActorTaskManagerGateway(new AkkaActorGateway(probe3.getRef(), leaderSessionID)), ici3, hardwareDescription, 1);
        // report some immediate heart beats
        assertTrue(cm.reportHeartBeat(instanceID1));
        assertTrue(cm.reportHeartBeat(instanceID2));
        assertTrue(cm.reportHeartBeat(instanceID3));
        // report heart beat for non-existing instance
        assertFalse(cm.reportHeartBeat(new InstanceID()));
        final long WAIT = 200;
        CommonTestUtils.sleepUninterruptibly(WAIT);
        Iterator<Instance> it = cm.getAllRegisteredInstances().iterator();
        Instance instance1 = it.next();
        long h1 = instance1.getLastHeartBeat();
        long h2 = it.next().getLastHeartBeat();
        long h3 = it.next().getLastHeartBeat();
        // send one heart beat again and verify that the
        assertTrue(cm.reportHeartBeat(instance1.getId()));
        long newH1 = instance1.getLastHeartBeat();
        long now = System.currentTimeMillis();
        assertTrue(now - h1 >= WAIT);
        assertTrue(now - h2 >= WAIT);
        assertTrue(now - h3 >= WAIT);
        assertTrue(now - newH1 <= WAIT);
        cm.shutdown();
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail("Test erroneous: " + e.getMessage());
    }
}

Example 23 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class InstanceTest method testAllocatingAndCancellingSlots.

@Test
public void testAllocatingAndCancellingSlots() {
    try {
        ResourceID resourceID = ResourceID.generate();
        HardwareDescription hardwareDescription = new HardwareDescription(4, 2L * 1024 * 1024 * 1024, 1024 * 1024 * 1024, 512 * 1024 * 1024);
        InetAddress address = InetAddress.getByName("127.0.0.1");
        TaskManagerLocation connection = new TaskManagerLocation(resourceID, address, 10001);
        Instance instance = new Instance(new ActorTaskManagerGateway(DummyActorGateway.INSTANCE), connection, new InstanceID(), hardwareDescription, 4);
        assertEquals(4, instance.getTotalNumberOfSlots());
        assertEquals(4, instance.getNumberOfAvailableSlots());
        assertEquals(0, instance.getNumberOfAllocatedSlots());
        SimpleSlot slot1 = instance.allocateSimpleSlot(new JobID());
        SimpleSlot slot2 = instance.allocateSimpleSlot(new JobID());
        SimpleSlot slot3 = instance.allocateSimpleSlot(new JobID());
        SimpleSlot slot4 = instance.allocateSimpleSlot(new JobID());
        assertNotNull(slot1);
        assertNotNull(slot2);
        assertNotNull(slot3);
        assertNotNull(slot4);
        assertEquals(0, instance.getNumberOfAvailableSlots());
        assertEquals(4, instance.getNumberOfAllocatedSlots());
        assertEquals(6, slot1.getSlotNumber() + slot2.getSlotNumber() + slot3.getSlotNumber() + slot4.getSlotNumber());
        // no more slots
        assertNull(instance.allocateSimpleSlot(new JobID()));
        try {
            instance.returnAllocatedSlot(slot2);
            fail("instance accepted a non-cancelled slot.");
        } catch (IllegalArgumentException e) {
        // good
        }
        // release the slots. this returns them to the instance
        slot1.releaseSlot();
        slot2.releaseSlot();
        slot3.releaseSlot();
        slot4.releaseSlot();
        assertEquals(4, instance.getNumberOfAvailableSlots());
        assertEquals(0, instance.getNumberOfAllocatedSlots());
        assertFalse(instance.returnAllocatedSlot(slot1));
        assertFalse(instance.returnAllocatedSlot(slot2));
        assertFalse(instance.returnAllocatedSlot(slot3));
        assertFalse(instance.returnAllocatedSlot(slot4));
        assertEquals(4, instance.getNumberOfAvailableSlots());
        assertEquals(0, instance.getNumberOfAllocatedSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 24 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class InstanceTest method testInstanceDies.

@Test
public void testInstanceDies() {
    try {
        ResourceID resourceID = ResourceID.generate();
        HardwareDescription hardwareDescription = new HardwareDescription(4, 2L * 1024 * 1024 * 1024, 1024 * 1024 * 1024, 512 * 1024 * 1024);
        InetAddress address = InetAddress.getByName("127.0.0.1");
        TaskManagerLocation connection = new TaskManagerLocation(resourceID, address, 10001);
        Instance instance = new Instance(new ActorTaskManagerGateway(DummyActorGateway.INSTANCE), connection, new InstanceID(), hardwareDescription, 3);
        assertEquals(3, instance.getNumberOfAvailableSlots());
        SimpleSlot slot1 = instance.allocateSimpleSlot(new JobID());
        SimpleSlot slot2 = instance.allocateSimpleSlot(new JobID());
        SimpleSlot slot3 = instance.allocateSimpleSlot(new JobID());
        instance.markDead();
        assertEquals(0, instance.getNumberOfAllocatedSlots());
        assertEquals(0, instance.getNumberOfAvailableSlots());
        assertTrue(slot1.isCanceled());
        assertTrue(slot2.isCanceled());
        assertTrue(slot3.isCanceled());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 25 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class SharedSlotsTest method testAllocateAndReleaseTwoLevels.

/**
	 * We allocate and release the structure below, starting by allocating a simple slot in the
	 * shared slot and finishing by releasing a simple slot.
	 * 
	 * <pre>
	 *     Shared(0)(root)
	 *        |
	 *        +-- Simple(2)(sink)
	 *        |
	 *        +-- Shared(1)(co-location-group)
	 *        |      |
	 *        |      +-- Simple(0)(tail)
	 *        |      +-- Simple(1)(head)
	 *        |
	 *        +-- Simple(0)(source)
	 * </pre>
	 */
@Test
public void testAllocateAndReleaseTwoLevels() {
    try {
        JobVertexID sourceId = new JobVertexID();
        JobVertexID headId = new JobVertexID();
        JobVertexID tailId = new JobVertexID();
        JobVertexID sinkId = new JobVertexID();
        JobVertex headVertex = new JobVertex("head", headId);
        JobVertex tailVertex = new JobVertex("tail", tailId);
        SlotSharingGroup sharingGroup = new SlotSharingGroup(sourceId, headId, tailId, sinkId);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        assertEquals(0, assignment.getNumberOfSlots());
        CoLocationGroup coLocationGroup = new CoLocationGroup(headVertex, tailVertex);
        CoLocationConstraint constraint = coLocationGroup.getLocationConstraint(0);
        assertFalse(constraint.isAssigned());
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        // allocate a shared slot
        SharedSlot sharedSlot = instance.allocateSharedSlot(new JobID(), assignment);
        // get the first simple slot
        SimpleSlot sourceSlot = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.LOCAL, sourceId);
        assertEquals(1, sharedSlot.getNumberLeaves());
        // get the first slot in the nested shared slot from the co-location constraint
        SimpleSlot headSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
        assertEquals(2, sharedSlot.getNumberLeaves());
        assertNotNull(constraint.getSharedSlot());
        assertTrue(constraint.getSharedSlot().isAlive());
        assertFalse(constraint.isAssigned());
        // we do not immediately lock the location
        headSlot.releaseSlot();
        assertEquals(1, sharedSlot.getNumberLeaves());
        assertNotNull(constraint.getSharedSlot());
        assertTrue(constraint.getSharedSlot().isReleased());
        assertFalse(constraint.isAssigned());
        // re-allocate the head slot
        headSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
        constraint.lockLocation();
        assertNotNull(constraint.getSharedSlot());
        assertTrue(constraint.isAssigned());
        assertTrue(constraint.isAssignedAndAlive());
        assertEquals(instance.getTaskManagerLocation(), constraint.getLocation());
        SimpleSlot tailSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
        assertEquals(constraint.getSharedSlot(), headSlot.getParent());
        assertEquals(constraint.getSharedSlot(), tailSlot.getParent());
        SimpleSlot sinkSlot = assignment.getSlotForTask(sinkId, Collections.<TaskManagerLocation>emptySet());
        assertEquals(4, sharedSlot.getNumberLeaves());
        // we release our co-location constraint tasks
        headSlot.releaseSlot();
        tailSlot.releaseSlot();
        assertEquals(2, sharedSlot.getNumberLeaves());
        assertTrue(headSlot.isReleased());
        assertTrue(tailSlot.isReleased());
        assertTrue(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        assertEquals(instance.getTaskManagerLocation(), constraint.getLocation());
        // we should have resources again for the co-location constraint
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(constraint.getGroupId()));
        // re-allocate head and tail from the constraint
        headSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
        tailSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
        assertEquals(4, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(constraint.getGroupId()));
        // verify some basic properties of the slots
        assertEquals(instance.getTaskManagerID(), sourceSlot.getTaskManagerID());
        assertEquals(instance.getTaskManagerID(), headSlot.getTaskManagerID());
        assertEquals(instance.getTaskManagerID(), tailSlot.getTaskManagerID());
        assertEquals(instance.getTaskManagerID(), sinkSlot.getTaskManagerID());
        assertEquals(sourceId, sourceSlot.getGroupID());
        assertEquals(sinkId, sinkSlot.getGroupID());
        assertNull(headSlot.getGroupID());
        assertNull(tailSlot.getGroupID());
        assertEquals(constraint.getGroupId(), constraint.getSharedSlot().getGroupID());
        // release all
        sourceSlot.releaseSlot();
        headSlot.releaseSlot();
        tailSlot.releaseSlot();
        sinkSlot.releaseSlot();
        assertTrue(sharedSlot.isReleased());
        assertTrue(sourceSlot.isReleased());
        assertTrue(headSlot.isReleased());
        assertTrue(tailSlot.isReleased());
        assertTrue(sinkSlot.isReleased());
        assertTrue(constraint.getSharedSlot().isReleased());
        assertTrue(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, assignment.getNumberOfSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}