Examples with JobVertexID org.apache.flink.runtime.jobgraph.JobVertexID used on opensource projects

Example 86 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CoLocationConstraintTest method testCreateConstraints.

@Test
public void testCreateConstraints() {
    try {
        JobVertexID id1 = new JobVertexID();
        JobVertexID id2 = new JobVertexID();
        JobVertex vertex1 = new JobVertex("vertex1", id1);
        vertex1.setParallelism(2);
        JobVertex vertex2 = new JobVertex("vertex2", id2);
        vertex2.setParallelism(3);
        CoLocationGroup group = new CoLocationGroup(vertex1, vertex2);
        AbstractID groupId = group.getId();
        assertNotNull(groupId);
        CoLocationConstraint constraint1 = group.getLocationConstraint(0);
        CoLocationConstraint constraint2 = group.getLocationConstraint(1);
        CoLocationConstraint constraint3 = group.getLocationConstraint(2);
        assertFalse(constraint1 == constraint2);
        assertFalse(constraint1 == constraint3);
        assertFalse(constraint2 == constraint3);
        assertEquals(groupId, constraint1.getGroupId());
        assertEquals(groupId, constraint2.getGroupId());
        assertEquals(groupId, constraint3.getGroupId());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 87 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CheckpointMessagesTest method testConfirmTaskCheckpointed.

@Test
public void testConfirmTaskCheckpointed() {
    try {
        AcknowledgeCheckpoint noState = new AcknowledgeCheckpoint(new JobID(), new ExecutionAttemptID(), 569345L);
        KeyGroupRange keyGroupRange = KeyGroupRange.of(42, 42);
        SubtaskState checkpointStateHandles = new SubtaskState(CheckpointCoordinatorTest.generateChainedStateHandle(new MyHandle()), CheckpointCoordinatorTest.generateChainedPartitionableStateHandle(new JobVertexID(), 0, 2, 8, false), null, CheckpointCoordinatorTest.generateKeyGroupState(keyGroupRange, Collections.singletonList(new MyHandle())), null);
        AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(new JobID(), new ExecutionAttemptID(), 87658976143L, new CheckpointMetrics(), checkpointStateHandles);
        testSerializabilityEqualsHashCode(noState);
        testSerializabilityEqualsHashCode(withState);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 88 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class ScheduleWithCoLocationHintTest method scheduleMixedCoLocationSlotSharing.

@Test
public void scheduleMixedCoLocationSlotSharing() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        JobVertexID jid4 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(1));
        scheduler.newInstanceAvailable(getRandomInstance(1));
        scheduler.newInstanceAvailable(getRandomInstance(1));
        scheduler.newInstanceAvailable(getRandomInstance(1));
        assertEquals(4, scheduler.getNumberOfAvailableSlots());
        CoLocationGroup grp = new CoLocationGroup();
        CoLocationConstraint clc1 = new CoLocationConstraint(grp);
        CoLocationConstraint clc2 = new CoLocationConstraint(grp);
        CoLocationConstraint clc3 = new CoLocationConstraint(grp);
        CoLocationConstraint clc4 = new CoLocationConstraint(grp);
        SlotSharingGroup shareGroup = new SlotSharingGroup();
        // first wave
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4), shareGroup), false);
        // second wave
        SimpleSlot s21 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4), shareGroup, clc1), false).get();
        SimpleSlot s22 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4), shareGroup, clc2), false).get();
        SimpleSlot s23 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4), shareGroup, clc3), false).get();
        SimpleSlot s24 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4), shareGroup, clc4), false).get();
        // third wave
        SimpleSlot s31 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 4), shareGroup, clc2), false).get();
        SimpleSlot s32 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 4), shareGroup, clc3), false).get();
        SimpleSlot s33 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 4), shareGroup, clc4), false).get();
        SimpleSlot s34 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 4), shareGroup, clc1), false).get();
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4), shareGroup), false);
        assertEquals(s21.getTaskManagerID(), s34.getTaskManagerID());
        assertEquals(s22.getTaskManagerID(), s31.getTaskManagerID());
        assertEquals(s23.getTaskManagerID(), s32.getTaskManagerID());
        assertEquals(s24.getTaskManagerID(), s33.getTaskManagerID());
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(12, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 89 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class ScheduleWithCoLocationHintTest method testSlotReleasedInBetween.

@Test
public void testSlotReleasedInBetween() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i1);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
        CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup, cc1), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup, cc2), false).get();
        s1.releaseSlot();
        s2.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc2), sharingGroup, cc1), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc1), sharingGroup, cc2), false).get();
        // still preserves the previous instance mapping)
        assertEquals(i1.getTaskManagerID(), s3.getTaskManagerID());
        assertEquals(i2.getTaskManagerID(), s4.getTaskManagerID());
        s3.releaseSlot();
        s4.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 90 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class SchedulerSlotSharingTest method testSequentialAllocateAndRelease.

@Test
public void testSequentialAllocateAndRelease() {
    try {
        final JobVertexID jid1 = new JobVertexID();
        final JobVertexID jid2 = new JobVertexID();
        final JobVertexID jid3 = new JobVertexID();
        final JobVertexID jid4 = new JobVertexID();
        final SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2, jid3, jid4);
        final Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(4));
        // allocate something from group 1 and 2 interleaved with schedule for group 3
        SimpleSlot slot_1_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4), sharingGroup), false).get();
        SimpleSlot slot_1_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4), sharingGroup), false).get();
        SimpleSlot slot_2_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4), sharingGroup), false).get();
        SimpleSlot slot_2_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4), sharingGroup), false).get();
        SimpleSlot slot_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1), sharingGroup), false).get();
        SimpleSlot slot_1_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4), sharingGroup), false).get();
        SimpleSlot slot_1_4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4), sharingGroup), false).get();
        SimpleSlot slot_2_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4), sharingGroup), false).get();
        SimpleSlot slot_2_4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4), sharingGroup), false).get();
        // release groups 1 and 2
        slot_1_1.releaseSlot();
        slot_1_2.releaseSlot();
        slot_1_3.releaseSlot();
        slot_1_4.releaseSlot();
        slot_2_1.releaseSlot();
        slot_2_2.releaseSlot();
        slot_2_3.releaseSlot();
        slot_2_4.releaseSlot();
        // allocate group 4
        SimpleSlot slot_4_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4), sharingGroup), false).get();
        SimpleSlot slot_4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4), sharingGroup), false).get();
        SimpleSlot slot_4_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4), sharingGroup), false).get();
        SimpleSlot slot_4_4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4), sharingGroup), false).get();
        // release groups 3 and 4
        slot_3.releaseSlot();
        slot_4_1.releaseSlot();
        slot_4_2.releaseSlot();
        slot_4_3.releaseSlot();
        slot_4_4.releaseSlot();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}