Examples with JobVertexID org.apache.flink.runtime.jobgraph.JobVertexID used on opensource projects

Example 11 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class SchedulerSlotSharingTest method testLocalizedAssignment3.

/**
	 * Tests that the scheduler can fall back to non-local
	 */
@Test
public void testLocalizedAssignment3() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        // schedule until the one instance is full
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc1), sharingGroup), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 4, loc1), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 4, loc1), sharingGroup), false).get();
        // schedule two more with preference of same instance --> need to go to other instance
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 3, 4, loc1), sharingGroup), false).get();
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 4, 4, loc1), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertNotNull(s3);
        assertNotNull(s4);
        assertNotNull(s5);
        assertNotNull(s6);
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, i1.getNumberOfAvailableSlots());
        assertEquals(0, i2.getNumberOfAvailableSlots());
        assertEquals(i1.getTaskManagerID(), s1.getTaskManagerID());
        assertEquals(i1.getTaskManagerID(), s2.getTaskManagerID());
        assertEquals(i1.getTaskManagerID(), s3.getTaskManagerID());
        assertEquals(i1.getTaskManagerID(), s4.getTaskManagerID());
        assertEquals(i2.getTaskManagerID(), s5.getTaskManagerID());
        assertEquals(i2.getTaskManagerID(), s6.getTaskManagerID());
        // check the scheduler's bookkeeping
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(2, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 12 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class SchedulerSlotSharingTest method testLocalizedAssignment2.

/**
	 * Tests that the scheduler assigns to new local slots, rather than to existing non-local slots
	 */
@Test
public void testLocalizedAssignment2() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        // schedule one to each instance
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc1), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertEquals(2, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, i1.getNumberOfAvailableSlots());
        assertEquals(2, i2.getNumberOfAvailableSlots());
        // schedule one from the other group to each instance
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc2), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc2), sharingGroup), false).get();
        assertNotNull(s3);
        assertNotNull(s4);
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, i1.getNumberOfAvailableSlots());
        assertEquals(0, i2.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 13 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CoLocationConstraintTest method testCreateConstraints.

@Test
public void testCreateConstraints() {
    try {
        JobVertexID id1 = new JobVertexID();
        JobVertexID id2 = new JobVertexID();
        JobVertex vertex1 = new JobVertex("vertex1", id1);
        vertex1.setParallelism(2);
        JobVertex vertex2 = new JobVertex("vertex2", id2);
        vertex2.setParallelism(3);
        CoLocationGroup group = new CoLocationGroup(vertex1, vertex2);
        AbstractID groupId = group.getId();
        assertNotNull(groupId);
        CoLocationConstraint constraint1 = group.getLocationConstraint(0);
        CoLocationConstraint constraint2 = group.getLocationConstraint(1);
        CoLocationConstraint constraint3 = group.getLocationConstraint(2);
        assertFalse(constraint1 == constraint2);
        assertFalse(constraint1 == constraint3);
        assertFalse(constraint2 == constraint3);
        assertEquals(groupId, constraint1.getGroupId());
        assertEquals(groupId, constraint2.getGroupId());
        assertEquals(groupId, constraint3.getGroupId());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 14 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CheckpointMessagesTest method testConfirmTaskCheckpointed.

@Test
public void testConfirmTaskCheckpointed() {
    try {
        AcknowledgeCheckpoint noState = new AcknowledgeCheckpoint(new JobID(), new ExecutionAttemptID(), 569345L);
        KeyGroupRange keyGroupRange = KeyGroupRange.of(42, 42);
        SubtaskState checkpointStateHandles = new SubtaskState(CheckpointCoordinatorTest.generateChainedStateHandle(new MyHandle()), CheckpointCoordinatorTest.generateChainedPartitionableStateHandle(new JobVertexID(), 0, 2, 8, false), null, CheckpointCoordinatorTest.generateKeyGroupState(keyGroupRange, Collections.singletonList(new MyHandle())), null);
        AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(new JobID(), new ExecutionAttemptID(), 87658976143L, new CheckpointMetrics(), checkpointStateHandles);
        testSerializabilityEqualsHashCode(noState);
        testSerializabilityEqualsHashCode(withState);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 15 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class ScheduleWithCoLocationHintTest method scheduleMixedCoLocationSlotSharing.

@Test
public void scheduleMixedCoLocationSlotSharing() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        JobVertexID jid4 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(1));
        scheduler.newInstanceAvailable(getRandomInstance(1));
        scheduler.newInstanceAvailable(getRandomInstance(1));
        scheduler.newInstanceAvailable(getRandomInstance(1));
        assertEquals(4, scheduler.getNumberOfAvailableSlots());
        CoLocationGroup grp = new CoLocationGroup();
        CoLocationConstraint clc1 = new CoLocationConstraint(grp);
        CoLocationConstraint clc2 = new CoLocationConstraint(grp);
        CoLocationConstraint clc3 = new CoLocationConstraint(grp);
        CoLocationConstraint clc4 = new CoLocationConstraint(grp);
        SlotSharingGroup shareGroup = new SlotSharingGroup();
        // first wave
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4), shareGroup), false);
        // second wave
        SimpleSlot s21 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4), shareGroup, clc1), false).get();
        SimpleSlot s22 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4), shareGroup, clc2), false).get();
        SimpleSlot s23 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4), shareGroup, clc3), false).get();
        SimpleSlot s24 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4), shareGroup, clc4), false).get();
        // third wave
        SimpleSlot s31 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 4), shareGroup, clc2), false).get();
        SimpleSlot s32 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 4), shareGroup, clc3), false).get();
        SimpleSlot s33 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 4), shareGroup, clc4), false).get();
        SimpleSlot s34 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 4), shareGroup, clc1), false).get();
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4), shareGroup), false);
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4), shareGroup), false);
        assertEquals(s21.getTaskManagerID(), s34.getTaskManagerID());
        assertEquals(s22.getTaskManagerID(), s31.getTaskManagerID());
        assertEquals(s23.getTaskManagerID(), s32.getTaskManagerID());
        assertEquals(s24.getTaskManagerID(), s33.getTaskManagerID());
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(12, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}