Examples with AbstractID org.apache.flink.util.AbstractID used on opensource projects

Example 16 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class SharedSlotsTest method allocateAndReleaseInMixedOrder.

@Test
public void allocateAndReleaseInMixedOrder() {
    try {
        JobID jobId = new JobID();
        JobVertexID vid1 = new JobVertexID();
        JobVertexID vid2 = new JobVertexID();
        JobVertexID vid3 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vid1, vid2, vid3);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        // allocate a shared slot
        SharedSlot sharedSlot = instance.allocateSharedSlot(jobId, assignment);
        // allocate a series of sub slots
        SimpleSlot sub1 = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.UNCONSTRAINED, vid1);
        SimpleSlot sub2 = assignment.getSlotForTask(vid2, NO_LOCATION);
        assertNotNull(sub1);
        assertNotNull(sub2);
        assertEquals(2, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfSlots());
        sub2.releaseSlot();
        assertEquals(1, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfSlots());
        SimpleSlot sub3 = assignment.getSlotForTask(vid3, NO_LOCATION);
        assertNotNull(sub3);
        assertEquals(2, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfSlots());
        sub3.releaseSlot();
        sub1.releaseSlot();
        assertTrue(sharedSlot.isReleased());
        assertEquals(0, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(0, assignment.getNumberOfSlots());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, assignment.getNumberOfSlots());
        assertNull(sharedSlot.allocateSharedSlot(new AbstractID()));
        assertNull(sharedSlot.allocateSubSlot(new AbstractID()));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 17 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class SharedSlotsTest method allocateSimpleSlotsAndReleaseFromLeaves.

@Test
public void allocateSimpleSlotsAndReleaseFromLeaves() {
    try {
        JobID jobId = new JobID();
        JobVertexID vid1 = new JobVertexID();
        JobVertexID vid2 = new JobVertexID();
        JobVertexID vid3 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vid1, vid2, vid3);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        // allocate a shared slot
        SharedSlot sharedSlot = instance.allocateSharedSlot(jobId, assignment);
        // allocate a series of sub slots
        SimpleSlot sub1 = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.UNCONSTRAINED, vid1);
        SimpleSlot sub2 = assignment.getSlotForTask(vid2, NO_LOCATION);
        SimpleSlot sub3 = assignment.getSlotForTask(vid3, NO_LOCATION);
        assertNotNull(sub1);
        assertNotNull(sub2);
        assertNotNull(sub3);
        assertEquals(3, sharedSlot.getNumberLeaves());
        assertEquals(1, assignment.getNumberOfSlots());
        // release from the leaves.
        sub2.releaseSlot();
        assertTrue(sharedSlot.isAlive());
        assertTrue(sub1.isAlive());
        assertTrue(sub2.isReleased());
        assertTrue(sub3.isAlive());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfSlots());
        assertEquals(2, sharedSlot.getNumberLeaves());
        sub1.releaseSlot();
        assertTrue(sharedSlot.isAlive());
        assertTrue(sub1.isReleased());
        assertTrue(sub2.isReleased());
        assertTrue(sub3.isAlive());
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfSlots());
        assertEquals(1, sharedSlot.getNumberLeaves());
        sub3.releaseSlot();
        assertTrue(sharedSlot.isReleased());
        assertTrue(sub1.isReleased());
        assertTrue(sub2.isReleased());
        assertTrue(sub3.isReleased());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(0, assignment.getNumberOfSlots());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, assignment.getNumberOfSlots());
        assertNull(sharedSlot.allocateSharedSlot(new AbstractID()));
        assertNull(sharedSlot.allocateSubSlot(new AbstractID()));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 18 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class SharedSlotsTest method allocateSimpleSlotsAndReleaseFromRoot.

@Test
public void allocateSimpleSlotsAndReleaseFromRoot() {
    try {
        JobID jobId = new JobID();
        JobVertexID vid1 = new JobVertexID();
        JobVertexID vid2 = new JobVertexID();
        JobVertexID vid3 = new JobVertexID();
        JobVertexID vid4 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vid1, vid2, vid3, vid4);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        // allocate a shared slot
        SharedSlot sharedSlot = instance.allocateSharedSlot(jobId, assignment);
        // allocate a series of sub slots
        SimpleSlot sub1 = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.LOCAL, vid1);
        assertNotNull(sub1);
        assertNull(sub1.getExecutedVertex());
        assertEquals(Locality.LOCAL, sub1.getLocality());
        assertEquals(1, sub1.getNumberLeaves());
        assertEquals(vid1, sub1.getGroupID());
        assertEquals(instance.getTaskManagerID(), sub1.getTaskManagerID());
        assertEquals(jobId, sub1.getJobID());
        assertEquals(sharedSlot, sub1.getParent());
        assertEquals(sharedSlot, sub1.getRoot());
        assertEquals(0, sub1.getRootSlotNumber());
        assertEquals(0, sub1.getSlotNumber());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid4));
        SimpleSlot sub2 = assignment.getSlotForTask(vid2, NO_LOCATION);
        assertNotNull(sub2);
        assertNull(sub2.getExecutedVertex());
        assertEquals(Locality.UNCONSTRAINED, sub2.getLocality());
        assertEquals(1, sub2.getNumberLeaves());
        assertEquals(vid2, sub2.getGroupID());
        assertEquals(instance.getTaskManagerID(), sub2.getTaskManagerID());
        assertEquals(jobId, sub2.getJobID());
        assertEquals(sharedSlot, sub2.getParent());
        assertEquals(sharedSlot, sub2.getRoot());
        assertEquals(0, sub2.getRootSlotNumber());
        assertEquals(1, sub2.getSlotNumber());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid4));
        SimpleSlot sub3 = assignment.getSlotForTask(vid3, Collections.singleton(instance.getTaskManagerLocation()));
        assertNotNull(sub3);
        assertNull(sub3.getExecutedVertex());
        assertEquals(Locality.LOCAL, sub3.getLocality());
        assertEquals(1, sub3.getNumberLeaves());
        assertEquals(vid3, sub3.getGroupID());
        assertEquals(instance.getTaskManagerID(), sub3.getTaskManagerID());
        assertEquals(jobId, sub3.getJobID());
        assertEquals(sharedSlot, sub3.getParent());
        assertEquals(sharedSlot, sub3.getRoot());
        assertEquals(0, sub3.getRootSlotNumber());
        assertEquals(2, sub3.getSlotNumber());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid4));
        SimpleSlot sub4 = assignment.getSlotForTask(vid4, Collections.singleton(SchedulerTestUtils.getRandomInstance(1).getTaskManagerLocation()));
        assertNotNull(sub4);
        assertNull(sub4.getExecutedVertex());
        assertEquals(Locality.NON_LOCAL, sub4.getLocality());
        assertEquals(1, sub4.getNumberLeaves());
        assertEquals(vid4, sub4.getGroupID());
        assertEquals(instance.getTaskManagerID(), sub4.getTaskManagerID());
        assertEquals(jobId, sub4.getJobID());
        assertEquals(sharedSlot, sub4.getParent());
        assertEquals(sharedSlot, sub4.getRoot());
        assertEquals(0, sub4.getRootSlotNumber());
        assertEquals(3, sub4.getSlotNumber());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid4));
        // release from the root.
        sharedSlot.releaseSlot();
        assertTrue(sharedSlot.isReleased());
        assertTrue(sub1.isReleased());
        assertTrue(sub2.isReleased());
        assertTrue(sub3.isReleased());
        assertTrue(sub4.isReleased());
        assertEquals(0, sharedSlot.getNumberLeaves());
        assertFalse(sharedSlot.hasChildren());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, assignment.getNumberOfSlots());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid4));
        assertNull(sharedSlot.allocateSharedSlot(new AbstractID()));
        assertNull(sharedSlot.allocateSubSlot(new AbstractID()));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 19 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class OperatorGroupTest method testIOMetricGroupInstantiation.

@Test
public void testIOMetricGroupInstantiation() {
    MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());
    TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id");
    TaskManagerJobMetricGroup jmGroup = new TaskManagerJobMetricGroup(registry, tmGroup, new JobID(), "myJobName");
    TaskMetricGroup taskGroup = new TaskMetricGroup(registry, jmGroup, new AbstractID(), new AbstractID(), "aTaskName", 11, 0);
    OperatorMetricGroup opGroup = new OperatorMetricGroup(registry, taskGroup, "myOpName");
    assertNotNull(opGroup.getIOMetricGroup());
    assertNotNull(opGroup.getIOMetricGroup().getNumRecordsInCounter());
    assertNotNull(opGroup.getIOMetricGroup().getNumRecordsOutCounter());
    registry.shutdown();
}

Example 20 with AbstractID

use of org.apache.flink.util.AbstractID in project flink by apache.

the class TaskManagerGroupTest method addAndRemoveJobs.

// ------------------------------------------------------------------------
//  adding and removing jobs
// ------------------------------------------------------------------------
@Test
public void addAndRemoveJobs() throws IOException {
    MetricRegistry registry = new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());
    final TaskManagerMetricGroup group = new TaskManagerMetricGroup(registry, "localhost", new AbstractID().toString());
    final JobID jid1 = new JobID();
    final JobID jid2 = new JobID();
    final String jobName1 = "testjob";
    final String jobName2 = "anotherJob";
    final JobVertexID vertex11 = new JobVertexID();
    final JobVertexID vertex12 = new JobVertexID();
    final JobVertexID vertex13 = new JobVertexID();
    final JobVertexID vertex21 = new JobVertexID();
    final ExecutionAttemptID execution11 = new ExecutionAttemptID();
    final ExecutionAttemptID execution12 = new ExecutionAttemptID();
    final ExecutionAttemptID execution13 = new ExecutionAttemptID();
    final ExecutionAttemptID execution21 = new ExecutionAttemptID();
    TaskMetricGroup tmGroup11 = group.addTaskForJob(jid1, jobName1, vertex11, execution11, "test", 17, 0);
    TaskMetricGroup tmGroup12 = group.addTaskForJob(jid1, jobName1, vertex12, execution12, "test", 13, 1);
    TaskMetricGroup tmGroup21 = group.addTaskForJob(jid2, jobName2, vertex21, execution21, "test", 7, 2);
    assertEquals(2, group.numRegisteredJobMetricGroups());
    assertFalse(tmGroup11.parent().isClosed());
    assertFalse(tmGroup12.parent().isClosed());
    assertFalse(tmGroup21.parent().isClosed());
    // close all for job 2 and one from job 1
    tmGroup11.close();
    tmGroup21.close();
    assertTrue(tmGroup11.isClosed());
    assertTrue(tmGroup21.isClosed());
    // job 2 should be removed, job should still be there
    assertFalse(tmGroup11.parent().isClosed());
    assertFalse(tmGroup12.parent().isClosed());
    assertTrue(tmGroup21.parent().isClosed());
    assertEquals(1, group.numRegisteredJobMetricGroups());
    // add one more to job one
    TaskMetricGroup tmGroup13 = group.addTaskForJob(jid1, jobName1, vertex13, execution13, "test", 0, 0);
    tmGroup12.close();
    tmGroup13.close();
    assertTrue(tmGroup11.parent().isClosed());
    assertTrue(tmGroup12.parent().isClosed());
    assertTrue(tmGroup13.parent().isClosed());
    assertEquals(0, group.numRegisteredJobMetricGroups());
    registry.shutdown();
}