Examples with TaskModel org.apache.samza.job.model.TaskModel used on opensource projects

Example 11 with TaskModel

use of org.apache.samza.job.model.TaskModel in project samza by apache.

the class TestGroupByContainerCount method testBalancerAfterContainerSameCustomAssignmentAndContainerIncrease.

/**
   * Verifies the ability to have a custom task-container mapping that is *deliberately* unbalanced.
   *
   * Before:
   *  C0  C1
   * --------
   *  T0  T1
   *      T2
   *      T3
   *      T4
   *      T5
   *
   *  After:
   *  C0  C1  C2
   * ------------
   *  T0  T1  T4
   *  T5  T2  T3
   *
   *  The key here is that C0, which is not one of the new containers was under-allocated.
   *  This is an important case because this scenario, while impossible with GroupByContainerCount.group()
   *  could occur when the grouper class is switched or if there is a custom mapping.
   */
@Test
public void testBalancerAfterContainerSameCustomAssignmentAndContainerIncrease() {
    Set<TaskModel> taskModels = generateTaskModels(6);
    Map<String, String> prevTaskToContainerMapping = new HashMap<>();
    prevTaskToContainerMapping.put(getTaskName(0).getTaskName(), "0");
    prevTaskToContainerMapping.put(getTaskName(1).getTaskName(), "1");
    prevTaskToContainerMapping.put(getTaskName(2).getTaskName(), "1");
    prevTaskToContainerMapping.put(getTaskName(3).getTaskName(), "1");
    prevTaskToContainerMapping.put(getTaskName(4).getTaskName(), "1");
    prevTaskToContainerMapping.put(getTaskName(5).getTaskName(), "1");
    when(taskAssignmentManager.readTaskAssignment()).thenReturn(prevTaskToContainerMapping);
    Set<ContainerModel> containers = new GroupByContainerCount(3).balance(taskModels, localityManager);
    Map<String, ContainerModel> containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getProcessorId(), container);
    }
    assertEquals(3, containers.size());
    ContainerModel container0 = containersMap.get("0");
    ContainerModel container1 = containersMap.get("1");
    ContainerModel container2 = containersMap.get("2");
    assertNotNull(container0);
    assertNotNull(container1);
    assertNotNull(container2);
    assertEquals("0", container0.getProcessorId());
    assertEquals("1", container1.getProcessorId());
    assertEquals("2", container2.getProcessorId());
    assertEquals(2, container0.getTasks().size());
    assertEquals(2, container1.getTasks().size());
    assertEquals(2, container1.getTasks().size());
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(5)));
    assertTrue(container1.getTasks().containsKey(getTaskName(1)));
    assertTrue(container1.getTasks().containsKey(getTaskName(2)));
    assertTrue(container2.getTasks().containsKey(getTaskName(4)));
    assertTrue(container2.getTasks().containsKey(getTaskName(3)));
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(0).getTaskName(), "0");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(1).getTaskName(), "1");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(2).getTaskName(), "1");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(3).getTaskName(), "2");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(4).getTaskName(), "2");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(5).getTaskName(), "0");
    verify(taskAssignmentManager, never()).deleteTaskContainerMappings(anyCollection());
}

Example 12 with TaskModel

use of org.apache.samza.job.model.TaskModel in project samza by apache.

the class TestGroupByContainerCount method testBalancerMultipleReblances.

/**
   * Before:
   *  C0  C1  C2  C3
   * ----------------
   *  T0  T1  T2  T3
   *  T4  T5  T6  T7
   *  T8
   *
   * Intermediate:
   *  C0  C1
   * --------
   *  T0  T1
   *  T4  T5
   *  T8  T7
   *  T6  T3
   *  T2
   *
   *  After:
   *  C0  C1  C2
   * ------------
   *  T0  T1  T6
   *  T4  T5  T2
   *  T8  T7  T3
   */
@Test
public void testBalancerMultipleReblances() {
    // Before
    Set<TaskModel> taskModels = generateTaskModels(9);
    Set<ContainerModel> prevContainers = new GroupByContainerCount(4).group(taskModels);
    Map<String, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
    when(taskAssignmentManager.readTaskAssignment()).thenReturn(prevTaskToContainerMapping);
    // First balance
    Set<ContainerModel> containers = new GroupByContainerCount(2).balance(taskModels, localityManager);
    Map<String, ContainerModel> containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getProcessorId(), container);
    }
    assertEquals(2, containers.size());
    ContainerModel container0 = containersMap.get("0");
    ContainerModel container1 = containersMap.get("1");
    assertNotNull(container0);
    assertNotNull(container1);
    assertEquals("0", container0.getProcessorId());
    assertEquals("1", container1.getProcessorId());
    assertEquals(5, container0.getTasks().size());
    assertEquals(4, container1.getTasks().size());
    // Tasks 0,4,8 and 1,5 should stay on the same original containers
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(4)));
    assertTrue(container0.getTasks().containsKey(getTaskName(8)));
    assertTrue(container1.getTasks().containsKey(getTaskName(1)));
    assertTrue(container1.getTasks().containsKey(getTaskName(5)));
    // Tasks 2,6 and 3,7 should be reassigned to the new containers.
    // Consistency is the goal with these reassignments
    assertTrue(container0.getTasks().containsKey(getTaskName(6)));
    assertTrue(container0.getTasks().containsKey(getTaskName(2)));
    assertTrue(container1.getTasks().containsKey(getTaskName(7)));
    assertTrue(container1.getTasks().containsKey(getTaskName(3)));
    // Verify task mappings are saved
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(0).getTaskName(), "0");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(4).getTaskName(), "0");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(8).getTaskName(), "0");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(6).getTaskName(), "0");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(2).getTaskName(), "0");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(1).getTaskName(), "1");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(5).getTaskName(), "1");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(7).getTaskName(), "1");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(3).getTaskName(), "1");
    verify(taskAssignmentManager, never()).deleteTaskContainerMappings(anyCollection());
    // Second balance
    prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
    TaskAssignmentManager taskAssignmentManager2 = mock(TaskAssignmentManager.class);
    when(taskAssignmentManager2.readTaskAssignment()).thenReturn(prevTaskToContainerMapping);
    LocalityManager localityManager2 = mock(LocalityManager.class);
    when(localityManager2.getTaskAssignmentManager()).thenReturn(taskAssignmentManager2);
    containers = new GroupByContainerCount(3).balance(taskModels, localityManager2);
    containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getProcessorId(), container);
    }
    assertEquals(3, containers.size());
    container0 = containersMap.get("0");
    container1 = containersMap.get("1");
    ContainerModel container2 = containersMap.get("2");
    assertNotNull(container0);
    assertNotNull(container1);
    assertNotNull(container2);
    assertEquals("0", container0.getProcessorId());
    assertEquals("1", container1.getProcessorId());
    assertEquals("2", container2.getProcessorId());
    assertEquals(3, container0.getTasks().size());
    assertEquals(3, container1.getTasks().size());
    assertEquals(3, container2.getTasks().size());
    // Tasks 0,4,8 and 1,5,7 should stay on the same original containers
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(4)));
    assertTrue(container0.getTasks().containsKey(getTaskName(8)));
    assertTrue(container1.getTasks().containsKey(getTaskName(1)));
    assertTrue(container1.getTasks().containsKey(getTaskName(5)));
    assertTrue(container1.getTasks().containsKey(getTaskName(7)));
    // Tasks 2,6 and 3 should be reassigned to the new container.
    // Consistency is the goal with these reassignments
    assertTrue(container2.getTasks().containsKey(getTaskName(6)));
    assertTrue(container2.getTasks().containsKey(getTaskName(2)));
    assertTrue(container2.getTasks().containsKey(getTaskName(3)));
    // Verify task mappings are saved
    verify(taskAssignmentManager2).writeTaskContainerMapping(getTaskName(0).getTaskName(), "0");
    verify(taskAssignmentManager2).writeTaskContainerMapping(getTaskName(4).getTaskName(), "0");
    verify(taskAssignmentManager2).writeTaskContainerMapping(getTaskName(8).getTaskName(), "0");
    verify(taskAssignmentManager2).writeTaskContainerMapping(getTaskName(1).getTaskName(), "1");
    verify(taskAssignmentManager2).writeTaskContainerMapping(getTaskName(5).getTaskName(), "1");
    verify(taskAssignmentManager2).writeTaskContainerMapping(getTaskName(7).getTaskName(), "1");
    verify(taskAssignmentManager2).writeTaskContainerMapping(getTaskName(6).getTaskName(), "2");
    verify(taskAssignmentManager2).writeTaskContainerMapping(getTaskName(2).getTaskName(), "2");
    verify(taskAssignmentManager2).writeTaskContainerMapping(getTaskName(3).getTaskName(), "2");
    verify(taskAssignmentManager2, never()).deleteTaskContainerMappings(anyCollection());
}

Example 13 with TaskModel

use of org.apache.samza.job.model.TaskModel in project samza by apache.

the class TestHostAwareContainerAllocator method getJobModelManager.

private static JobModelManager getJobModelManager(int containerCount) {
    //Ideally, the JobModelReader should be constructed independent of HttpServer.
    //That way it becomes easier to mock objects. Save it for later.
    HttpServer server = new MockHttpServer("/", 7777, null, new ServletHolder(DefaultServlet.class));
    Map<String, ContainerModel> containers = new java.util.HashMap<>();
    for (int i = 0; i < containerCount; i++) {
        ContainerModel container = new ContainerModel(String.valueOf(i), i, new HashMap<TaskName, TaskModel>());
        containers.put(String.valueOf(i), container);
    }
    JobModel jobModel = new JobModel(getConfig(), containers);
    return new JobModelManager(jobModel, server, null);
}

Example 14 with TaskModel

use of org.apache.samza.job.model.TaskModel in project samza by apache.

the class TestGroupByContainerCount method testBalancerAfterContainerSameCustomAssignment.

/**
   * Verifies the ability to have a custom task-container mapping that is *deliberately* unbalanced.
   *
   * Before:
   *  C0  C1
   * --------
   *  T0  T6
   *  T1  T7
   *  T2  T8
   *  T3
   *  T4
   *  T5
   *
   *  After:
   *  C0  C1
   * --------
   *  T0  T6
   *  T1  T7
   *  T2  T8
   *  T3
   *  T4
   *  T5
   */
@Test
public void testBalancerAfterContainerSameCustomAssignment() {
    Set<TaskModel> taskModels = generateTaskModels(9);
    Map<String, String> prevTaskToContainerMapping = new HashMap<>();
    prevTaskToContainerMapping.put(getTaskName(0).getTaskName(), "0");
    prevTaskToContainerMapping.put(getTaskName(1).getTaskName(), "0");
    prevTaskToContainerMapping.put(getTaskName(2).getTaskName(), "0");
    prevTaskToContainerMapping.put(getTaskName(3).getTaskName(), "0");
    prevTaskToContainerMapping.put(getTaskName(4).getTaskName(), "0");
    prevTaskToContainerMapping.put(getTaskName(5).getTaskName(), "0");
    prevTaskToContainerMapping.put(getTaskName(6).getTaskName(), "1");
    prevTaskToContainerMapping.put(getTaskName(7).getTaskName(), "1");
    prevTaskToContainerMapping.put(getTaskName(8).getTaskName(), "1");
    when(taskAssignmentManager.readTaskAssignment()).thenReturn(prevTaskToContainerMapping);
    Set<ContainerModel> containers = new GroupByContainerCount(2).balance(taskModels, localityManager);
    Map<String, ContainerModel> containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getProcessorId(), container);
    }
    assertEquals(2, containers.size());
    ContainerModel container0 = containersMap.get("0");
    ContainerModel container1 = containersMap.get("1");
    assertNotNull(container0);
    assertNotNull(container1);
    assertEquals("0", container0.getProcessorId());
    assertEquals("1", container1.getProcessorId());
    assertEquals(6, container0.getTasks().size());
    assertEquals(3, container1.getTasks().size());
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(1)));
    assertTrue(container0.getTasks().containsKey(getTaskName(2)));
    assertTrue(container0.getTasks().containsKey(getTaskName(3)));
    assertTrue(container0.getTasks().containsKey(getTaskName(4)));
    assertTrue(container0.getTasks().containsKey(getTaskName(5)));
    assertTrue(container1.getTasks().containsKey(getTaskName(6)));
    assertTrue(container1.getTasks().containsKey(getTaskName(7)));
    assertTrue(container1.getTasks().containsKey(getTaskName(8)));
    verify(taskAssignmentManager, never()).writeTaskContainerMapping(anyString(), anyString());
    verify(taskAssignmentManager, never()).deleteTaskContainerMappings(anyCollection());
}

Example 15 with TaskModel

use of org.apache.samza.job.model.TaskModel in project samza by apache.

the class TestGroupByContainerCount method testBalancerAfterContainerSame.

/**
   * Before:
   *  C0  C1
   * --------
   *  T0  T1
   *  T2  T3
   *  T4  T5
   *  T6  T7
   *  T8
   *
   *  After:
   *  C0  C1
   * --------
   *  T0  T1
   *  T2  T3
   *  T4  T5
   *  T6  T7
   *  T8
   */
@Test
public void testBalancerAfterContainerSame() {
    Set<TaskModel> taskModels = generateTaskModels(9);
    Set<ContainerModel> prevContainers = new GroupByContainerCount(2).group(taskModels);
    Map<String, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
    when(taskAssignmentManager.readTaskAssignment()).thenReturn(prevTaskToContainerMapping);
    Set<ContainerModel> containers = new GroupByContainerCount(2).balance(taskModels, localityManager);
    Map<String, ContainerModel> containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getProcessorId(), container);
    }
    assertEquals(2, containers.size());
    ContainerModel container0 = containersMap.get("0");
    ContainerModel container1 = containersMap.get("1");
    assertNotNull(container0);
    assertNotNull(container1);
    assertEquals("0", container0.getProcessorId());
    assertEquals("1", container1.getProcessorId());
    assertEquals(5, container0.getTasks().size());
    assertEquals(4, container1.getTasks().size());
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(2)));
    assertTrue(container0.getTasks().containsKey(getTaskName(4)));
    assertTrue(container0.getTasks().containsKey(getTaskName(6)));
    assertTrue(container0.getTasks().containsKey(getTaskName(8)));
    assertTrue(container1.getTasks().containsKey(getTaskName(1)));
    assertTrue(container1.getTasks().containsKey(getTaskName(3)));
    assertTrue(container1.getTasks().containsKey(getTaskName(5)));
    assertTrue(container1.getTasks().containsKey(getTaskName(7)));
    verify(taskAssignmentManager, never()).writeTaskContainerMapping(anyString(), anyString());
    verify(taskAssignmentManager, never()).deleteTaskContainerMappings(anyCollection());
}