Examples with ContainerModel org.apache.samza.job.model.ContainerModel used on opensource projects

Example 16 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestGroupByContainerCount method testGroupHappyPath.

@Test
public void testGroupHappyPath() {
    Set<TaskModel> taskModels = generateTaskModels(5);
    Set<ContainerModel> containers = new GroupByContainerCount(2).group(taskModels);
    Map<String, ContainerModel> containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getId(), container);
    }
    assertEquals(2, containers.size());
    ContainerModel container0 = containersMap.get("0");
    ContainerModel container1 = containersMap.get("1");
    assertNotNull(container0);
    assertNotNull(container1);
    assertEquals("0", container0.getId());
    assertEquals("1", container1.getId());
    assertEquals(3, container0.getTasks().size());
    assertEquals(2, container1.getTasks().size());
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(2)));
    assertTrue(container0.getTasks().containsKey(getTaskName(4)));
    assertTrue(container1.getTasks().containsKey(getTaskName(1)));
    assertTrue(container1.getTasks().containsKey(getTaskName(3)));
}

Example 17 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestGroupByContainerCount method testBalancerMultipleReblances.

/**
 * Before:
 *  C0  C1  C2  C3
 * ----------------
 *  T0  T1  T2  T3
 *  T4  T5  T6  T7
 *  T8
 *
 * Intermediate:
 *  C0  C1
 * --------
 *  T0  T1
 *  T4  T5
 *  T8  T7
 *  T6  T3
 *  T2
 *
 *  After:
 *  C0  C1  C2
 * ------------
 *  T0  T1  T6
 *  T4  T5  T2
 *  T8  T7  T3
 */
@Test
public void testBalancerMultipleReblances() {
    // Before
    Set<TaskModel> taskModels = generateTaskModels(9);
    Set<ContainerModel> prevContainers = new GroupByContainerCount(4).group(taskModels);
    Map<TaskName, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
    GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(new HashMap<>(), new HashMap<>(), new HashMap<>(), prevTaskToContainerMapping);
    // First balance
    Set<ContainerModel> containers = new GroupByContainerCount(2).group(taskModels, grouperMetadata);
    Map<String, ContainerModel> containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getId(), container);
    }
    assertEquals(2, containers.size());
    ContainerModel container0 = containersMap.get("0");
    ContainerModel container1 = containersMap.get("1");
    assertNotNull(container0);
    assertNotNull(container1);
    assertEquals("0", container0.getId());
    assertEquals("1", container1.getId());
    assertEquals(5, container0.getTasks().size());
    assertEquals(4, container1.getTasks().size());
    // Tasks 0,4,8 and 1,5 should stay on the same original containers
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(4)));
    assertTrue(container0.getTasks().containsKey(getTaskName(8)));
    assertTrue(container1.getTasks().containsKey(getTaskName(1)));
    assertTrue(container1.getTasks().containsKey(getTaskName(5)));
    // Tasks 2,6 and 3,7 should be reassigned to the new containers.
    // Consistency is the goal with these reassignments
    assertTrue(container0.getTasks().containsKey(getTaskName(6)));
    assertTrue(container0.getTasks().containsKey(getTaskName(2)));
    assertTrue(container1.getTasks().containsKey(getTaskName(7)));
    assertTrue(container1.getTasks().containsKey(getTaskName(3)));
    // Second balance
    prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
    GrouperMetadataImpl grouperMetadata1 = new GrouperMetadataImpl(new HashMap<>(), new HashMap<>(), new HashMap<>(), prevTaskToContainerMapping);
    containers = new GroupByContainerCount(3).group(taskModels, grouperMetadata1);
    containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getId(), container);
    }
    assertEquals(3, containers.size());
    container0 = containersMap.get("0");
    container1 = containersMap.get("1");
    ContainerModel container2 = containersMap.get("2");
    assertNotNull(container0);
    assertNotNull(container1);
    assertNotNull(container2);
    assertEquals("0", container0.getId());
    assertEquals("1", container1.getId());
    assertEquals("2", container2.getId());
    assertEquals(3, container0.getTasks().size());
    assertEquals(3, container1.getTasks().size());
    assertEquals(3, container2.getTasks().size());
    // Tasks 0,4,8 and 1,5,7 should stay on the same original containers
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(4)));
    assertTrue(container0.getTasks().containsKey(getTaskName(8)));
    assertTrue(container1.getTasks().containsKey(getTaskName(1)));
    assertTrue(container1.getTasks().containsKey(getTaskName(5)));
    assertTrue(container1.getTasks().containsKey(getTaskName(7)));
    // Tasks 2,6 and 3 should be reassigned to the new container.
    // Consistency is the goal with these reassignments
    assertTrue(container2.getTasks().containsKey(getTaskName(6)));
    assertTrue(container2.getTasks().containsKey(getTaskName(2)));
    assertTrue(container2.getTasks().containsKey(getTaskName(3)));
}

Example 18 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestGroupByContainerCount method testBalancerAfterContainerSameCustomAssignmentAndContainerIncrease.

/**
 * Verifies the ability to have a custom task-container mapping that is *deliberately* unbalanced.
 *
 * Before:
 *  C0  C1
 * --------
 *  T0  T1
 *      T2
 *      T3
 *      T4
 *      T5
 *
 *  After:
 *  C0  C1  C2
 * ------------
 *  T0  T1  T4
 *  T5  T2  T3
 *
 *  The key here is that C0, which is not one of the new containers was under-allocated.
 *  This is an important case because this scenario, while impossible with GroupByContainerCount.group()
 *  could occur when the grouper class is switched or if there is a custom mapping.
 */
@Test
public void testBalancerAfterContainerSameCustomAssignmentAndContainerIncrease() {
    Set<TaskModel> taskModels = generateTaskModels(6);
    Map<TaskName, String> prevTaskToContainerMapping = new HashMap<>();
    prevTaskToContainerMapping.put(getTaskName(0), "0");
    prevTaskToContainerMapping.put(getTaskName(1), "1");
    prevTaskToContainerMapping.put(getTaskName(2), "1");
    prevTaskToContainerMapping.put(getTaskName(3), "1");
    prevTaskToContainerMapping.put(getTaskName(4), "1");
    prevTaskToContainerMapping.put(getTaskName(5), "1");
    GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(new HashMap<>(), new HashMap<>(), new HashMap<>(), prevTaskToContainerMapping);
    Set<ContainerModel> containers = new GroupByContainerCount(3).group(taskModels, grouperMetadata);
    Map<String, ContainerModel> containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getId(), container);
    }
    assertEquals(3, containers.size());
    ContainerModel container0 = containersMap.get("0");
    ContainerModel container1 = containersMap.get("1");
    ContainerModel container2 = containersMap.get("2");
    assertNotNull(container0);
    assertNotNull(container1);
    assertNotNull(container2);
    assertEquals("0", container0.getId());
    assertEquals("1", container1.getId());
    assertEquals("2", container2.getId());
    assertEquals(2, container0.getTasks().size());
    assertEquals(2, container1.getTasks().size());
    assertEquals(2, container1.getTasks().size());
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(5)));
    assertTrue(container1.getTasks().containsKey(getTaskName(1)));
    assertTrue(container1.getTasks().containsKey(getTaskName(2)));
    assertTrue(container2.getTasks().containsKey(getTaskName(4)));
    assertTrue(container2.getTasks().containsKey(getTaskName(3)));
}

Example 19 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestTaskNameGrouperProxy method getContainerMap.

// Create a container map with two tasks in Container 0 and Container 1, and one in Container 2.
private static Set<ContainerModel> getContainerMap() {
    Set<ContainerModel> retVal = new HashSet<>();
    Map<TaskName, TaskModel> tasksForContainer0 = new HashMap<>();
    tasksForContainer0.put(new TaskName("Partition 0"), getTaskModel(0));
    tasksForContainer0.put(new TaskName("Partition 1"), getTaskModel(1));
    retVal.add(new ContainerModel("0", tasksForContainer0));
    Map<TaskName, TaskModel> tasksForContainer1 = new HashMap<>();
    tasksForContainer1.put(new TaskName("Partition 2"), getTaskModel(2));
    tasksForContainer1.put(new TaskName("Partition 3"), getTaskModel(3));
    retVal.add(new ContainerModel("1", tasksForContainer1));
    retVal.add(new ContainerModel("2", Collections.singletonMap(new TaskName("Partition 4"), getTaskModel(4))));
    return retVal;
}

Example 20 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestKafkaChangelogStateBackendFactory method testGetChangelogSSPsForContainerNoChangelogs.

@Test
public void testGetChangelogSSPsForContainerNoChangelogs() {
    KafkaChangelogStateBackendFactory factory = new KafkaChangelogStateBackendFactory();
    TaskName taskName0 = new TaskName("task0");
    TaskName taskName1 = new TaskName("task1");
    TaskModel taskModel0 = new TaskModel(taskName0, ImmutableSet.of(new SystemStreamPartition("input", "stream", new Partition(0))), new Partition(10));
    TaskModel taskModel1 = new TaskModel(taskName1, ImmutableSet.of(new SystemStreamPartition("input", "stream", new Partition(1))), new Partition(11));
    ContainerModel containerModel = new ContainerModel("processorId", ImmutableMap.of(taskName0, taskModel0, taskName1, taskModel1));
    Assert.assertEquals(Collections.emptySet(), factory.getChangelogSSPForContainer(Collections.emptyMap(), new ContainerContextImpl(containerModel, null)));
}