Examples with ContainerModel org.apache.samza.job.model.ContainerModel used on opensource projects

Example 86 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestGroupByContainerCount method testBalancerAfterContainerSame.

/**
 * Before:
 *  C0  C1
 * --------
 *  T0  T1
 *  T2  T3
 *  T4  T5
 *  T6  T7
 *  T8
 *
 *  After:
 *  C0  C1
 * --------
 *  T0  T1
 *  T2  T3
 *  T4  T5
 *  T6  T7
 *  T8
 */
@Test
public void testBalancerAfterContainerSame() {
    Set<TaskModel> taskModels = generateTaskModels(9);
    Set<ContainerModel> prevContainers = new GroupByContainerCount(2).group(taskModels);
    Map<TaskName, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
    GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(new HashMap<>(), new HashMap<>(), new HashMap<>(), prevTaskToContainerMapping);
    Set<ContainerModel> containers = new GroupByContainerCount(2).group(taskModels, grouperMetadata);
    Map<String, ContainerModel> containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getId(), container);
    }
    assertEquals(2, containers.size());
    ContainerModel container0 = containersMap.get("0");
    ContainerModel container1 = containersMap.get("1");
    assertNotNull(container0);
    assertNotNull(container1);
    assertEquals("0", container0.getId());
    assertEquals("1", container1.getId());
    assertEquals(5, container0.getTasks().size());
    assertEquals(4, container1.getTasks().size());
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(2)));
    assertTrue(container0.getTasks().containsKey(getTaskName(4)));
    assertTrue(container0.getTasks().containsKey(getTaskName(6)));
    assertTrue(container0.getTasks().containsKey(getTaskName(8)));
    assertTrue(container1.getTasks().containsKey(getTaskName(1)));
    assertTrue(container1.getTasks().containsKey(getTaskName(3)));
    assertTrue(container1.getTasks().containsKey(getTaskName(5)));
    assertTrue(container1.getTasks().containsKey(getTaskName(7)));
}

Example 87 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestGroupByContainerCount method testBalancerAfterContainerSameCustomAssignment.

/**
 * Verifies the ability to have a custom task-container mapping that is *deliberately* unbalanced.
 *
 * Before:
 *  C0  C1
 * --------
 *  T0  T6
 *  T1  T7
 *  T2  T8
 *  T3
 *  T4
 *  T5
 *
 *  After:
 *  C0  C1
 * --------
 *  T0  T6
 *  T1  T7
 *  T2  T8
 *  T3
 *  T4
 *  T5
 */
@Test
public void testBalancerAfterContainerSameCustomAssignment() {
    Set<TaskModel> taskModels = generateTaskModels(9);
    Map<TaskName, String> prevTaskToContainerMapping = new HashMap<>();
    prevTaskToContainerMapping.put(getTaskName(0), "0");
    prevTaskToContainerMapping.put(getTaskName(1), "0");
    prevTaskToContainerMapping.put(getTaskName(2), "0");
    prevTaskToContainerMapping.put(getTaskName(3), "0");
    prevTaskToContainerMapping.put(getTaskName(4), "0");
    prevTaskToContainerMapping.put(getTaskName(5), "0");
    prevTaskToContainerMapping.put(getTaskName(6), "1");
    prevTaskToContainerMapping.put(getTaskName(7), "1");
    prevTaskToContainerMapping.put(getTaskName(8), "1");
    GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(new HashMap<>(), new HashMap<>(), new HashMap<>(), prevTaskToContainerMapping);
    Set<ContainerModel> containers = new GroupByContainerCount(2).group(taskModels, grouperMetadata);
    Map<String, ContainerModel> containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getId(), container);
    }
    assertEquals(2, containers.size());
    ContainerModel container0 = containersMap.get("0");
    ContainerModel container1 = containersMap.get("1");
    assertNotNull(container0);
    assertNotNull(container1);
    assertEquals("0", container0.getId());
    assertEquals("1", container1.getId());
    assertEquals(6, container0.getTasks().size());
    assertEquals(3, container1.getTasks().size());
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(1)));
    assertTrue(container0.getTasks().containsKey(getTaskName(2)));
    assertTrue(container0.getTasks().containsKey(getTaskName(3)));
    assertTrue(container0.getTasks().containsKey(getTaskName(4)));
    assertTrue(container0.getTasks().containsKey(getTaskName(5)));
    assertTrue(container1.getTasks().containsKey(getTaskName(6)));
    assertTrue(container1.getTasks().containsKey(getTaskName(7)));
    assertTrue(container1.getTasks().containsKey(getTaskName(8)));
}

Example 88 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class JobModelManagerTestUtil method getJobModelManager.

public static JobModelManager getJobModelManager(Config config, int containerCount, HttpServer server) {
    Map<String, ContainerModel> containers = new java.util.HashMap<>();
    for (int i = 0; i < containerCount; i++) {
        ContainerModel container = new ContainerModel(String.valueOf(i), new HashMap<>());
        containers.put(String.valueOf(i), container);
    }
    JobModel jobModel = new JobModel(config, containers);
    return new JobModelManager(jobModel, server);
}

Example 89 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class ContainerMocks method generateTaskAssignments.

public static Map<String, String> generateTaskAssignments(int numContainers, int taskCount) {
    Map<String, String> mapping = new HashMap<>(taskCount);
    Set<ContainerModel> containers = generateContainerModels(numContainers, taskCount);
    for (ContainerModel container : containers) {
        for (TaskName taskName : container.getTasks().keySet()) {
            mapping.put(taskName.getTaskName(), container.getId());
        }
    }
    return mapping;
}

Example 90 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestJobModelHelper method testExistingStandbyContainers.

@Test
public void testExistingStandbyContainers() {
    Map<String, ContainerModel> oldContainerModels = ImmutableMap.of("0", new ContainerModel("0", ImmutableMap.of(taskName(0), taskModel(0, 0))), "1", new ContainerModel("1", ImmutableMap.of(taskName(1), taskModel(1, 1))), "0-0", new ContainerModel("0-0", ImmutableMap.of(standbyTaskName(0, 0), standbyTaskModel(0, 0))), "1-0", new ContainerModel("1-0", ImmutableMap.of(standbyTaskName(1, 0), standbyTaskModel(1, 0))));
    setupOldTaskAssignments(oldContainerModels);
    Map<String, ContainerModel> containerModels = new ImmutableMap.Builder<String, ContainerModel>().put("0", new ContainerModel("0", ImmutableMap.of(taskName(0), taskModel(0, 0)))).put("1", new ContainerModel("1", ImmutableMap.of(taskName(1), taskModel(1, 1)))).put("0-0", new ContainerModel("0-0", ImmutableMap.of(standbyTaskName(0, 0), standbyTaskModel(0, 0)))).put("1-0", new ContainerModel("1-0", ImmutableMap.of(standbyTaskName(1, 0), standbyTaskModel(1, 0)))).put("0-1", new ContainerModel("0-1", ImmutableMap.of(standbyTaskName(0, 1), standbyTaskModel(0, 1)))).put("1-1", new ContainerModel("1-1", ImmutableMap.of(standbyTaskName(1, 1), standbyTaskModel(1, 1)))).build();
    runAndCheckNewJobModel(config(), containerModels);
    verifyGrouperMetadata(anyHostProcessorIdToLocationId(), anyHostTaskNameToLocationId(2), activeTaskToSSPs(oldContainerModels), activeTaskToContainer(oldContainerModels));
    verifyNewTaskAssignments(taskNameStrings(oldContainerModels, TaskMode.Standby), null, containerToTaskToMode(containerModels), sspToTasks(containerModels));
}