Examples with ContainerModel org.apache.samza.job.model.ContainerModel used on opensource projects

Example 51 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestGroupByContainerCount method testGroupTaskCountIncrease.

@Test
public void testGroupTaskCountIncrease() {
    int taskCount = 3;
    Set<TaskModel> taskModels = generateTaskModels(taskCount);
    Set<ContainerModel> prevContainers = new GroupByContainerCount(2).group(generateTaskModels(taskCount - 1));
    Map<String, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
    when(taskAssignmentManager.readTaskAssignment()).thenReturn(prevTaskToContainerMapping);
    Set<ContainerModel> containers = new GroupByContainerCount(1).balance(taskModels, localityManager);
    // Results should be the same as calling group
    Map<String, ContainerModel> containersMap = new HashMap<>();
    for (ContainerModel container : containers) {
        containersMap.put(container.getProcessorId(), container);
    }
    assertEquals(1, containers.size());
    ContainerModel container0 = containersMap.get("0");
    assertNotNull(container0);
    assertEquals("0", container0.getProcessorId());
    assertEquals(3, container0.getTasks().size());
    assertTrue(container0.getTasks().containsKey(getTaskName(0)));
    assertTrue(container0.getTasks().containsKey(getTaskName(1)));
    assertTrue(container0.getTasks().containsKey(getTaskName(2)));
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(0).getTaskName(), "0");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(1).getTaskName(), "0");
    verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(2).getTaskName(), "0");
    verify(taskAssignmentManager).deleteTaskContainerMappings(anyCollection());
}

Example 52 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class StorageRecovery method getTaskStorageManagers.

/**
   * create one TaskStorageManager for each task. Add all of them to the
   * List<TaskStorageManager>
   */
@SuppressWarnings({ "unchecked", "rawtypes" })
private void getTaskStorageManagers() {
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(Util.javaMapAsScalaMap(systemAdmins), 5000, SystemClock.instance());
    for (ContainerModel containerModel : containers.values()) {
        HashMap<String, StorageEngine> taskStores = new HashMap<String, StorageEngine>();
        SamzaContainerContext containerContext = new SamzaContainerContext(containerModel.getProcessorId(), jobConfig, containerModel.getTasks().keySet());
        for (TaskModel taskModel : containerModel.getTasks().values()) {
            HashMap<String, SystemConsumer> storeConsumers = getStoreConsumers();
            for (Entry<String, StorageEngineFactory<?, ?>> entry : storageEngineFactories.entrySet()) {
                String storeName = entry.getKey();
                if (changeLogSystemStreams.containsKey(storeName)) {
                    SystemStreamPartition changeLogSystemStreamPartition = new SystemStreamPartition(changeLogSystemStreams.get(storeName), taskModel.getChangelogPartition());
                    File storePartitionDir = TaskStorageManager.getStorePartitionDir(storeBaseDir, storeName, taskModel.getTaskName());
                    log.info("Got storage engine directory: " + storePartitionDir);
                    StorageEngine storageEngine = (entry.getValue()).getStorageEngine(storeName, storePartitionDir, (Serde) new ByteSerde(), (Serde) new ByteSerde(), null, new MetricsRegistryMap(), changeLogSystemStreamPartition, containerContext);
                    taskStores.put(storeName, storageEngine);
                }
            }
            TaskStorageManager taskStorageManager = new TaskStorageManager(taskModel.getTaskName(), Util.javaMapAsScalaMap(taskStores), Util.javaMapAsScalaMap(storeConsumers), Util.javaMapAsScalaMap(changeLogSystemStreams), maxPartitionNumber, streamMetadataCache, storeBaseDir, storeBaseDir, taskModel.getChangelogPartition(), Util.javaMapAsScalaMap(systemAdmins), new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
            taskStorageManagers.add(taskStorageManager);
        }
    }
}

Example 53 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class GroupByContainerCount method buildContainerModels.

/**
   * Translates the list of TaskGroup instances to a set of ContainerModel instances, using the
   * set of TaskModel instances.
   *
   * @param tasks             the TaskModels to assign to the ContainerModels.
   * @param containerTasks    the TaskGroups defining how the tasks should be grouped.
   * @return                  a mutable set of ContainerModels.
   */
private Set<ContainerModel> buildContainerModels(Set<TaskModel> tasks, List<TaskGroup> containerTasks) {
    // Map task names to models
    Map<String, TaskModel> taskNameToModel = new HashMap<>();
    for (TaskModel model : tasks) {
        taskNameToModel.put(model.getTaskName().getTaskName(), model);
    }
    // Build container models
    Set<ContainerModel> containerModels = new HashSet<>();
    for (TaskGroup container : containerTasks) {
        Map<TaskName, TaskModel> containerTaskModels = new HashMap<>();
        for (String taskName : container.taskNames) {
            TaskModel model = taskNameToModel.get(taskName);
            containerTaskModels.put(model.getTaskName(), model);
        }
        containerModels.add(new ContainerModel(container.containerId, Integer.valueOf(container.containerId), containerTaskModels));
    }
    return Collections.unmodifiableSet(containerModels);
}

Example 54 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class MockTaskProxy method getJobModel.

@Override
protected JobModel getJobModel(JobInstance jobInstance) {
    if (jobInstance.getJobId().contains("Bad") || jobInstance.getJobName().contains("Bad")) {
        throw new IllegalArgumentException("No tasks found.");
    }
    TaskModel task1Model = new TaskModel(new TaskName(TASK_1_NAME), SYSTEM_STREAM_PARTITIONS, CHANGE_LOG_PARTITION);
    TaskModel task2Model = new TaskModel(new TaskName(TASK_2_NAME), SYSTEM_STREAM_PARTITIONS, CHANGE_LOG_PARTITION);
    ContainerModel task1ContainerModel = new ContainerModel(TASK_1_CONTAINER_ID, 1, ImmutableMap.of(new TaskName(TASK_1_NAME), task1Model));
    ContainerModel task2ContainerModel = new ContainerModel(TASK_2_CONTAINER_ID, 2, ImmutableMap.of(new TaskName(TASK_2_NAME), task2Model));
    return new JobModel(new MapConfig(), ImmutableMap.of(TASK_1_CONTAINER_ID, task1ContainerModel, TASK_2_CONTAINER_ID, task2ContainerModel));
}

Example 55 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class StorageRecovery method getChangeLogMaxPartitionNumber.

/**
 * get the max partition number of the changelog stream
 */
private void getChangeLogMaxPartitionNumber() {
    int maxPartitionId = 0;
    for (ContainerModel containerModel : containers.values()) {
        for (TaskModel taskModel : containerModel.getTasks().values()) {
            maxPartitionId = Math.max(maxPartitionId, taskModel.getChangelogPartition().getPartitionId());
        }
    }
    maxPartitionNumber = maxPartitionId + 1;
}