Examples with ContainerModel org.apache.samza.job.model.ContainerModel used on opensource projects

Example 26 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestZkUtils method testPublishNewJobModel.

@Test
public void testPublishNewJobModel() {
    ZkKeyBuilder keyBuilder = new ZkKeyBuilder("test");
    String root = keyBuilder.getRootPath();
    zkClient.deleteRecursive(root);
    String version = "1";
    String oldVersion = "0";
    zkUtils.validatePaths(new String[] { root, keyBuilder.getJobModelPathPrefix(), keyBuilder.getJobModelVersionPath() });
    zkUtils.publishJobModelVersion(oldVersion, version);
    Assert.assertEquals(version, zkUtils.getJobModelVersion());
    String newerVersion = Long.toString(Long.valueOf(version) + 1);
    zkUtils.publishJobModelVersion(version, newerVersion);
    Assert.assertEquals(newerVersion, zkUtils.getJobModelVersion());
    try {
        // invalid new version
        zkUtils.publishJobModelVersion(oldVersion, "10");
        Assert.fail("publish invalid version should've failed");
    } catch (SamzaException e) {
    // expected
    }
    // create job model
    Map<String, String> configMap = new HashMap<>();
    Map<String, ContainerModel> containers = new HashMap<>();
    MapConfig config = new MapConfig(configMap);
    JobModel jobModel = new JobModel(config, containers);
    zkUtils.publishJobModel(version, jobModel);
    Assert.assertEquals(jobModel, zkUtils.getJobModel(version));
}

Example 27 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class KafkaChangelogStateBackendFactory method filterStandbySystemStreams.

@VisibleForTesting
Map<String, SystemStream> filterStandbySystemStreams(Map<String, SystemStream> changelogSystemStreams, ContainerModel containerModel) {
    Map<SystemStreamPartition, String> changelogSSPToStore = new HashMap<>();
    changelogSystemStreams.forEach((storeName, systemStream) -> containerModel.getTasks().forEach((taskName, taskModel) -> changelogSSPToStore.put(new SystemStreamPartition(systemStream, taskModel.getChangelogPartition()), storeName)));
    Set<TaskModel> standbyTaskModels = containerModel.getTasks().values().stream().filter(taskModel -> taskModel.getTaskMode().equals(TaskMode.Standby)).collect(Collectors.toSet());
    // remove all standby task changelog ssps
    standbyTaskModels.forEach((taskModel) -> {
        changelogSystemStreams.forEach((storeName, systemStream) -> {
            SystemStreamPartition ssp = new SystemStreamPartition(systemStream, taskModel.getChangelogPartition());
            changelogSSPToStore.remove(ssp);
        });
    });
    // changelogSystemStreams correspond only to active tasks (since those of standby-tasks moved to sideInputs above)
    return MapUtils.invertMap(changelogSSPToStore).entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, x -> x.getValue().getSystemStream()));
}

Example 28 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestRemoteTableDescriptor method createMockContext.

private Context createMockContext(TableDescriptor tableDescriptor) {
    Context context = mock(Context.class);
    ContainerContext containerContext = mock(ContainerContext.class);
    when(context.getContainerContext()).thenReturn(containerContext);
    MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
    when(metricsRegistry.newTimer(anyString(), anyString())).thenReturn(mock(Timer.class));
    when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(mock(Counter.class));
    when(containerContext.getContainerMetricsRegistry()).thenReturn(metricsRegistry);
    TaskContextImpl taskContext = mock(TaskContextImpl.class);
    when(context.getTaskContext()).thenReturn(taskContext);
    TaskName taskName = new TaskName("MyTask");
    TaskModel taskModel = mock(TaskModel.class);
    when(taskModel.getTaskName()).thenReturn(taskName);
    when(context.getTaskContext().getTaskModel()).thenReturn(taskModel);
    ContainerModel containerModel = mock(ContainerModel.class);
    when(containerModel.getTasks()).thenReturn(ImmutableMap.of(taskName, taskModel));
    when(containerContext.getContainerModel()).thenReturn(containerModel);
    String containerId = "container-1";
    JobModel jobModel = mock(JobModel.class);
    when(taskContext.getJobModel()).thenReturn(jobModel);
    when(jobModel.getContainers()).thenReturn(ImmutableMap.of(containerId, containerModel));
    JobContext jobContext = mock(JobContext.class);
    Config jobConfig = new MapConfig(tableDescriptor.toConfig(new MapConfig()));
    when(jobContext.getConfig()).thenReturn(jobConfig);
    when(context.getJobContext()).thenReturn(jobContext);
    return context;
}

Example 29 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestDiagnosticsStreamMessage method getSampleContainerModels.

public static Map<String, ContainerModel> getSampleContainerModels() {
    Map<String, ContainerModel> containerModels = new HashMap<>();
    Map<TaskName, TaskModel> tasks = new HashMap<>();
    Set<SystemStreamPartition> sspsForTask1 = new HashSet<>();
    sspsForTask1.add(new SystemStreamPartition("kafka", "test-stream", new Partition(0)));
    tasks.put(new TaskName("Partition 0"), new TaskModel(new TaskName("Partition 0"), sspsForTask1, new Partition(0)));
    Set<SystemStreamPartition> sspsForTask2 = new HashSet<>();
    sspsForTask2.add(new SystemStreamPartition("kafka", "test-stream", new Partition(1)));
    tasks.put(new TaskName("Partition 1"), new TaskModel(new TaskName("Partition 1"), sspsForTask2, new Partition(1)));
    containerModels.put("0", new ContainerModel("0", tasks));
    return containerModels;
}

Example 30 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestJobModelCalculator method testPreviousChangelogPartitionsMaintained.

@Test
public void testPreviousChangelogPartitionsMaintained() {
    // existing changelog mapping has 2 tasks, but the job model ultimately will need 4 tasks
    // intentionally using an "out-of-order" changelog mapping to make sure it gets maintained
    Map<TaskName, Integer> changelogPartitionMapping = ImmutableMap.of(taskName(0), 1, taskName(1), 0);
    Config config = config(ImmutableList.of(SYSTEM_STREAM0, SYSTEM_STREAM1), ImmutableMap.of());
    // these task models have special changelog partitions from the previous mapping
    TaskModel taskModel0 = new TaskModel(taskName(0), ImmutableSet.of(new SystemStreamPartition(SYSTEM_STREAM0, new Partition(0)), new SystemStreamPartition(SYSTEM_STREAM1, new Partition(0))), new Partition(1));
    TaskModel taskModel1 = new TaskModel(taskName(1), ImmutableSet.of(new SystemStreamPartition(SYSTEM_STREAM0, new Partition(1)), new SystemStreamPartition(SYSTEM_STREAM1, new Partition(1))), new Partition(0));
    // tasks 2 and 3 will get assigned new changelog partitions
    Map<String, ContainerModel> containerModels = ImmutableMap.of("0", new ContainerModel("0", ImmutableMap.of(taskName(0), taskModel0, taskName(2), taskModel(2, 2, 2))), "1", new ContainerModel("1", ImmutableMap.of(taskName(1), taskModel1, taskName(3), taskModel(3, 3))));
    JobModel expected = new JobModel(config, containerModels);
    JobModel actual = JobModelCalculator.INSTANCE.calculateJobModel(config, changelogPartitionMapping, this.streamMetadataCache, this.grouperMetadata);
    assertEquals(expected, actual);
}