Examples with ContainerModel org.apache.samza.job.model.ContainerModel used on opensource projects

Example 31 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestJobModelCalculator method testSSPMatcherConfigJobFactoryRegexNotMatched.

@Test
public void testSSPMatcherConfigJobFactoryRegexNotMatched() {
    Map<TaskName, Integer> changelogPartitionMapping = changelogPartitionMapping(4);
    Config config = config(ImmutableList.of(SYSTEM_STREAM0, SYSTEM_STREAM1), ImmutableMap.of(JobConfig.SSP_MATCHER_CLASS, Partition0Or1Filter.class.getName(), JobConfig.SSP_MATCHER_CONFIG_JOB_FACTORY_REGEX, ".*MyJobFactory", // this needs to not match the regex in the line above
    JobConfig.STREAM_JOB_FACTORY_CLASS, "org.apache.samza.custom.OtherJobFactory"));
    Map<String, ContainerModel> containerModels = ImmutableMap.of("0", new ContainerModel("0", ImmutableMap.of(taskName(0), taskModel(0, 0, 0), taskName(2), taskModel(2, 2, 2))), "1", new ContainerModel("1", ImmutableMap.of(taskName(1), taskModel(1, 1, 1), taskName(3), taskModel(3, 3))));
    JobModel expected = new JobModel(config, containerModels);
    JobModel actual = JobModelCalculator.INSTANCE.calculateJobModel(config, changelogPartitionMapping, this.streamMetadataCache, this.grouperMetadata);
    assertEquals(expected, actual);
}

Example 32 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestJobModelCalculator method testBasicSingleStream.

@Test
public void testBasicSingleStream() {
    addStreamMetadataCacheMetadata(this.streamMetadataCache, ImmutableMap.of(SYSTEM_STREAM0, buildSystemStreamMetadata(4)));
    Map<TaskName, Integer> changeLogPartitionMapping = changelogPartitionMapping(4);
    Config config = config(ImmutableList.of(SYSTEM_STREAM0), ImmutableMap.of());
    Map<String, ContainerModel> containerModels = ImmutableMap.of("0", new ContainerModel("0", ImmutableMap.of(taskName(0), taskModel(0, 0), taskName(2), taskModel(2, 2))), "1", new ContainerModel("1", ImmutableMap.of(taskName(1), taskModel(1, 1), taskName(3), taskModel(3, 3))));
    JobModel expected = new JobModel(config, containerModels);
    JobModel actual = JobModelCalculator.INSTANCE.calculateJobModel(config, changeLogPartitionMapping, this.streamMetadataCache, this.grouperMetadata);
    assertEquals(expected, actual);
}

Example 33 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestJobModelCalculator method testWithRegexTopicRewriters.

@Test
public void testWithRegexTopicRewriters() {
    // this is the SystemStream that is directly in the config
    SystemStream existingSystemStream = new SystemStream("existingSystem", "existingStream");
    addStreamMetadataCacheMetadata(this.streamMetadataCache, ImmutableMap.of(SYSTEM_STREAM0, buildSystemStreamMetadata(4), SYSTEM_STREAM1, buildSystemStreamMetadata(3), existingSystemStream, buildSystemStreamMetadata(1)));
    Map<TaskName, Integer> changelogPartitionMapping = changelogPartitionMapping(4);
    PowerMockito.mockStatic(ConfigUtil.class);
    // add SYSTEM_STREAM0 for one rewriter
    PowerMockito.when(ConfigUtil.applyRewriter(any(), eq(REGEX_REWRITER0))).thenAnswer(invocation -> addSystemStreamInput(SYSTEM_STREAM0, invocation.getArgumentAt(0, Config.class)));
    // add SYSTEM_STREAM1 for another rewriter
    PowerMockito.when(ConfigUtil.applyRewriter(any(), eq(REGEX_REWRITER1))).thenAnswer(invocation -> addSystemStreamInput(SYSTEM_STREAM1, invocation.getArgumentAt(0, Config.class)));
    Config config = config(ImmutableList.of(existingSystemStream), ImmutableMap.of(JobConfig.CONFIG_REWRITERS, String.format("%s,%s", REGEX_REWRITER0, REGEX_REWRITER1), String.format(JobConfig.CONFIG_REWRITER_CLASS, REGEX_REWRITER0), RegExTopicGenerator.class.getName(), String.format(JobConfig.CONFIG_REWRITER_CLASS, REGEX_REWRITER1), RegExTopicGenerator.class.getName()));
    Set<SystemStreamPartition> sspsForTask0 = ImmutableSet.of(new SystemStreamPartition(SYSTEM_STREAM0, new Partition(0)), new SystemStreamPartition(SYSTEM_STREAM1, new Partition(0)), new SystemStreamPartition(existingSystemStream, new Partition(0)));
    TaskModel taskModel0 = new TaskModel(taskName(0), sspsForTask0, new Partition(0));
    Map<String, ContainerModel> containerModels = ImmutableMap.of("0", new ContainerModel("0", ImmutableMap.of(taskName(0), taskModel0, taskName(2), taskModel(2, 2, 2))), "1", new ContainerModel("1", ImmutableMap.of(taskName(1), taskModel(1, 1, 1), taskName(3), taskModel(3, 3))));
    Map<String, String> expectedConfigMap = new HashMap<>(config);
    expectedConfigMap.put(TaskConfig.INPUT_STREAMS, String.format("%s,%s,%s", taskInputString(existingSystemStream), taskInputString(SYSTEM_STREAM0), taskInputString(SYSTEM_STREAM1)));
    JobModel expected = new JobModel(new MapConfig(expectedConfigMap), containerModels);
    JobModel actual = JobModelCalculator.INSTANCE.calculateJobModel(config, changelogPartitionMapping, this.streamMetadataCache, this.grouperMetadata);
    assertEquals(expected, actual);
}

Example 34 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestJobModelCalculator method testBasicMultipleStreams.

@Test
public void testBasicMultipleStreams() {
    Map<TaskName, Integer> changelogPartitionMapping = changelogPartitionMapping(4);
    Config config = config(ImmutableList.of(SYSTEM_STREAM0, SYSTEM_STREAM1), ImmutableMap.of());
    Map<String, ContainerModel> containerModels = ImmutableMap.of("0", new ContainerModel("0", ImmutableMap.of(taskName(0), taskModel(0, 0, 0), taskName(2), taskModel(2, 2, 2))), "1", new ContainerModel("1", ImmutableMap.of(taskName(1), taskModel(1, 1, 1), taskName(3), taskModel(3, 3))));
    JobModel expected = new JobModel(config, containerModels);
    JobModel actual = JobModelCalculator.INSTANCE.calculateJobModel(config, changelogPartitionMapping, this.streamMetadataCache, this.grouperMetadata);
    assertEquals(expected, actual);
}

Example 35 with ContainerModel

use of org.apache.samza.job.model.ContainerModel in project samza by apache.

the class TestJobModelCalculator method testCustomTaskNameGrouper.

@Test
public void testCustomTaskNameGrouper() {
    Map<TaskName, Integer> changelogPartitionMapping = changelogPartitionMapping(4);
    Config config = config(ImmutableList.of(SYSTEM_STREAM0, SYSTEM_STREAM1), ImmutableMap.of(TaskConfig.GROUPER_FACTORY, Task0SeparateFactory.class.getName()));
    when(this.grouperMetadata.getProcessorLocality()).thenReturn(ImmutableMap.of("0", mock(LocationId.class), "1", mock(LocationId.class)));
    Map<String, ContainerModel> containerModels = ImmutableMap.of("0", new ContainerModel("0", ImmutableMap.of(taskName(0), taskModel(0, 0, 0))), "1", new ContainerModel("1", ImmutableMap.of(taskName(1), taskModel(1, 1, 1), taskName(2), taskModel(2, 2, 2), taskName(3), taskModel(3, 3))));
    JobModel expected = new JobModel(config, containerModels);
    JobModel actual = JobModelCalculator.INSTANCE.calculateJobModel(config, changelogPartitionMapping, this.streamMetadataCache, this.grouperMetadata);
    assertEquals(expected, actual);
}