Examples with LocationId org.apache.samza.runtime.LocationId used on opensource projects

Example 1 with LocationId

use of org.apache.samza.runtime.LocationId in project samza by apache.

the class TestGroupByContainerIds method testShouldGenerateCorrectContainerModelWhenTaskLocalityIsEmpty.

@Test
public void testShouldGenerateCorrectContainerModelWhenTaskLocalityIsEmpty() {
    TaskNameGrouper taskNameGrouper = buildSimpleGrouper(3);
    String testProcessorId1 = "testProcessorId1";
    String testProcessorId2 = "testProcessorId2";
    String testProcessorId3 = "testProcessorId3";
    LocationId testLocationId1 = new LocationId("testLocationId1");
    LocationId testLocationId2 = new LocationId("testLocationId2");
    LocationId testLocationId3 = new LocationId("testLocationId3");
    TaskName testTaskName1 = new TaskName("testTasKId1");
    TaskName testTaskName2 = new TaskName("testTaskId2");
    TaskName testTaskName3 = new TaskName("testTaskId3");
    TaskModel testTaskModel1 = new TaskModel(testTaskName1, new HashSet<>(), new Partition(0));
    TaskModel testTaskModel2 = new TaskModel(testTaskName2, new HashSet<>(), new Partition(1));
    TaskModel testTaskModel3 = new TaskModel(testTaskName3, new HashSet<>(), new Partition(2));
    Map<String, LocationId> processorLocality = ImmutableMap.of(testProcessorId1, testLocationId1, testProcessorId2, testLocationId2, testProcessorId3, testLocationId3);
    Map<TaskName, LocationId> taskLocality = ImmutableMap.of(testTaskName1, testLocationId1);
    GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(processorLocality, taskLocality, new HashMap<>(), new HashMap<>());
    Set<TaskModel> taskModels = ImmutableSet.of(testTaskModel1, testTaskModel2, testTaskModel3);
    Set<ContainerModel> expectedContainerModels = ImmutableSet.of(new ContainerModel(testProcessorId1, ImmutableMap.of(testTaskName1, testTaskModel1)), new ContainerModel(testProcessorId2, ImmutableMap.of(testTaskName2, testTaskModel2)), new ContainerModel(testProcessorId3, ImmutableMap.of(testTaskName3, testTaskModel3)));
    Set<ContainerModel> actualContainerModels = taskNameGrouper.group(taskModels, grouperMetadata);
    assertEquals(expectedContainerModels, actualContainerModels);
}

Example 2 with LocationId

use of org.apache.samza.runtime.LocationId in project samza by apache.

the class TestGroupByContainerIds method testShouldGenerateIdenticalTaskDistributionWhenNoChangeInProcessorGroup.

@Test
public void testShouldGenerateIdenticalTaskDistributionWhenNoChangeInProcessorGroup() {
    TaskNameGrouper taskNameGrouper = buildSimpleGrouper(3);
    String testProcessorId1 = "testProcessorId1";
    String testProcessorId2 = "testProcessorId2";
    String testProcessorId3 = "testProcessorId3";
    LocationId testLocationId1 = new LocationId("testLocationId1");
    LocationId testLocationId2 = new LocationId("testLocationId2");
    LocationId testLocationId3 = new LocationId("testLocationId3");
    TaskName testTaskName1 = new TaskName("testTasKId1");
    TaskName testTaskName2 = new TaskName("testTaskId2");
    TaskName testTaskName3 = new TaskName("testTaskId3");
    TaskModel testTaskModel1 = new TaskModel(testTaskName1, new HashSet<>(), new Partition(0));
    TaskModel testTaskModel2 = new TaskModel(testTaskName2, new HashSet<>(), new Partition(1));
    TaskModel testTaskModel3 = new TaskModel(testTaskName3, new HashSet<>(), new Partition(2));
    Map<String, LocationId> processorLocality = ImmutableMap.of(testProcessorId1, testLocationId1, testProcessorId2, testLocationId2, testProcessorId3, testLocationId3);
    Map<TaskName, LocationId> taskLocality = ImmutableMap.of(testTaskName1, testLocationId1, testTaskName2, testLocationId2, testTaskName3, testLocationId3);
    GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(processorLocality, taskLocality, new HashMap<>(), new HashMap<>());
    Set<TaskModel> taskModels = ImmutableSet.of(testTaskModel1, testTaskModel2, testTaskModel3);
    Set<ContainerModel> expectedContainerModels = ImmutableSet.of(new ContainerModel(testProcessorId1, ImmutableMap.of(testTaskName1, testTaskModel1)), new ContainerModel(testProcessorId2, ImmutableMap.of(testTaskName2, testTaskModel2)), new ContainerModel(testProcessorId3, ImmutableMap.of(testTaskName3, testTaskModel3)));
    Set<ContainerModel> actualContainerModels = taskNameGrouper.group(taskModels, grouperMetadata);
    assertEquals(expectedContainerModels, actualContainerModels);
    actualContainerModels = taskNameGrouper.group(taskModels, grouperMetadata);
    assertEquals(expectedContainerModels, actualContainerModels);
}

Example 3 with LocationId

use of org.apache.samza.runtime.LocationId in project samza by apache.

the class TestGroupByContainerIds method testGenerateContainerModelForSingleContainer.

@Test
public void testGenerateContainerModelForSingleContainer() {
    TaskNameGrouper taskNameGrouper = buildSimpleGrouper(1);
    String testProcessorId1 = "testProcessorId1";
    LocationId testLocationId1 = new LocationId("testLocationId1");
    LocationId testLocationId2 = new LocationId("testLocationId2");
    LocationId testLocationId3 = new LocationId("testLocationId3");
    TaskName testTaskName1 = new TaskName("testTasKId1");
    TaskName testTaskName2 = new TaskName("testTaskId2");
    TaskName testTaskName3 = new TaskName("testTaskId3");
    TaskModel testTaskModel1 = new TaskModel(testTaskName1, new HashSet<>(), new Partition(0));
    TaskModel testTaskModel2 = new TaskModel(testTaskName2, new HashSet<>(), new Partition(1));
    TaskModel testTaskModel3 = new TaskModel(testTaskName3, new HashSet<>(), new Partition(2));
    Map<String, LocationId> processorLocality = ImmutableMap.of(testProcessorId1, testLocationId1);
    Map<TaskName, LocationId> taskLocality = ImmutableMap.of(testTaskName1, testLocationId1, testTaskName2, testLocationId2, testTaskName3, testLocationId3);
    GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(processorLocality, taskLocality, new HashMap<>(), new HashMap<>());
    Set<TaskModel> taskModels = ImmutableSet.of(testTaskModel1, testTaskModel2, testTaskModel3);
    Set<ContainerModel> expectedContainerModels = ImmutableSet.of(new ContainerModel(testProcessorId1, ImmutableMap.of(testTaskName1, testTaskModel1, testTaskName2, testTaskModel2, testTaskName3, testTaskModel3)));
    Set<ContainerModel> actualContainerModels = taskNameGrouper.group(taskModels, grouperMetadata);
    assertEquals(expectedContainerModels, actualContainerModels);
}

Example 4 with LocationId

use of org.apache.samza.runtime.LocationId in project samza by apache.

the class ZkJobCoordinator method getGrouperMetadata.

/**
 * Builds the {@link GrouperMetadataImpl} based upon provided {@param jobModelVersion}
 * and {@param processorNodes}.
 * @param jobModelVersion the most recent jobModelVersion available in the zookeeper.
 * @param processorNodes the list of live processors in the zookeeper.
 * @return the built grouper metadata.
 */
private GrouperMetadataImpl getGrouperMetadata(String jobModelVersion, List<ProcessorNode> processorNodes) {
    Map<TaskName, String> taskToProcessorId = new HashMap<>();
    Map<TaskName, List<SystemStreamPartition>> taskToSSPs = new HashMap<>();
    if (jobModelVersion != null) {
        JobModel jobModel = readJobModelFromMetadataStore(jobModelVersion);
        for (ContainerModel containerModel : jobModel.getContainers().values()) {
            for (TaskModel taskModel : containerModel.getTasks().values()) {
                taskToProcessorId.put(taskModel.getTaskName(), containerModel.getId());
                for (SystemStreamPartition partition : taskModel.getSystemStreamPartitions()) {
                    taskToSSPs.computeIfAbsent(taskModel.getTaskName(), k -> new ArrayList<>());
                    taskToSSPs.get(taskModel.getTaskName()).add(partition);
                }
            }
        }
    }
    Map<String, LocationId> processorLocality = new HashMap<>();
    for (ProcessorNode processorNode : processorNodes) {
        ProcessorData processorData = processorNode.getProcessorData();
        processorLocality.put(processorData.getProcessorId(), processorData.getLocationId());
    }
    Map<TaskName, LocationId> taskLocality = zkUtils.readTaskLocality();
    return new GrouperMetadataImpl(processorLocality, taskLocality, taskToSSPs, taskToProcessorId);
}

Example 5 with LocationId

use of org.apache.samza.runtime.LocationId in project samza by apache.

the class TestZkUtils method testReadTaskLocalityShouldReturnAllTheExistingLocalityValue.

@Test
public void testReadTaskLocalityShouldReturnAllTheExistingLocalityValue() {
    zkUtils.writeTaskLocality(new TaskName("task-1"), new LocationId("LocationId-1"));
    zkUtils.writeTaskLocality(new TaskName("task-2"), new LocationId("LocationId-2"));
    zkUtils.writeTaskLocality(new TaskName("task-3"), new LocationId("LocationId-3"));
    zkUtils.writeTaskLocality(new TaskName("task-4"), new LocationId("LocationId-4"));
    zkUtils.writeTaskLocality(new TaskName("task-5"), new LocationId("LocationId-5"));
    Map<TaskName, LocationId> taskLocality = ImmutableMap.of(new TaskName("task-1"), new LocationId("LocationId-1"), new TaskName("task-2"), new LocationId("LocationId-2"), new TaskName("task-3"), new LocationId("LocationId-3"), new TaskName("task-4"), new LocationId("LocationId-4"), new TaskName("task-5"), new LocationId("LocationId-5"));
    Assert.assertEquals(taskLocality, zkUtils.readTaskLocality());
}