Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 16 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestContainerProcessManager method testContainerProcessManager.

@Test
public void testContainerProcessManager() throws Exception {
    Map<String, String> conf = new HashMap<>();
    conf.putAll(getConfig());
    conf.put("cluster-manager.container.memory.mb", "500");
    conf.put("cluster-manager.container.cpu.cores", "5");
    SamzaApplicationState state = new SamzaApplicationState(getJobModelManager(1));
    MockClusterResourceManagerCallback callback = new MockClusterResourceManagerCallback();
    MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    LocalityManager mockLocalityManager = mock(LocalityManager.class);
    when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("0", "host1"))));
    ContainerManager containerManager = buildContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, true, false, mockLocalityManager, faultDomainManager);
    ContainerProcessManager cpm = buildContainerProcessManager(new ClusterManagerConfig(new MapConfig(conf)), state, clusterResourceManager, Optional.empty());
    ContainerAllocator allocator = (ContainerAllocator) getPrivateFieldFromCpm("containerAllocator", cpm).get(cpm);
    assertEquals(ContainerAllocator.class, allocator.getClass());
    // Asserts that samza exposed container configs is honored by allocator thread
    assertEquals(500, allocator.containerMemoryMb);
    assertEquals(5, allocator.containerNumCpuCores);
    conf.clear();
    conf.putAll(getConfigWithHostAffinity());
    conf.put("cluster-manager.container.memory.mb", "500");
    conf.put("cluster-manager.container.cpu.cores", "5");
    state = new SamzaApplicationState(getJobModelManager(1));
    callback = new MockClusterResourceManagerCallback();
    clusterResourceManager = new MockClusterResourceManager(callback, state);
    cpm = new ContainerProcessManager(new ClusterManagerConfig(new MapConfig(conf)), state, new MetricsRegistryMap(), clusterResourceManager, Optional.empty(), containerManager, mockLocalityManager, false);
    allocator = (ContainerAllocator) getPrivateFieldFromCpm("containerAllocator", cpm).get(cpm);
    assertEquals(ContainerAllocator.class, allocator.getClass());
    // Asserts that samza exposed container configs is honored by allocator thread
    assertEquals(500, allocator.containerMemoryMb);
    assertEquals(5, allocator.containerNumCpuCores);
}

Example 17 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestContainerProcessManager method testRerequestOnAnyHostIfContainerStartFails.

@Test
public void testRerequestOnAnyHostIfContainerStartFails() throws Exception {
    SamzaApplicationState state = new SamzaApplicationState(getJobModelManager(1));
    Map<String, String> configMap = new HashMap<>();
    configMap.putAll(getConfig());
    MockClusterResourceManagerCallback callback = new MockClusterResourceManagerCallback();
    MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    LocalityManager mockLocalityManager = mock(LocalityManager.class);
    when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("1", "host1"))));
    ContainerManager containerManager = buildContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, Boolean.valueOf(config.get(ClusterManagerConfig.HOST_AFFINITY_ENABLED)), false, mockLocalityManager, faultDomainManager);
    MockContainerAllocatorWithoutHostAffinity allocator = new MockContainerAllocatorWithoutHostAffinity(clusterResourceManager, new MapConfig(config), state, containerManager);
    ContainerProcessManager manager = new ContainerProcessManager(new ClusterManagerConfig(config), state, new MetricsRegistryMap(), clusterResourceManager, Optional.of(allocator), containerManager, mockLocalityManager, false);
    manager.start();
    SamzaResource resource = new SamzaResource(1, 1024, "host1", "resource-1");
    state.pendingProcessors.put("1", resource);
    Assert.assertEquals(clusterResourceManager.resourceRequests.size(), 1);
    manager.onStreamProcessorLaunchFailure(resource, new Exception("cannot launch container!"));
    Assert.assertEquals(clusterResourceManager.resourceRequests.size(), 2);
    Assert.assertEquals(clusterResourceManager.resourceRequests.get(1).getHost(), ResourceRequestState.ANY_HOST);
    manager.stop();
}

Example 18 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestStandbyAllocator method getJobModelWithStandby.

// Helper method to create a jobmodel with given number of containers, tasks and replication factor
public static JobModel getJobModelWithStandby(int nContainers, int nTasks, int replicationFactor) {
    Map<String, ContainerModel> containerModels = new HashMap<>();
    int taskID = 0;
    for (int j = 0; j < nContainers; j++) {
        Map<TaskName, TaskModel> tasks = new HashMap<>();
        for (int i = 0; i < nTasks; i++) {
            TaskModel taskModel = getTaskModel(taskID++);
            tasks.put(taskModel.getTaskName(), taskModel);
        }
        containerModels.put(String.valueOf(j), new ContainerModel(String.valueOf(j), tasks));
    }
    Map<String, ContainerModel> standbyContainerModels = new HashMap<>();
    for (int i = 0; i < replicationFactor - 1; i++) {
        for (String containerID : containerModels.keySet()) {
            String standbyContainerId = StandbyTaskUtil.getStandbyContainerId(containerID, i);
            Map<TaskName, TaskModel> standbyTasks = getStandbyTasks(containerModels.get(containerID).getTasks(), i);
            standbyContainerModels.put(standbyContainerId, new ContainerModel(standbyContainerId, standbyTasks));
        }
    }
    containerModels.putAll(standbyContainerModels);
    return new JobModel(new MapConfig(), containerModels);
}

Example 19 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestGroupByPartition method testNoTaskOnlyContainsBroadcastStreams.

@Test
public void testNoTaskOnlyContainsBroadcastStreams() {
    Config config = new MapConfig(ImmutableMap.of("task.broadcast.inputs", "SystemA.StreamA#0, SystemA.StreamB#1"));
    GroupByPartition grouper = new GroupByPartition(config);
    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(ImmutableSet.of(aa0, ab1, ab2));
    Map<TaskName, Set<SystemStreamPartition>> expectedResult = ImmutableMap.<TaskName, Set<SystemStreamPartition>>builder().put(new TaskName("Partition 2"), ImmutableSet.of(aa0, ab1, ab2)).build();
    assertEquals(expectedResult, result);
}

Example 20 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestClusterBasedJobCoordinatorRunner method testRunClusterBasedJobCoordinator.

@Test
public void testRunClusterBasedJobCoordinator() throws Exception {
    Config submissionConfig = new MapConfig(ImmutableMap.of(JobConfig.CONFIG_LOADER_FACTORY, PropertiesConfigLoaderFactory.class.getName(), PropertiesConfigLoaderFactory.CONFIG_LOADER_PROPERTIES_PREFIX + "path", getClass().getResource("/test.properties").getPath()));
    Config fullConfig = ConfigUtil.loadConfig(submissionConfig);
    StreamApplication mockApplication = mock(StreamApplication.class);
    PowerMockito.mockStatic(System.class, ApplicationUtil.class, JobCoordinatorLaunchUtil.class);
    PowerMockito.when(System.getenv(eq(ShellCommandConfig.ENV_SUBMISSION_CONFIG))).thenReturn(SamzaObjectMapper.getObjectMapper().writeValueAsString(submissionConfig));
    PowerMockito.when(ApplicationUtil.fromConfig(any())).thenReturn(mockApplication);
    PowerMockito.doNothing().when(JobCoordinatorLaunchUtil.class, "run", mockApplication, fullConfig);
    ClusterBasedJobCoordinatorRunner.runClusterBasedJobCoordinator(null);
    PowerMockito.verifyStatic(times(1));
    JobCoordinatorLaunchUtil.run(mockApplication, fullConfig);
}