Search in sources :

Example 1 with ProcessorLocality

use of org.apache.samza.job.model.ProcessorLocality in project samza by apache.

the class TestContainerProcessManager method testContainerProcessManager.

@Test
public void testContainerProcessManager() throws Exception {
    Map<String, String> conf = new HashMap<>();
    conf.putAll(getConfig());
    conf.put("cluster-manager.container.memory.mb", "500");
    conf.put("cluster-manager.container.cpu.cores", "5");
    SamzaApplicationState state = new SamzaApplicationState(getJobModelManager(1));
    MockClusterResourceManagerCallback callback = new MockClusterResourceManagerCallback();
    MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    LocalityManager mockLocalityManager = mock(LocalityManager.class);
    when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("0", "host1"))));
    ContainerManager containerManager = buildContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, true, false, mockLocalityManager, faultDomainManager);
    ContainerProcessManager cpm = buildContainerProcessManager(new ClusterManagerConfig(new MapConfig(conf)), state, clusterResourceManager, Optional.empty());
    ContainerAllocator allocator = (ContainerAllocator) getPrivateFieldFromCpm("containerAllocator", cpm).get(cpm);
    assertEquals(ContainerAllocator.class, allocator.getClass());
    // Asserts that samza exposed container configs is honored by allocator thread
    assertEquals(500, allocator.containerMemoryMb);
    assertEquals(5, allocator.containerNumCpuCores);
    conf.clear();
    conf.putAll(getConfigWithHostAffinity());
    conf.put("cluster-manager.container.memory.mb", "500");
    conf.put("cluster-manager.container.cpu.cores", "5");
    state = new SamzaApplicationState(getJobModelManager(1));
    callback = new MockClusterResourceManagerCallback();
    clusterResourceManager = new MockClusterResourceManager(callback, state);
    cpm = new ContainerProcessManager(new ClusterManagerConfig(new MapConfig(conf)), state, new MetricsRegistryMap(), clusterResourceManager, Optional.empty(), containerManager, mockLocalityManager, false);
    allocator = (ContainerAllocator) getPrivateFieldFromCpm("containerAllocator", cpm).get(cpm);
    assertEquals(ContainerAllocator.class, allocator.getClass());
    // Asserts that samza exposed container configs is honored by allocator thread
    assertEquals(500, allocator.containerMemoryMb);
    assertEquals(5, allocator.containerNumCpuCores);
}
Also used : HashMap(java.util.HashMap) Matchers.anyString(org.mockito.Matchers.anyString) LocalityModel(org.apache.samza.job.model.LocalityModel) ProcessorLocality(org.apache.samza.job.model.ProcessorLocality) ClusterManagerConfig(org.apache.samza.config.ClusterManagerConfig) MapConfig(org.apache.samza.config.MapConfig) LocalityManager(org.apache.samza.container.LocalityManager) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) Test(org.junit.Test)

Example 2 with ProcessorLocality

use of org.apache.samza.job.model.ProcessorLocality in project samza by apache.

the class TestContainerProcessManager method testRerequestOnAnyHostIfContainerStartFails.

@Test
public void testRerequestOnAnyHostIfContainerStartFails() throws Exception {
    SamzaApplicationState state = new SamzaApplicationState(getJobModelManager(1));
    Map<String, String> configMap = new HashMap<>();
    configMap.putAll(getConfig());
    MockClusterResourceManagerCallback callback = new MockClusterResourceManagerCallback();
    MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    LocalityManager mockLocalityManager = mock(LocalityManager.class);
    when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("1", "host1"))));
    ContainerManager containerManager = buildContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, Boolean.valueOf(config.get(ClusterManagerConfig.HOST_AFFINITY_ENABLED)), false, mockLocalityManager, faultDomainManager);
    MockContainerAllocatorWithoutHostAffinity allocator = new MockContainerAllocatorWithoutHostAffinity(clusterResourceManager, new MapConfig(config), state, containerManager);
    ContainerProcessManager manager = new ContainerProcessManager(new ClusterManagerConfig(config), state, new MetricsRegistryMap(), clusterResourceManager, Optional.of(allocator), containerManager, mockLocalityManager, false);
    manager.start();
    SamzaResource resource = new SamzaResource(1, 1024, "host1", "resource-1");
    state.pendingProcessors.put("1", resource);
    Assert.assertEquals(clusterResourceManager.resourceRequests.size(), 1);
    manager.onStreamProcessorLaunchFailure(resource, new Exception("cannot launch container!"));
    Assert.assertEquals(clusterResourceManager.resourceRequests.size(), 2);
    Assert.assertEquals(clusterResourceManager.resourceRequests.get(1).getHost(), ResourceRequestState.ANY_HOST);
    manager.stop();
}
Also used : HashMap(java.util.HashMap) Matchers.anyString(org.mockito.Matchers.anyString) LocalityModel(org.apache.samza.job.model.LocalityModel) ProcessorLocality(org.apache.samza.job.model.ProcessorLocality) ClusterManagerConfig(org.apache.samza.config.ClusterManagerConfig) MapConfig(org.apache.samza.config.MapConfig) LocalityManager(org.apache.samza.container.LocalityManager) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) Test(org.junit.Test)

Example 3 with ProcessorLocality

use of org.apache.samza.job.model.ProcessorLocality in project samza by apache.

the class LocalityManager method readLocality.

/**
 * Fetch the processor locality information from the {@link MetadataStore}. In YARN deployment model, the
 * processor refers to the samza container.
 *
 * @return the {@code LocalityModel} for the job
 */
public LocalityModel readLocality() {
    Map<String, ProcessorLocality> containerLocalityMap = new HashMap<>();
    metadataStore.all().forEach((processorId, valueBytes) -> {
        if (valueBytes != null) {
            String locationId = valueSerde.fromBytes(valueBytes);
            containerLocalityMap.put(processorId, new ProcessorLocality(processorId, locationId));
        }
    });
    if (LOG.isDebugEnabled()) {
        for (Map.Entry<String, ProcessorLocality> entry : containerLocalityMap.entrySet()) {
            LOG.debug(String.format("Locality for container %s: %s", entry.getKey(), entry.getValue().host()));
        }
    }
    return new LocalityModel(containerLocalityMap);
}
Also used : ProcessorLocality(org.apache.samza.job.model.ProcessorLocality) HashMap(java.util.HashMap) LocalityModel(org.apache.samza.job.model.LocalityModel) Map(java.util.Map) HashMap(java.util.HashMap)

Example 4 with ProcessorLocality

use of org.apache.samza.job.model.ProcessorLocality in project samza by apache.

the class TestContainerPlacementActions method setup.

@Before
public void setup() throws Exception {
    server = new MockHttpServer("/", 7777, null, new ServletHolder(DefaultServlet.class));
    // Utils Related to Container Placement Metadata store
    CoordinatorStreamStoreTestUtil coordinatorStreamStoreTestUtil = new CoordinatorStreamStoreTestUtil(config);
    coordinatorStreamStore = coordinatorStreamStoreTestUtil.getCoordinatorStreamStore();
    coordinatorStreamStore.init();
    containerPlacementMetadataStore = new ContainerPlacementMetadataStore(coordinatorStreamStore);
    containerPlacementMetadataStore.start();
    // Utils Related to Cluster manager:
    config = new MapConfig(configVals, getConfigWithHostAffinityAndRetries(true, 1, true));
    state = new SamzaApplicationState(JobModelManagerTestUtil.getJobModelManager(getConfig(), 2, server));
    callback = mock(ClusterResourceManager.Callback.class);
    MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
    localityManager = mock(LocalityManager.class);
    when(localityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("0", "host-1"), "1", new ProcessorLocality("1", "host-2"))));
    containerManager = spy(new ContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, true, false, localityManager, faultDomainManager, config));
    allocatorWithHostAffinity = new MockContainerAllocatorWithHostAffinity(clusterResourceManager, config, state, containerManager);
    cpm = new ContainerProcessManager(clusterManagerConfig, state, new MetricsRegistryMap(), clusterResourceManager, Optional.of(allocatorWithHostAffinity), containerManager, localityManager, false);
}
Also used : ServletHolder(org.eclipse.jetty.servlet.ServletHolder) ContainerPlacementMetadataStore(org.apache.samza.clustermanager.container.placement.ContainerPlacementMetadataStore) LocalityModel(org.apache.samza.job.model.LocalityModel) CoordinatorStreamStoreTestUtil(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStoreTestUtil) ProcessorLocality(org.apache.samza.job.model.ProcessorLocality) ClusterManagerConfig(org.apache.samza.config.ClusterManagerConfig) MockHttpServer(org.apache.samza.testUtils.MockHttpServer) MapConfig(org.apache.samza.config.MapConfig) LocalityManager(org.apache.samza.container.LocalityManager) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) Before(org.junit.Before)

Example 5 with ProcessorLocality

use of org.apache.samza.job.model.ProcessorLocality in project samza by apache.

the class TestContainerPlacementActions method setupStandby.

public void setupStandby() throws Exception {
    LocalityManager mockLocalityManager = mock(LocalityManager.class);
    when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("0", "host-1"), "1", new ProcessorLocality("1", "host-2"), "0-0", new ProcessorLocality("0", "host-2"), "1-0", new ProcessorLocality("0", "host-1"))));
    state = new SamzaApplicationState(getJobModelManagerWithStandby());
    callback = mock(ClusterResourceManager.Callback.class);
    MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
    // Enable standby
    containerManager = spy(new ContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, true, true, mockLocalityManager, faultDomainManager, config));
    allocatorWithHostAffinity = new MockContainerAllocatorWithHostAffinity(clusterResourceManager, config, state, containerManager);
    cpm = new ContainerProcessManager(clusterManagerConfig, state, new MetricsRegistryMap(), clusterResourceManager, Optional.of(allocatorWithHostAffinity), containerManager, mockLocalityManager, false);
}
Also used : ProcessorLocality(org.apache.samza.job.model.ProcessorLocality) ClusterManagerConfig(org.apache.samza.config.ClusterManagerConfig) LocalityManager(org.apache.samza.container.LocalityManager) LocalityModel(org.apache.samza.job.model.LocalityModel) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap)

Aggregations

ProcessorLocality (org.apache.samza.job.model.ProcessorLocality)18 LocalityModel (org.apache.samza.job.model.LocalityModel)12 LocalityManager (org.apache.samza.container.LocalityManager)10 HashMap (java.util.HashMap)8 ClusterManagerConfig (org.apache.samza.config.ClusterManagerConfig)6 MetricsRegistryMap (org.apache.samza.metrics.MetricsRegistryMap)6 MapConfig (org.apache.samza.config.MapConfig)5 Test (org.junit.Test)5 JobConfig (org.apache.samza.config.JobConfig)4 LocationId (org.apache.samza.runtime.LocationId)4 Matchers.anyString (org.mockito.Matchers.anyString)4 Map (java.util.Map)3 Config (org.apache.samza.config.Config)3 CoordinatorStreamStore (org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore)3 ArrayList (java.util.ArrayList)2 List (java.util.List)2 Optional (java.util.Optional)2 Collectors (java.util.stream.Collectors)2 StringUtils (org.apache.commons.lang3.StringUtils)2 ContainerPlacementMetadataStore (org.apache.samza.clustermanager.container.placement.ContainerPlacementMetadataStore)2