use of org.apache.samza.job.model.ProcessorLocality in project samza by apache.
the class TestContainerProcessManager method testContainerProcessManager.
@Test
public void testContainerProcessManager() throws Exception {
Map<String, String> conf = new HashMap<>();
conf.putAll(getConfig());
conf.put("cluster-manager.container.memory.mb", "500");
conf.put("cluster-manager.container.cpu.cores", "5");
SamzaApplicationState state = new SamzaApplicationState(getJobModelManager(1));
MockClusterResourceManagerCallback callback = new MockClusterResourceManagerCallback();
MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
LocalityManager mockLocalityManager = mock(LocalityManager.class);
when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("0", "host1"))));
ContainerManager containerManager = buildContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, true, false, mockLocalityManager, faultDomainManager);
ContainerProcessManager cpm = buildContainerProcessManager(new ClusterManagerConfig(new MapConfig(conf)), state, clusterResourceManager, Optional.empty());
ContainerAllocator allocator = (ContainerAllocator) getPrivateFieldFromCpm("containerAllocator", cpm).get(cpm);
assertEquals(ContainerAllocator.class, allocator.getClass());
// Asserts that samza exposed container configs is honored by allocator thread
assertEquals(500, allocator.containerMemoryMb);
assertEquals(5, allocator.containerNumCpuCores);
conf.clear();
conf.putAll(getConfigWithHostAffinity());
conf.put("cluster-manager.container.memory.mb", "500");
conf.put("cluster-manager.container.cpu.cores", "5");
state = new SamzaApplicationState(getJobModelManager(1));
callback = new MockClusterResourceManagerCallback();
clusterResourceManager = new MockClusterResourceManager(callback, state);
cpm = new ContainerProcessManager(new ClusterManagerConfig(new MapConfig(conf)), state, new MetricsRegistryMap(), clusterResourceManager, Optional.empty(), containerManager, mockLocalityManager, false);
allocator = (ContainerAllocator) getPrivateFieldFromCpm("containerAllocator", cpm).get(cpm);
assertEquals(ContainerAllocator.class, allocator.getClass());
// Asserts that samza exposed container configs is honored by allocator thread
assertEquals(500, allocator.containerMemoryMb);
assertEquals(5, allocator.containerNumCpuCores);
}
use of org.apache.samza.job.model.ProcessorLocality in project samza by apache.
the class TestContainerProcessManager method testRerequestOnAnyHostIfContainerStartFails.
@Test
public void testRerequestOnAnyHostIfContainerStartFails() throws Exception {
SamzaApplicationState state = new SamzaApplicationState(getJobModelManager(1));
Map<String, String> configMap = new HashMap<>();
configMap.putAll(getConfig());
MockClusterResourceManagerCallback callback = new MockClusterResourceManagerCallback();
MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
LocalityManager mockLocalityManager = mock(LocalityManager.class);
when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("1", "host1"))));
ContainerManager containerManager = buildContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, Boolean.valueOf(config.get(ClusterManagerConfig.HOST_AFFINITY_ENABLED)), false, mockLocalityManager, faultDomainManager);
MockContainerAllocatorWithoutHostAffinity allocator = new MockContainerAllocatorWithoutHostAffinity(clusterResourceManager, new MapConfig(config), state, containerManager);
ContainerProcessManager manager = new ContainerProcessManager(new ClusterManagerConfig(config), state, new MetricsRegistryMap(), clusterResourceManager, Optional.of(allocator), containerManager, mockLocalityManager, false);
manager.start();
SamzaResource resource = new SamzaResource(1, 1024, "host1", "resource-1");
state.pendingProcessors.put("1", resource);
Assert.assertEquals(clusterResourceManager.resourceRequests.size(), 1);
manager.onStreamProcessorLaunchFailure(resource, new Exception("cannot launch container!"));
Assert.assertEquals(clusterResourceManager.resourceRequests.size(), 2);
Assert.assertEquals(clusterResourceManager.resourceRequests.get(1).getHost(), ResourceRequestState.ANY_HOST);
manager.stop();
}
use of org.apache.samza.job.model.ProcessorLocality in project samza by apache.
the class LocalityManager method readLocality.
/**
* Fetch the processor locality information from the {@link MetadataStore}. In YARN deployment model, the
* processor refers to the samza container.
*
* @return the {@code LocalityModel} for the job
*/
public LocalityModel readLocality() {
Map<String, ProcessorLocality> containerLocalityMap = new HashMap<>();
metadataStore.all().forEach((processorId, valueBytes) -> {
if (valueBytes != null) {
String locationId = valueSerde.fromBytes(valueBytes);
containerLocalityMap.put(processorId, new ProcessorLocality(processorId, locationId));
}
});
if (LOG.isDebugEnabled()) {
for (Map.Entry<String, ProcessorLocality> entry : containerLocalityMap.entrySet()) {
LOG.debug(String.format("Locality for container %s: %s", entry.getKey(), entry.getValue().host()));
}
}
return new LocalityModel(containerLocalityMap);
}
use of org.apache.samza.job.model.ProcessorLocality in project samza by apache.
the class TestContainerPlacementActions method setup.
@Before
public void setup() throws Exception {
server = new MockHttpServer("/", 7777, null, new ServletHolder(DefaultServlet.class));
// Utils Related to Container Placement Metadata store
CoordinatorStreamStoreTestUtil coordinatorStreamStoreTestUtil = new CoordinatorStreamStoreTestUtil(config);
coordinatorStreamStore = coordinatorStreamStoreTestUtil.getCoordinatorStreamStore();
coordinatorStreamStore.init();
containerPlacementMetadataStore = new ContainerPlacementMetadataStore(coordinatorStreamStore);
containerPlacementMetadataStore.start();
// Utils Related to Cluster manager:
config = new MapConfig(configVals, getConfigWithHostAffinityAndRetries(true, 1, true));
state = new SamzaApplicationState(JobModelManagerTestUtil.getJobModelManager(getConfig(), 2, server));
callback = mock(ClusterResourceManager.Callback.class);
MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
localityManager = mock(LocalityManager.class);
when(localityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("0", "host-1"), "1", new ProcessorLocality("1", "host-2"))));
containerManager = spy(new ContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, true, false, localityManager, faultDomainManager, config));
allocatorWithHostAffinity = new MockContainerAllocatorWithHostAffinity(clusterResourceManager, config, state, containerManager);
cpm = new ContainerProcessManager(clusterManagerConfig, state, new MetricsRegistryMap(), clusterResourceManager, Optional.of(allocatorWithHostAffinity), containerManager, localityManager, false);
}
use of org.apache.samza.job.model.ProcessorLocality in project samza by apache.
the class TestContainerPlacementActions method setupStandby.
public void setupStandby() throws Exception {
LocalityManager mockLocalityManager = mock(LocalityManager.class);
when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("0", "host-1"), "1", new ProcessorLocality("1", "host-2"), "0-0", new ProcessorLocality("0", "host-2"), "1-0", new ProcessorLocality("0", "host-1"))));
state = new SamzaApplicationState(getJobModelManagerWithStandby());
callback = mock(ClusterResourceManager.Callback.class);
MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
// Enable standby
containerManager = spy(new ContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, true, true, mockLocalityManager, faultDomainManager, config));
allocatorWithHostAffinity = new MockContainerAllocatorWithHostAffinity(clusterResourceManager, config, state, containerManager);
cpm = new ContainerProcessManager(clusterManagerConfig, state, new MetricsRegistryMap(), clusterResourceManager, Optional.of(allocatorWithHostAffinity), containerManager, mockLocalityManager, false);
}
Aggregations