Examples with LocalityModel org.apache.samza.job.model.LocalityModel used on opensource projects

Example 16 with LocalityModel

use of org.apache.samza.job.model.LocalityModel in project samza by apache.

the class TestContainerProcessManager method testAllBufferedResourcesAreUtilized.

@Test
public void testAllBufferedResourcesAreUtilized() throws Exception {
    Map<String, String> config = new HashMap<>();
    config.putAll(getConfigWithHostAffinity());
    config.put("job.container.count", "2");
    config.put("cluster-manager.container.retry.count", "2");
    config.put("cluster-manager.container.request.timeout.ms", "10000");
    Config cfg = new MapConfig(config);
    // 1. Request two containers on hosts - host1 and host2
    SamzaApplicationState state = new SamzaApplicationState(getJobModelManager(2));
    MockClusterResourceManagerCallback callback = new MockClusterResourceManagerCallback();
    MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    LocalityManager mockLocalityManager = mock(LocalityManager.class);
    when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("0", "host1"), "1", new ProcessorLocality("1", "host2"))));
    ContainerManager containerManager = buildContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, Boolean.parseBoolean(config.get(ClusterManagerConfig.HOST_AFFINITY_ENABLED)), false, mockLocalityManager, faultDomainManager);
    MockContainerAllocatorWithHostAffinity allocator = new MockContainerAllocatorWithHostAffinity(clusterResourceManager, cfg, state, containerManager);
    ContainerProcessManager cpm = spy(buildContainerProcessManager(new ClusterManagerConfig(cfg), state, clusterResourceManager, Optional.of(allocator), mockLocalityManager, false, faultDomainManager));
    cpm.start();
    assertFalse(cpm.shouldShutdown());
    // 2. When the task manager starts, there should have been a pending request on host1 and host2
    assertEquals(2, allocator.getContainerRequestState().numPendingRequests());
    // 3. Allocate an extra resource on host1 and no resource on host2 yet.
    SamzaResource resource1 = new SamzaResource(1, 1000, "host1", "id1");
    SamzaResource resource2 = new SamzaResource(1, 1000, "host1", "id2");
    cpm.onResourceAllocated(resource1);
    cpm.onResourceAllocated(resource2);
    // 4. Wait for the container to start on host1 and immediately fail
    if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    cpm.onStreamProcessorLaunchSuccess(resource1);
    assertEquals("host2", allocator.getContainerRequestState().peekPendingRequest().getPreferredHost());
    assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
    cpm.onResourceCompleted(new SamzaResourceStatus(resource1.getContainerId(), "App Error", 1));
    verify(cpm).onResourceCompletedWithUnknownStatus(any(SamzaResourceStatus.class), anyString(), anyString(), anyInt());
    assertEquals(2, allocator.getContainerRequestState().numPendingRequests());
    assertFalse(cpm.shouldShutdown());
    assertFalse(state.jobHealthy.get());
    assertEquals(3, clusterResourceManager.resourceRequests.size());
    assertEquals(0, clusterResourceManager.releasedResources.size());
    // 5. Do not allocate any further resource on host1, and verify that the re-run of the container on host1 uses the
    // previously allocated extra resource
    SamzaResource resource3 = new SamzaResource(1, 1000, "host2", "id3");
    cpm.onResourceAllocated(resource3);
    if (!allocator.awaitContainersStart(2, 2, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    cpm.onStreamProcessorLaunchSuccess(resource2);
    cpm.onStreamProcessorLaunchSuccess(resource3);
    assertTrue(state.jobHealthy.get());
    cpm.stop();
}

Example 17 with LocalityModel

use of org.apache.samza.job.model.LocalityModel in project samza by apache.

the class TestContainerProcessManager method testContainerRequestedRetriesNotExceedingWindowOnFailureWithUnknownCode.

private void testContainerRequestedRetriesNotExceedingWindowOnFailureWithUnknownCode(boolean withHostAffinity, boolean failAfterRetries) throws Exception {
    int maxRetries = 3;
    String processorId = "0";
    ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(getConfigWithHostAffinityAndRetries(withHostAffinity, maxRetries, failAfterRetries));
    SamzaApplicationState state = new SamzaApplicationState(getJobModelManager(1));
    MockClusterResourceManagerCallback callback = new MockClusterResourceManagerCallback();
    MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    LocalityManager mockLocalityManager = mock(LocalityManager.class);
    if (withHostAffinity) {
        when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("0", "host1"))));
    } else {
        when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(new HashMap<>()));
    }
    ContainerManager containerManager = buildContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, clusterManagerConfig.getHostAffinityEnabled(), false, mockLocalityManager, faultDomainManager);
    MockContainerAllocatorWithoutHostAffinity allocator = new MockContainerAllocatorWithoutHostAffinity(clusterResourceManager, clusterManagerConfig, state, containerManager);
    ContainerProcessManager cpm = buildContainerProcessManager(clusterManagerConfig, state, clusterResourceManager, Optional.of(allocator), mockLocalityManager, false, faultDomainManager);
    // start triggers a request
    cpm.start();
    assertFalse(cpm.shouldShutdown());
    assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
    assertEquals(0, allocator.getContainerRequestState().numDelayedRequests());
    SamzaResource container = new SamzaResource(1, 1024, "host1", "id0");
    cpm.onResourceAllocated(container);
    // Allow container to run and update state
    if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    cpm.onStreamProcessorLaunchSuccess(container);
    // Mock 2nd failure not exceeding retry window.
    cpm.getProcessorFailures().put(processorId, new ProcessorFailure(1, Instant.now(), Duration.ZERO));
    cpm.onResourceCompleted(new SamzaResourceStatus(container.getContainerId(), "diagnostics", 1));
    assertEquals(false, cpm.getJobFailureCriteriaMet());
    assertEquals(2, cpm.getProcessorFailures().get(processorId).getCount());
    assertFalse(cpm.shouldShutdown());
    assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
    assertEquals(0, allocator.getContainerRequestState().numDelayedRequests());
    cpm.onResourceAllocated(container);
    // Allow container to run and update state
    if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    cpm.onStreamProcessorLaunchSuccess(container);
    // Mock 3rd failure not exceeding retry window.
    cpm.getProcessorFailures().put(processorId, new ProcessorFailure(2, Instant.now(), Duration.ZERO));
    cpm.onResourceCompleted(new SamzaResourceStatus(container.getContainerId(), "diagnostics", 1));
    assertEquals(false, cpm.getJobFailureCriteriaMet());
    assertEquals(3, cpm.getProcessorFailures().get(processorId).getCount());
    assertFalse(cpm.shouldShutdown());
    if (withHostAffinity) {
        assertEquals(0, allocator.getContainerRequestState().numPendingRequests());
        assertEquals(1, allocator.getContainerRequestState().numDelayedRequests());
    } else {
        assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
        assertEquals(0, allocator.getContainerRequestState().numDelayedRequests());
    }
    cpm.onResourceAllocated(container);
    if (withHostAffinity) {
        if (allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
            // No delayed retry requests for there host affinity is disabled. Call back should return immediately.
            fail("Expecting a delayed request so allocator callback should have timed out waiting for a response.");
        }
        // For the sake of testing the mocked 4th failure below, send delayed requests now.
        SamzaResourceRequest request = allocator.getContainerRequestState().getDelayedRequestsQueue().poll();
        SamzaResourceRequest fastForwardRequest = new SamzaResourceRequest(request.getNumCores(), request.getMemoryMB(), request.getPreferredHost(), request.getProcessorId(), Instant.now().minusSeconds(1));
        allocator.getContainerRequestState().getDelayedRequestsQueue().add(fastForwardRequest);
        int numSent = allocator.getContainerRequestState().sendPendingDelayedResourceRequests();
        assertEquals(1, numSent);
        cpm.onResourceAllocated(container);
    }
    if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
        // No delayed retry requests for there host affinity is disabled. Call back should return immediately.
        fail("Timed out waiting for the containers to start");
    }
    cpm.onStreamProcessorLaunchSuccess(container);
    // Mock 4th failure not exceeding retry window.
    cpm.getProcessorFailures().put(processorId, new ProcessorFailure(3, Instant.now(), Duration.ZERO));
    cpm.onResourceCompleted(new SamzaResourceStatus(container.getContainerId(), "diagnostics", 1));
    // expecting failed container
    assertEquals(failAfterRetries, cpm.getJobFailureCriteriaMet());
    // count won't update on failure
    assertEquals(3, cpm.getProcessorFailures().get(processorId).getCount());
    if (failAfterRetries) {
        assertTrue(cpm.shouldShutdown());
    } else {
        assertFalse(cpm.shouldShutdown());
    }
    assertEquals(0, allocator.getContainerRequestState().numPendingRequests());
    assertEquals(0, allocator.getContainerRequestState().numDelayedRequests());
    cpm.stop();
}

Example 18 with LocalityModel

use of org.apache.samza.job.model.LocalityModel in project samza by apache.

the class ContainerProcessManager method start.

public void start() {
    LOG.info("Starting the container process manager");
    int containerRetryCount = clusterManagerConfig.getContainerRetryCount();
    if (containerRetryCount > -1) {
        LOG.info("Max retries on restarting failed containers: {}", containerRetryCount);
    } else {
        LOG.info("Infinite retries on restarting failed containers");
    }
    if (jvmMetrics != null) {
        jvmMetrics.start();
    }
    if (metricsReporters != null) {
        metricsReporters.values().forEach(reporter -> reporter.start());
    }
    if (diagnosticsManager.isPresent()) {
        diagnosticsManager.get().start();
    }
    // In AM-HA, clusterResourceManager receives already running containers
    // and invokes onStreamProcessorLaunchSuccess which inturn updates state
    // hence state has to be set prior to starting clusterResourceManager.
    state.processorCount.set(state.jobModelManager.jobModel().getContainers().size());
    state.neededProcessors.set(state.jobModelManager.jobModel().getContainers().size());
    LOG.info("Starting the cluster resource manager");
    clusterResourceManager.start();
    // Request initial set of containers
    LocalityModel localityModel = localityManager.readLocality();
    Map<String, String> processorToHost = new HashMap<>();
    state.jobModelManager.jobModel().getContainers().keySet().forEach((processorId) -> {
        String host = Optional.ofNullable(localityModel.getProcessorLocality(processorId)).map(ProcessorLocality::host).filter(StringUtils::isNotBlank).orElse(null);
        processorToHost.put(processorId, host);
    });
    if (jobConfig.getApplicationMasterHighAvailabilityEnabled()) {
        // don't request resource for container that is already running
        state.runningProcessors.forEach((processorId, samzaResource) -> {
            LOG.info("Not requesting container for processorId: {} since its already running as containerId: {}", processorId, samzaResource.getContainerId());
            processorToHost.remove(processorId);
            if (restartContainers) {
                clusterResourceManager.stopStreamProcessor(samzaResource);
            }
        });
    }
    containerAllocator.requestResources(processorToHost);
    // Start container allocator thread
    LOG.info("Starting the container allocator thread");
    allocatorThread.start();
    LOG.info("Starting the container process manager");
}

Example 19 with LocalityModel

use of org.apache.samza.job.model.LocalityModel in project samza by apache.

the class TestLocalityServlet method testReadContainerLocalityWithNoLocality.

@Test
public void testReadContainerLocalityWithNoLocality() throws Exception {
    final LocalityModel expectedLocality = new LocalityModel(Collections.emptyMap());
    URL url = new URL(webApp.getUrl().toString() + "locality");
    when(localityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of()));
    String response = HttpUtil.read(url, 1000, new ExponentialSleepStrategy());
    LocalityModel locality = mapper.readValue(response, LocalityModel.class);
    assertEquals("Expected empty response but got " + locality, locality, expectedLocality);
}