Examples with SamzaResource org.apache.samza.clustermanager.SamzaResource used on opensource projects

Example 1 with SamzaResource

use of org.apache.samza.clustermanager.SamzaResource in project samza by apache.

the class TestApplicationMasterRestClient method assignMetricValues.

private void assignMetricValues(SamzaApplicationState samzaAppState, MetricsRegistryMap registry) {
    SamzaAppMasterMetrics metrics = new SamzaAppMasterMetrics(new MapConfig(), samzaAppState, registry);
    metrics.start();
    samzaAppState.runningProcessors.put("dummyContainer", // 1 container
    new SamzaResource(1, 2, AM_HOST_NAME, "dummyResourceId"));
    samzaAppState.neededProcessors.set(2);
    samzaAppState.completedProcessors.set(3);
    samzaAppState.failedContainers.set(4);
    samzaAppState.releasedContainers.set(5);
    samzaAppState.processorCount.set(6);
    samzaAppState.jobHealthy.set(true);
}

Example 2 with SamzaResource

use of org.apache.samza.clustermanager.SamzaResource in project samza by apache.

the class TestYarnClusterResourceManager method testAllocatedResourceExpiryForYarn.

@Test
public void testAllocatedResourceExpiryForYarn() {
    // start the cluster manager
    YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient, callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config);
    SamzaResource allocatedResource = mock(SamzaResource.class);
    when(allocatedResource.getTimestamp()).thenReturn(System.currentTimeMillis() - Duration.ofMinutes(10).toMillis());
    Assert.assertTrue(yarnClusterResourceManager.isResourceExpired(allocatedResource));
}

Example 3 with SamzaResource

use of org.apache.samza.clustermanager.SamzaResource in project samza by apache.

the class TestYarnClusterResourceManager method testStopStreamProcessorForContainerStartedInCurrentLifecycle.

@Test
public void testStopStreamProcessorForContainerStartedInCurrentLifecycle() {
    YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient, callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config);
    SamzaResource allocatedContainerResource = mock(SamzaResource.class);
    Container runningContainer = mock(Container.class);
    ContainerId runningContainerId = mock(ContainerId.class);
    NodeId runningNodeId = mock(NodeId.class);
    when(runningContainer.getId()).thenReturn(runningContainerId);
    when(runningContainer.getNodeId()).thenReturn(runningNodeId);
    yarnClusterResourceManager.getAllocatedResources().put(allocatedContainerResource, runningContainer);
    yarnClusterResourceManager.stopStreamProcessor(allocatedContainerResource);
    verify(asyncNMClient, times(1)).stopContainerAsync(runningContainerId, runningNodeId);
}

Example 4 with SamzaResource

use of org.apache.samza.clustermanager.SamzaResource in project samza by apache.

the class TestYarnClusterResourceManager method testAMHACallbackInvokedForPreviousAttemptContainers.

@Test
public void testAMHACallbackInvokedForPreviousAttemptContainers() {
    String previousAttemptContainerId = "0";
    String previousAttemptYarnContainerId = "container_1607304997422_0008_02_000002";
    // create mocks
    YarnAppState yarnAppState = Mockito.spy(new YarnAppState(0, mock(ContainerId.class), "host", 8080, 8081));
    ContainerId containerId = mock(ContainerId.class);
    when(containerId.toString()).thenReturn(previousAttemptYarnContainerId);
    YarnContainer yarnContainer = mock(YarnContainer.class);
    Resource resource = mock(Resource.class);
    when(resource.getMemory()).thenReturn(1024);
    Mockito.when(resource.getVirtualCores()).thenReturn(1);
    Mockito.when(yarnContainer.resource()).thenReturn(resource);
    Mockito.when(yarnContainer.id()).thenReturn(containerId);
    NodeId nodeId = mock(NodeId.class);
    when(nodeId.getHost()).thenReturn("host");
    when(yarnContainer.nodeId()).thenReturn(nodeId);
    yarnAppState.pendingProcessors.put(previousAttemptContainerId, yarnContainer);
    Set<ContainerId> previousAttemptContainers = new HashSet<>();
    previousAttemptContainers.add(containerId);
    when(lifecycle.onInit()).thenReturn(previousAttemptContainers);
    Map<String, String> configMap = new HashMap<>();
    configMap.put(JobConfig.YARN_AM_HIGH_AVAILABILITY_ENABLED, "true");
    Config config = new MapConfig(configMap);
    // start the cluster manager
    YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient, callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config);
    yarnClusterResourceManager.start();
    verify(lifecycle).onInit();
    ArgumentCaptor<SamzaResource> samzaResourceArgumentCaptor = ArgumentCaptor.forClass(SamzaResource.class);
    verify(callback).onStreamProcessorLaunchSuccess(samzaResourceArgumentCaptor.capture());
    ArgumentCaptor<Integer> containerFromPreviousAttemptCaptor = ArgumentCaptor.forClass(Integer.class);
    verify(metrics).setContainersFromPreviousAttempts(containerFromPreviousAttemptCaptor.capture());
    SamzaResource samzaResource = samzaResourceArgumentCaptor.getValue();
    assertEquals(previousAttemptYarnContainerId, samzaResource.getContainerId());
    assertEquals(1, containerFromPreviousAttemptCaptor.getValue().intValue());
}

Example 5 with SamzaResource

use of org.apache.samza.clustermanager.SamzaResource in project samza by apache.

the class TestYarnClusterResourceManager method testStopStreamProcessorForContainerFromPreviousAttempt.

@Test
public void testStopStreamProcessorForContainerFromPreviousAttempt() {
    String containerId = "Yarn_Container_id_0";
    String processorId = "Container_id_0";
    YarnContainer runningYarnContainer = mock(YarnContainer.class);
    ContainerId previousRunningContainerId = mock(ContainerId.class);
    YarnAppState yarnAppState = Mockito.spy(new YarnAppState(0, mock(ContainerId.class), "host", 8080, 8081));
    yarnAppState.runningProcessors.put(processorId, runningYarnContainer);
    when(runningYarnContainer.id()).thenReturn(previousRunningContainerId);
    when(previousRunningContainerId.toString()).thenReturn(containerId);
    YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient, callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config);
    SamzaResource containerResourceFromPreviousRun = mock(SamzaResource.class);
    when(containerResourceFromPreviousRun.getContainerId()).thenReturn(containerId);
    yarnClusterResourceManager.stopStreamProcessor(containerResourceFromPreviousRun);
    verify(asyncClient, times(1)).releaseAssignedContainer(previousRunningContainerId);
}