Examples with MetricsRegistryMap org.apache.samza.metrics.MetricsRegistryMap used on opensource projects

Example 6 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestContainerProcessManager method testAppMasterWithFwk.

@Test
public void testAppMasterWithFwk() {
    ContainerProcessManager taskManager = new ContainerProcessManager(new MapConfig(config), state, new MetricsRegistryMap(), manager);
    taskManager.start();
    SamzaResource container2 = new SamzaResource(1, 1024, "", "id0");
    assertFalse(taskManager.shouldShutdown());
    taskManager.onResourceAllocated(container2);
    configVals.put(JobConfig.SAMZA_FWK_PATH(), "/export/content/whatever");
    Config config1 = new MapConfig(configVals);
    ContainerProcessManager taskManager1 = new ContainerProcessManager(new MapConfig(config), state, new MetricsRegistryMap(), manager);
    taskManager1.start();
    taskManager1.onResourceAllocated(container2);
}

Example 7 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestContainerProcessManager method testTaskManagerShouldStopWhenContainersFinish.

/**
   * Test Task Manager should stop when all containers finish
   */
@Test
public void testTaskManagerShouldStopWhenContainersFinish() {
    Config conf = getConfig();
    ContainerProcessManager taskManager = new ContainerProcessManager(new MapConfig(conf), state, new MetricsRegistryMap(), manager);
    taskManager.start();
    assertFalse(taskManager.shouldShutdown());
    taskManager.onResourceCompleted(new SamzaResourceStatus("123", "diagnostics", SamzaResourceStatus.SUCCESS));
    assertTrue(taskManager.shouldShutdown());
}

Example 8 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestContainerProcessManager method testContainerProcessManager.

@Test
public void testContainerProcessManager() throws Exception {
    Map<String, String> conf = new HashMap<>();
    conf.putAll(getConfig());
    conf.put("yarn.container.memory.mb", "500");
    conf.put("yarn.container.cpu.cores", "5");
    ContainerProcessManager taskManager = new ContainerProcessManager(new MapConfig(conf), state, new MetricsRegistryMap(), manager);
    AbstractContainerAllocator allocator = (AbstractContainerAllocator) getPrivateFieldFromTaskManager("containerAllocator", taskManager).get(taskManager);
    assertEquals(ContainerAllocator.class, allocator.getClass());
    // Asserts that samza exposed container configs is honored by allocator thread
    assertEquals(500, allocator.containerMemoryMb);
    assertEquals(5, allocator.containerNumCpuCores);
    conf.clear();
    conf.putAll(getConfigWithHostAffinity());
    conf.put("yarn.container.memory.mb", "500");
    conf.put("yarn.container.cpu.cores", "5");
    taskManager = new ContainerProcessManager(new MapConfig(conf), state, new MetricsRegistryMap(), manager);
    allocator = (AbstractContainerAllocator) getPrivateFieldFromTaskManager("containerAllocator", taskManager).get(taskManager);
    assertEquals(HostAwareContainerAllocator.class, allocator.getClass());
    // Asserts that samza exposed container configs is honored by allocator thread
    assertEquals(500, allocator.containerMemoryMb);
    assertEquals(5, allocator.containerNumCpuCores);
}

Example 9 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestContainerProcessManager method testNewContainerRequestedOnFailureWithUnknownCode.

/**
   * Test Task Manager should request a new container when a task fails with unknown exit code
   * When host-affinity is not enabled, it will always request for ANY_HOST
   */
@Test
public void testNewContainerRequestedOnFailureWithUnknownCode() throws Exception {
    Config conf = getConfig();
    ContainerProcessManager taskManager = new ContainerProcessManager(new MapConfig(conf), state, new MetricsRegistryMap(), manager);
    MockContainerAllocator allocator = new MockContainerAllocator(manager, conf, state);
    getPrivateFieldFromTaskManager("containerAllocator", taskManager).set(taskManager, allocator);
    Thread thread = new Thread(allocator);
    getPrivateFieldFromTaskManager("allocatorThread", taskManager).set(taskManager, thread);
    // start triggers a request
    taskManager.start();
    assertFalse(taskManager.shouldShutdown());
    assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
    SamzaResource container = new SamzaResource(1, 1024, "abc", "id0");
    taskManager.onResourceAllocated(container);
    // Allow container to run and update state
    Thread.sleep(300);
    // Create first container failure
    taskManager.onResourceCompleted(new SamzaResourceStatus(container.getResourceID(), "diagnostics", 1));
    // The above failure should trigger a container request
    assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
    assertEquals(ResourceRequestState.ANY_HOST, allocator.getContainerRequestState().peekPendingRequest().getPreferredHost());
    assertFalse(taskManager.shouldShutdown());
    assertFalse(state.jobHealthy.get());
    assertEquals(2, manager.resourceRequests.size());
    assertEquals(0, manager.releasedResources.size());
    taskManager.onResourceAllocated(container);
    // Allow container to run and update state
    Thread.sleep(1000);
    assertTrue(state.jobHealthy.get());
    // Create a second failure
    taskManager.onResourceCompleted(new SamzaResourceStatus(container.getResourceID(), "diagnostics", 1));
    // The above failure should trigger a job shutdown because our retry count is set to 1
    assertEquals(0, allocator.getContainerRequestState().numPendingRequests());
    assertEquals(2, manager.resourceRequests.size());
    assertEquals(0, manager.releasedResources.size());
    assertFalse(state.jobHealthy.get());
    assertTrue(taskManager.shouldShutdown());
    assertEquals(SamzaApplicationState.SamzaAppStatus.FAILED, state.status);
    taskManager.stop();
}

Example 10 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestContainerProcessManager method testOnShutdown.

@Test
public void testOnShutdown() throws Exception {
    Config conf = getConfig();
    ContainerProcessManager taskManager = new ContainerProcessManager(new MapConfig(conf), state, new MetricsRegistryMap(), manager);
    taskManager.start();
    Thread.sleep(100);
    Thread allocatorThread = (Thread) getPrivateFieldFromTaskManager("allocatorThread", taskManager).get(taskManager);
    assertTrue(allocatorThread.isAlive());
    taskManager.stop();
    Thread.sleep(100);
    assertFalse(allocatorThread.isAlive());
}