Examples with LocalityModel org.apache.samza.job.model.LocalityModel used on opensource projects

Example 11 with LocalityModel

use of org.apache.samza.job.model.LocalityModel in project samza by apache.

the class YarnJobValidationTool method validateJmxMetrics.

public void validateJmxMetrics() throws Exception {
    MetricsRegistry metricsRegistry = new MetricsRegistryMap();
    CoordinatorStreamStore coordinatorStreamStore = new CoordinatorStreamStore(config, metricsRegistry);
    coordinatorStreamStore.init();
    try {
        LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetConfig.TYPE));
        validator.init(config);
        LocalityModel localityModel = localityManager.readLocality();
        for (ProcessorLocality processorLocality : localityModel.getProcessorLocalities().values()) {
            String containerId = processorLocality.id();
            String jmxUrl = processorLocality.jmxTunnelingUrl();
            if (StringUtils.isNotBlank(jmxUrl)) {
                log.info("validate container " + containerId + " metrics with JMX: " + jmxUrl);
                JmxMetricsAccessor jmxMetrics = new JmxMetricsAccessor(jmxUrl);
                jmxMetrics.connect();
                validator.validate(jmxMetrics);
                jmxMetrics.close();
                log.info("validate container " + containerId + " successfully");
            }
        }
        validator.complete();
    } finally {
        coordinatorStreamStore.close();
    }
}

Example 12 with LocalityModel

use of org.apache.samza.job.model.LocalityModel in project samza by apache.

the class LocalityServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
    response.setContentType("application/json");
    response.setStatus(HttpServletResponse.SC_OK);
    LocalityModel localityModel = localityManager.readLocality();
    if (request.getParameterMap().size() == 1) {
        String processorId = request.getParameter(PROCESSOR_ID_PARAM);
        ProcessorLocality processorLocality = Optional.ofNullable(localityModel.getProcessorLocality(processorId)).orElse(new ProcessorLocality(processorId, ""));
        mapper.writeValue(response.getWriter(), processorLocality);
    } else {
        mapper.writeValue(response.getWriter(), localityModel);
    }
}

Example 13 with LocalityModel

use of org.apache.samza.job.model.LocalityModel in project samza by apache.

the class TestContainerAllocatorWithoutHostAffinity method setup.

@Before
public void setup() throws Exception {
    LocalityManager mockLocalityManager = mock(LocalityManager.class);
    when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(new HashMap<>()));
    CoordinatorStreamStoreTestUtil coordinatorStreamStoreTestUtil = new CoordinatorStreamStoreTestUtil(config);
    CoordinatorStreamStore coordinatorStreamStore = coordinatorStreamStoreTestUtil.getCoordinatorStreamStore();
    coordinatorStreamStore.init();
    containerPlacementMetadataStore = new ContainerPlacementMetadataStore(coordinatorStreamStore);
    containerPlacementMetadataStore.start();
    containerAllocator = new ContainerAllocator(manager, config, state, false, new ContainerManager(containerPlacementMetadataStore, state, manager, false, false, mockLocalityManager, faultDomainManager, config));
    requestState = new MockContainerRequestState(manager, false);
    Field requestStateField = containerAllocator.getClass().getDeclaredField("resourceRequestState");
    requestStateField.setAccessible(true);
    requestStateField.set(containerAllocator, requestState);
    allocatorThread = new Thread(containerAllocator);
}

Example 14 with LocalityModel

use of org.apache.samza.job.model.LocalityModel in project samza by apache.

the class TestContainerAllocatorWithoutHostAffinity method testExpiredRequestInfiniteLoop.

/**
 * See SAMZA-2601: we want to prevent an infinite loop in the case of expired request call with host affinity
 * disabled. This test make sure we don't have that infinite loop.
 */
@Test
public void testExpiredRequestInfiniteLoop() throws Exception {
    Config override = new MapConfig(new HashMap<String, String>() {

        {
            // override to have a proper sleep interval for this test
            put("cluster-manager.allocator.sleep.ms", "100");
        }
    });
    LocalityManager mockLocalityManager = mock(LocalityManager.class);
    when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(new HashMap<>()));
    ContainerManager containerManager = new ContainerManager(containerPlacementMetadataStore, state, manager, false, false, mockLocalityManager, faultDomainManager, config);
    containerAllocator = MockContainerAllocatorWithoutHostAffinity.createContainerAllocatorWithConfigOverride(manager, config, state, containerManager, override);
    MockContainerAllocatorWithoutHostAffinity mockAllocator = (MockContainerAllocatorWithoutHostAffinity) containerAllocator;
    mockAllocator.setOverrideIsRequestExpired();
    allocatorThread = new Thread(containerAllocator);
    Map<String, String> containersToHostMapping = new HashMap<String, String>() {

        {
            put("0", null);
            put("1", null);
            put("2", null);
            put("3", null);
        }
    };
    allocatorThread.start();
    mockAllocator.requestResources(containersToHostMapping);
    // Wait for at least one expired request call is made, which should happen.
    // If the test passes, this should return immediately (within 100 ms). Only when the test fails will it exhaust the
    // timeout, which is worth the wait to find out the failure
    assertTrue(mockAllocator.awaitIsRequestExpiredCall(TimeUnit.SECONDS.toMillis(10)));
    // TODO: we can eliminate the thread sleep if the whole container allocator and test codes are refactored to use
    // a Clock which can be simulated and controlled.
    Thread.sleep(500);
    // Given that we wait for 500 ms above, and a sleep interval of 100 ms, we should roughly see 5 times the
    // isRequestExpired is called. We give some extra buffer here (<100). Because if we do run into infinite loop,
    // isRequestExpired would be called MILLIONS of times (4~5 million times after a dozen of runs on my machine).
    assertTrue(String.format("Too many call count: %d. Seems to be in infinite loop", mockAllocator.getExpiredRequestCallCount()), mockAllocator.getExpiredRequestCallCount() < 100);
}

Example 15 with LocalityModel

use of org.apache.samza.job.model.LocalityModel in project samza by apache.

the class TestContainerProcessManager method buildContainerProcessManager.

private ContainerProcessManager buildContainerProcessManager(ClusterManagerConfig clusterManagerConfig, SamzaApplicationState state, ClusterResourceManager clusterResourceManager, Optional<ContainerAllocator> allocator, boolean restartContainer) {
    LocalityManager mockLocalityManager = mock(LocalityManager.class);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(new HashMap<>()));
    return buildContainerProcessManager(clusterManagerConfig, state, clusterResourceManager, allocator, mockLocalityManager, restartContainer, faultDomainManager);
}