Examples with ContainerPlacementRequestMessage org.apache.samza.container.placement.ContainerPlacementRequestMessage used on opensource projects

Example 1 with ContainerPlacementRequestMessage

use of org.apache.samza.container.placement.ContainerPlacementRequestMessage in project samza by apache.

the class TestContainerPlacementObjectMapper method testContainerPlacementRequestMessage.

private void testContainerPlacementRequestMessage(ContainerPlacementRequestMessage requestMessage) throws IOException {
    ObjectMapper objectMapper = ContainerPlacementMessageObjectMapper.getObjectMapper();
    ContainerPlacementMessage message = objectMapper.readValue(objectMapper.writeValueAsString(requestMessage), ContainerPlacementMessage.class);
    assertTrue(message instanceof ContainerPlacementRequestMessage);
    ContainerPlacementRequestMessage deserializedRequest = (ContainerPlacementRequestMessage) message;
    assertEquals(requestMessage, deserializedRequest);
}

Example 2 with ContainerPlacementRequestMessage

use of org.apache.samza.container.placement.ContainerPlacementRequestMessage in project samza by apache.

the class ContainerPlacementMetadataStore method writeContainerPlacementRequestMessage.

/**
 * Writes a {@link ContainerPlacementRequestMessage} to the underlying metastore. This method should be used by external controllers
 * to issue a request to JobCoordinator
 *
 * @param deploymentId identifier of the deployment
 * @param processorId logical id of the samza container 0,1,2
 * @param destinationHost host where the container is desired to move
 * @param requestExpiry optional per request expiry timeout for requests to cluster manager
 * @param timestamp timestamp of the request
 * @return uuid generated for the request
 */
public UUID writeContainerPlacementRequestMessage(String deploymentId, String processorId, String destinationHost, Duration requestExpiry, long timestamp) {
    Preconditions.checkState(!stopped, "Underlying metadata store not available");
    UUID uuid = UUID.randomUUID();
    ContainerPlacementRequestMessage message = new ContainerPlacementRequestMessage(uuid, deploymentId, processorId, destinationHost, requestExpiry, timestamp);
    try {
        containerPlacementMessageStore.put(toContainerPlacementMessageKey(message.getUuid(), message.getClass()), objectMapper.writeValueAsBytes(message));
        containerPlacementMessageStore.flush();
    } catch (Exception ex) {
        throw new SamzaException(String.format("ContainerPlacementRequestMessage might have been not written to metastore %s", message), ex);
    }
    return uuid;
}

Example 3 with ContainerPlacementRequestMessage

use of org.apache.samza.container.placement.ContainerPlacementRequestMessage in project samza by apache.

the class ContainerPlacementMetadataStore method readAllContainerPlacementRequestMessages.

@VisibleForTesting
List<ContainerPlacementRequestMessage> readAllContainerPlacementRequestMessages() {
    Preconditions.checkState(!stopped, "Underlying metadata store not available");
    List<ContainerPlacementRequestMessage> newActions = new ArrayList<>();
    Map<String, byte[]> messageBytes = containerPlacementMessageStore.all();
    for (byte[] action : messageBytes.values()) {
        try {
            ContainerPlacementMessage message = objectMapper.readValue(action, ContainerPlacementMessage.class);
            if (message instanceof ContainerPlacementRequestMessage) {
                newActions.add((ContainerPlacementRequestMessage) message);
            }
        } catch (IOException e) {
            throw new SamzaException(e);
        }
    }
    // Sort the actions in order of timestamp
    newActions.sort(Comparator.comparingLong(ContainerPlacementRequestMessage::getTimestamp));
    return newActions;
}

Example 4 with ContainerPlacementRequestMessage

use of org.apache.samza.container.placement.ContainerPlacementRequestMessage in project samza by apache.

the class ContainerPlacementRequestAllocator method run.

@Override
public void run() {
    while (isRunning && containerPlacementMetadataStore.isRunning()) {
        try {
            for (ContainerPlacementRequestMessage message : containerPlacementMetadataStore.readAllContainerPlacementRequestMessages()) {
                // in response to a Container Placement Action
                if (message.getDeploymentId().equals(appRunId)) {
                    LOG.debug("Received a container placement message {}", message);
                    containerProcessManager.registerContainerPlacementAction(message);
                } else {
                    // Delete the ContainerPlacementMessages from the previous deployment
                    containerPlacementMetadataStore.deleteAllContainerPlacementMessages(message.getUuid());
                }
            }
            Thread.sleep(containerPlacementHandlerSleepMs);
        } catch (InterruptedException e) {
            LOG.warn("Got InterruptedException in ContainerPlacementRequestAllocator thread.", e);
            Thread.currentThread().interrupt();
        } catch (Exception e) {
            LOG.error("Got an exception while reading ContainerPlacementRequestMessage in ContainerPlacementRequestAllocator thread", e);
        }
    }
}

Example 5 with ContainerPlacementRequestMessage

use of org.apache.samza.container.placement.ContainerPlacementRequestMessage in project samza by apache.

the class TestContainerPlacementActions method testContainerPlacementsForJobRunningInDegradedState.

@Test(timeout = 20000)
public void testContainerPlacementsForJobRunningInDegradedState() throws Exception {
    // Set failure after retries to false to enable job running in degraded state
    config = new MapConfig(configVals, getConfigWithHostAffinityAndRetries(true, 1, false));
    state = new SamzaApplicationState(JobModelManagerTestUtil.getJobModelManager(getConfig(), 2, this.server));
    callback = mock(ClusterResourceManager.Callback.class);
    MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
    containerManager = spy(new ContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, true, false, localityManager, faultDomainManager, config));
    allocatorWithHostAffinity = new MockContainerAllocatorWithHostAffinity(clusterResourceManager, config, state, containerManager);
    cpm = new ContainerProcessManager(clusterManagerConfig, state, new MetricsRegistryMap(), clusterResourceManager, Optional.of(allocatorWithHostAffinity), containerManager, localityManager, false);
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                cpm.onResourcesAvailable((List<SamzaResource>) args[0]);
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onResourcesAvailable(anyList());
    // Mimic stream processor launch failure only on host-2,
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                cpm.onStreamProcessorLaunchSuccess((SamzaResource) args[0]);
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onStreamProcessorLaunchSuccess(any());
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                cpm.onResourcesCompleted((List<SamzaResourceStatus>) args[0]);
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onResourcesCompleted(anyList());
    cpm.start();
    if (!allocatorWithHostAffinity.awaitContainersStart(2, 5, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    while (state.runningProcessors.size() != 2) {
        Thread.sleep(100);
    }
    // App is in running state with two containers running
    assertEquals(state.runningProcessors.size(), 2);
    assertEquals(state.runningProcessors.get("0").getHost(), "host-1");
    assertEquals(state.runningProcessors.get("1").getHost(), "host-2");
    assertEquals(state.preferredHostRequests.get(), 2);
    assertEquals(state.anyHostRequests.get(), 0);
    // Trigger a container failure
    clusterResourceManager.stopStreamProcessor(state.runningProcessors.get("1"), -103);
    // Wait for container to start
    if (!allocatorWithHostAffinity.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    while (state.runningProcessors.size() != 2) {
        Thread.sleep(100);
    }
    // Trigger a container failure again
    clusterResourceManager.stopStreamProcessor(state.runningProcessors.get("1"), -103);
    // Ensure that this container has exhausted all retires
    while (state.failedProcessors.size() != 1 && state.runningProcessors.size() != 1) {
        Thread.sleep(100);
    }
    // At this point the application should only have one container running
    assertEquals(state.runningProcessors.get("0").getHost(), "host-1");
    assertEquals(state.runningProcessors.size(), 1);
    assertEquals(state.pendingProcessors.size(), 0);
    assertTrue(state.failedProcessors.containsKey("1"));
    ContainerPlacementRequestMessage requestMessage = new ContainerPlacementRequestMessage(UUID.randomUUID(), "app-attempt-001", "1", "host-3", System.currentTimeMillis());
    ContainerPlacementMetadata metadata = containerManager.registerContainerPlacementActionForTest(requestMessage, allocatorWithHostAffinity);
    // Wait for the ControlAction to complete
    if (!allocatorWithHostAffinity.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    // Wait for both the containers to be in running state & control action metadata to succeed
    while (state.runningProcessors.size() != 2 && metadata.getActionStatus() != ContainerPlacementMessage.StatusCode.SUCCEEDED) {
        Thread.sleep(100);
    }
    assertEquals(state.preferredHostRequests.get(), 4);
    assertEquals(state.runningProcessors.size(), 2);
    // Container 1 should not go to host-3
    assertEquals(state.runningProcessors.get("0").getHost(), "host-1");
    assertEquals(state.runningProcessors.get("1").getHost(), "host-3");
    assertEquals(state.anyHostRequests.get(), 0);
    // Failed processors must be empty
    assertEquals(state.failedProcessors.size(), 0);
}