Examples with ContainerPlacementResponseMessage org.apache.samza.container.placement.ContainerPlacementResponseMessage used on opensource projects

Example 1 with ContainerPlacementResponseMessage

use of org.apache.samza.container.placement.ContainerPlacementResponseMessage in project samza by apache.

the class TestContainerPlacementObjectMapper method testContainerPlacementResponseMessage.

private void testContainerPlacementResponseMessage(ContainerPlacementResponseMessage responseMessage) throws IOException {
    ObjectMapper objectMapper = ContainerPlacementMessageObjectMapper.getObjectMapper();
    ContainerPlacementMessage message = objectMapper.readValue(objectMapper.writeValueAsString(responseMessage), ContainerPlacementMessage.class);
    assertTrue(message instanceof ContainerPlacementResponseMessage);
    ContainerPlacementResponseMessage deserializedResponse = (ContainerPlacementResponseMessage) message;
    assertEquals(responseMessage, deserializedResponse);
}

Example 2 with ContainerPlacementResponseMessage

use of org.apache.samza.container.placement.ContainerPlacementResponseMessage in project samza by apache.

the class ContainerPlacementMetadataStore method readContainerPlacementResponseMessage.

/**
 * Reads a {@link ContainerPlacementResponseMessage} from the underlying metastore
 * @param uuid uuid of the response message
 * @return ContainerPlacementResponseMessage if present
 */
public Optional<ContainerPlacementResponseMessage> readContainerPlacementResponseMessage(UUID uuid) {
    Preconditions.checkState(!stopped, "Underlying metadata store not available");
    Preconditions.checkNotNull(uuid, "uuid cannot be null");
    byte[] messageBytes = containerPlacementMessageStore.get(toContainerPlacementMessageKey(uuid, ContainerPlacementResponseMessage.class));
    if (ArrayUtils.isNotEmpty(messageBytes)) {
        try {
            ContainerPlacementResponseMessage requestMessage = (ContainerPlacementResponseMessage) objectMapper.readValue(messageBytes, ContainerPlacementMessage.class);
            return Optional.of(requestMessage);
        } catch (IOException e) {
            throw new SamzaException(String.format("Error reading the ContainerPlacementResponseMessage for uuid: %s", uuid), e);
        }
    }
    return Optional.empty();
}

Example 3 with ContainerPlacementResponseMessage

use of org.apache.samza.container.placement.ContainerPlacementResponseMessage in project samza by apache.

the class TestContainerPlacementActions method assertBadRequests.

private void assertBadRequests(String processorId, String destinationHost, ContainerManager containerManager, ContainerAllocator allocator) throws InterruptedException {
    ContainerPlacementRequestMessage requestMessage = new ContainerPlacementRequestMessage(UUID.randomUUID(), "app-Attemp-001", processorId, destinationHost, System.currentTimeMillis());
    ContainerPlacementMetadata metadata = containerManager.registerContainerPlacementActionForTest(requestMessage, allocator);
    assertNull(metadata);
    Optional<ContainerPlacementResponseMessage> responseMessage = containerPlacementMetadataStore.readContainerPlacementResponseMessage(requestMessage.getUuid());
    while (true) {
        if (responseMessage.isPresent() && responseMessage.get().getStatusCode() == ContainerPlacementMessage.StatusCode.BAD_REQUEST) {
            break;
        }
        Thread.sleep(100);
        responseMessage = containerPlacementMetadataStore.readContainerPlacementResponseMessage(requestMessage.getUuid());
    }
    assertEquals(responseMessage.get().getStatusCode(), ContainerPlacementMessage.StatusCode.BAD_REQUEST);
    assertResponseMessage(responseMessage.get(), requestMessage);
    // Request shall be deleted as soon as it is acted upon
    assertFalse(containerPlacementMetadataStore.readContainerPlacementRequestMessage(requestMessage.getUuid()).isPresent());
}

Example 4 with ContainerPlacementResponseMessage

use of org.apache.samza.container.placement.ContainerPlacementResponseMessage in project samza by apache.

the class TestContainerPlacementActions method testActiveContainerLaunchFailureOnControlActionShouldFallbackToSourceHost.

@Test(timeout = 10000)
public void testActiveContainerLaunchFailureOnControlActionShouldFallbackToSourceHost() throws Exception {
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                cpm.onResourcesAvailable((List<SamzaResource>) args[0]);
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onResourcesAvailable(anyList());
    // Mimic stream processor launch failure only on host-3
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                SamzaResource host3Resource = (SamzaResource) args[0];
                if (host3Resource.getHost().equals("host-3")) {
                    cpm.onStreamProcessorLaunchFailure(host3Resource, new Throwable("Custom Exception for Host-3"));
                } else {
                    cpm.onStreamProcessorLaunchSuccess((SamzaResource) args[0]);
                }
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onStreamProcessorLaunchSuccess(any());
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                cpm.onResourcesCompleted((List<SamzaResourceStatus>) args[0]);
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onResourcesCompleted(anyList());
    cpm.start();
    if (!allocatorWithHostAffinity.awaitContainersStart(2, 5, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    while (state.runningProcessors.size() != 2) {
        Thread.sleep(100);
    }
    // App is in running state with two containers running
    assertEquals(state.runningProcessors.size(), 2);
    assertEquals(state.runningProcessors.get("0").getHost(), "host-1");
    assertEquals(state.runningProcessors.get("1").getHost(), "host-2");
    assertEquals(state.preferredHostRequests.get(), 2);
    assertEquals(state.anyHostRequests.get(), 0);
    // Take a container placement action to move a container with container id 0
    ContainerPlacementRequestMessage requestMessage = new ContainerPlacementRequestMessage(UUID.randomUUID(), "app-attempt-001", "0", "host-3", System.currentTimeMillis());
    ContainerPlacementMetadata metadata = containerManager.registerContainerPlacementActionForTest(requestMessage, allocatorWithHostAffinity);
    // Wait for the ControlAction to complete
    if (!allocatorWithHostAffinity.awaitContainersStart(1, 3, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    while (state.runningProcessors.size() != 2) {
        Thread.sleep(100);
    }
    assertEquals(state.preferredHostRequests.get(), 4);
    assertEquals(state.runningProcessors.size(), 2);
    // Container 0 should fallback to source host
    assertEquals(state.runningProcessors.get("0").getHost(), "host-1");
    assertEquals(state.runningProcessors.get("1").getHost(), "host-2");
    assertEquals(state.anyHostRequests.get(), 0);
    // Control Action should be failed in this case
    assertEquals(metadata.getActionStatus(), ContainerPlacementMessage.StatusCode.FAILED);
    Optional<ContainerPlacementResponseMessage> responseMessage = containerPlacementMetadataStore.readContainerPlacementResponseMessage(requestMessage.getUuid());
    assertTrue(responseMessage.isPresent());
    assertEquals(responseMessage.get().getStatusCode(), ContainerPlacementMessage.StatusCode.FAILED);
    assertResponseMessage(responseMessage.get(), requestMessage);
    // Request shall be deleted as soon as it is acted upon
    assertFalse(containerPlacementMetadataStore.readContainerPlacementRequestMessage(requestMessage.getUuid()).isPresent());
}

Example 5 with ContainerPlacementResponseMessage

use of org.apache.samza.container.placement.ContainerPlacementResponseMessage in project samza by apache.

the class TestContainerPlacementActions method testContainerSuccessfulMoveActionWithStandbyEnabled.

@Test(timeout = 30000)
public void testContainerSuccessfulMoveActionWithStandbyEnabled() throws Exception {
    // Setup standby for job
    setupStandby();
    // Spawn a Request Allocator Thread
    ContainerPlacementRequestAllocator requestAllocator = new ContainerPlacementRequestAllocator(containerPlacementMetadataStore, cpm, new ApplicationConfig(config), 100);
    Thread requestAllocatorThread = new Thread(requestAllocator, "ContainerPlacement Request Allocator Thread");
    requestAllocatorThread.start();
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                cpm.onResourcesAvailable((List<SamzaResource>) args[0]);
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onResourcesAvailable(anyList());
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                cpm.onStreamProcessorLaunchSuccess((SamzaResource) args[0]);
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onStreamProcessorLaunchSuccess(any());
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                cpm.onResourcesCompleted((List<SamzaResourceStatus>) args[0]);
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onResourcesCompleted(anyList());
    cpm.start();
    if (!allocatorWithHostAffinity.awaitContainersStart(4, 4, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    while (state.runningProcessors.size() != 4) {
        Thread.sleep(100);
    }
    // First running state of the app
    Consumer<SamzaApplicationState> stateCheck = (SamzaApplicationState state) -> {
        assertEquals(4, state.runningProcessors.size());
        assertEquals("host-1", state.runningProcessors.get("0").getHost());
        assertEquals("host-2", state.runningProcessors.get("1").getHost());
        assertEquals("host-2", state.runningProcessors.get("0-0").getHost());
        assertEquals("host-1", state.runningProcessors.get("1-0").getHost());
        assertEquals(4, state.preferredHostRequests.get());
        assertEquals(0, state.failedStandbyAllocations.get());
        assertEquals(0, state.anyHostRequests.get());
    };
    // Invoke a state check
    stateCheck.accept(state);
    // Initiate a bad container placement action to move a standby to its active host and vice versa
    // which should fail because this violates standby constraints
    UUID badRequest1 = containerPlacementMetadataStore.writeContainerPlacementRequestMessage("appAttempt-001", "0-0", "host-1", null, System.currentTimeMillis());
    UUID badRequest2 = containerPlacementMetadataStore.writeContainerPlacementRequestMessage("appAttempt-001", "0", "host-2", null, System.currentTimeMillis() + 100);
    // Wait for the ControlActions to complete
    while (true) {
        if (containerPlacementMetadataStore.readContainerPlacementResponseMessage(badRequest2).isPresent() && containerPlacementMetadataStore.readContainerPlacementResponseMessage(badRequest2).get().getStatusCode() == ContainerPlacementMessage.StatusCode.BAD_REQUEST) {
            break;
        }
        Thread.sleep(100);
    }
    // App running state should remain the same
    stateCheck.accept(state);
    Optional<ContainerPlacementResponseMessage> responseMessageMove1 = containerPlacementMetadataStore.readContainerPlacementResponseMessage(badRequest1);
    Optional<ContainerPlacementResponseMessage> responseMessageMove2 = containerPlacementMetadataStore.readContainerPlacementResponseMessage(badRequest2);
    // Assert that both the requests were bad
    assertTrue(responseMessageMove1.isPresent());
    assertEquals(responseMessageMove1.get().getStatusCode(), ContainerPlacementMessage.StatusCode.BAD_REQUEST);
    assertTrue(responseMessageMove2.isPresent());
    assertEquals(responseMessageMove2.get().getStatusCode(), ContainerPlacementMessage.StatusCode.BAD_REQUEST);
    // Initiate a standby failover which is supposed to be done in two steps
    // Step 1. Move the standby container to any other host: move 0-0 to say host-3
    // Step 2. Move the active container to the standby's host: move 0 to host-1
    // Action will get executed first
    UUID standbyMoveRequest = containerPlacementMetadataStore.writeContainerPlacementRequestMessage("appAttempt-001", "0-0", "host-3", null, System.currentTimeMillis());
    // Action will get executed when standbyMoveRequest move request is complete
    UUID activeMoveRequest = containerPlacementMetadataStore.writeContainerPlacementRequestMessage("appAttempt-001", "0", "host-2", null, System.currentTimeMillis() + 100);
    // Wait for the ControlActions to complete
    while (true) {
        if (containerPlacementMetadataStore.readContainerPlacementResponseMessage(activeMoveRequest).isPresent() && containerPlacementMetadataStore.readContainerPlacementResponseMessage(activeMoveRequest).get().getStatusCode() == ContainerPlacementMessage.StatusCode.SUCCEEDED) {
            break;
        }
        Thread.sleep(100);
    }
    assertEquals(4, state.runningProcessors.size());
    assertEquals("host-2", state.runningProcessors.get("0").getHost());
    assertEquals("host-2", state.runningProcessors.get("1").getHost());
    assertEquals("host-3", state.runningProcessors.get("0-0").getHost());
    assertEquals("host-1", state.runningProcessors.get("1-0").getHost());
    assertEquals(6, state.preferredHostRequests.get());
    assertEquals(0, state.failedStandbyAllocations.get());
    assertEquals(0, state.anyHostRequests.get());
    Optional<ContainerPlacementResponseMessage> responseStandbyMove = containerPlacementMetadataStore.readContainerPlacementResponseMessage(standbyMoveRequest);
    Optional<ContainerPlacementResponseMessage> responseActiveMove = containerPlacementMetadataStore.readContainerPlacementResponseMessage(activeMoveRequest);
    assertTrue(responseStandbyMove.isPresent());
    assertEquals(responseStandbyMove.get().getStatusCode(), ContainerPlacementMessage.StatusCode.SUCCEEDED);
    assertTrue(responseActiveMove.isPresent());
    assertEquals(responseActiveMove.get().getStatusCode(), ContainerPlacementMessage.StatusCode.SUCCEEDED);
    // Request should be deleted as soon as ita accepted / being acted upon
    assertFalse(containerPlacementMetadataStore.readContainerPlacementRequestMessage(standbyMoveRequest).isPresent());
    assertFalse(containerPlacementMetadataStore.readContainerPlacementRequestMessage(activeMoveRequest).isPresent());
    // Cleanup Request Allocator Thread
    cleanUpRequestAllocatorThread(requestAllocator, requestAllocatorThread);
}