Search in sources :

Example 6 with SamzaContainer

use of org.apache.samza.container.SamzaContainer in project samza by apache.

the class TestStreamProcessor method testStopShouldBeIdempotent.

@Test
public void testStopShouldBeIdempotent() {
    JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
    ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
    SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
    MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
    StreamProcessor streamProcessor = PowerMockito.spy(new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, Optional.empty(), Optional.empty(), Optional.empty(), sp -> lifecycleListener, mockJobCoordinator, Mockito.mock(MetadataStore.class)));
    Mockito.doNothing().when(mockJobCoordinator).stop();
    Mockito.doNothing().when(mockSamzaContainer).shutdown();
    Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false);
    Mockito.when(mockSamzaContainer.getStatus()).thenReturn(SamzaContainerStatus.STARTED).thenReturn(SamzaContainerStatus.STOPPED);
    streamProcessor.state = State.RUNNING;
    streamProcessor.stop();
    assertEquals(State.STOPPING, streamProcessor.state);
}
Also used : SamzaContainer(org.apache.samza.container.SamzaContainer) JobCoordinator(org.apache.samza.coordinator.JobCoordinator) TimeoutException(java.util.concurrent.TimeoutException) HashMap(java.util.HashMap) TaskFactory(org.apache.samza.task.TaskFactory) AtomicReference(java.util.concurrent.atomic.AtomicReference) Matchers.anyString(org.mockito.Matchers.anyString) ConcurrentMap(java.util.concurrent.ConcurrentMap) MetadataStore(org.apache.samza.metadatastore.MetadataStore) State(org.apache.samza.processor.StreamProcessor.State) Duration(java.time.Duration) Map(java.util.Map) MetricsReporter(org.apache.samza.metrics.MetricsReporter) StreamTask(org.apache.samza.task.StreamTask) After(org.junit.After) Mockito.doAnswer(org.mockito.Mockito.doAnswer) RunLoop(org.apache.samza.container.RunLoop) MapConfig(org.apache.samza.config.MapConfig) PowerMockito(org.powermock.api.mockito.PowerMockito) Mockito.doReturn(org.mockito.Mockito.doReturn) ExecutorService(java.util.concurrent.ExecutorService) JobModel(org.apache.samza.job.model.JobModel) Before(org.junit.Before) ImmutableMap(com.google.common.collect.ImmutableMap) TaskConfig(org.apache.samza.config.TaskConfig) Assert.assertNotNull(org.junit.Assert.assertNotNull) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) StreamTaskFactory(org.apache.samza.task.StreamTaskFactory) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) SamzaContainerStatus(org.apache.samza.container.SamzaContainerStatus) SamzaException(org.apache.samza.SamzaException) ProcessorLifecycleListener(org.apache.samza.runtime.ProcessorLifecycleListener) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) TimeUnit(java.util.concurrent.TimeUnit) Matchers.any(org.mockito.Matchers.any) CountDownLatch(java.util.concurrent.CountDownLatch) Mockito(org.mockito.Mockito) ChronoUnit(java.time.temporal.ChronoUnit) ContainerModel(org.apache.samza.job.model.ContainerModel) Optional(java.util.Optional) Config(org.apache.samza.config.Config) Assert(org.junit.Assert) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.mock(org.mockito.Mockito.mock) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) JobCoordinator(org.apache.samza.coordinator.JobCoordinator) MapConfig(org.apache.samza.config.MapConfig) ProcessorLifecycleListener(org.apache.samza.runtime.ProcessorLifecycleListener) SamzaContainer(org.apache.samza.container.SamzaContainer) Test(org.junit.Test)

Example 7 with SamzaContainer

use of org.apache.samza.container.SamzaContainer in project samza by apache.

the class TestStreamProcessor method testContainerFailureCorrectlyStopsProcessor.

/**
 * Tests that a failure in container correctly stops a running JobCoordinator and propagates the exception
 * through the StreamProcessor
 *
 * Assertions:
 * - JobCoordinator has been stopped from the JobCoordinatorListener callback
 * - ProcessorLifecycleListener#afterStop(Throwable) has been invoked w/ non-null Throwable
 */
@Test
public void testContainerFailureCorrectlyStopsProcessor() throws InterruptedException {
    JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
    Throwable expectedThrowable = new SamzaException("Failure in Container!");
    AtomicReference<Throwable> actualThrowable = new AtomicReference<>();
    final CountDownLatch runLoopStartedLatch = new CountDownLatch(1);
    RunLoop failingRunLoop = mock(RunLoop.class);
    doAnswer(invocation -> {
        try {
            runLoopStartedLatch.countDown();
            throw expectedThrowable;
        } catch (InterruptedException ie) {
            ie.printStackTrace();
        }
        return null;
    }).when(failingRunLoop).run();
    SamzaContainer mockContainer = StreamProcessorTestUtils.getDummyContainer(failingRunLoop, mock(StreamTask.class));
    final CountDownLatch processorListenerFailed = new CountDownLatch(1);
    TestableStreamProcessor processor = new TestableStreamProcessor(new MapConfig(), new HashMap<>(), mock(StreamTaskFactory.class), new ProcessorLifecycleListener() {

        @Override
        public void beforeStart() {
            processorListenerState.put(ListenerCallback.BEFORE_START, true);
        }

        @Override
        public void afterStart() {
            processorListenerState.put(ListenerCallback.AFTER_START, true);
        }

        @Override
        public void afterStop() {
            processorListenerState.put(ListenerCallback.AFTER_STOP, true);
        }

        @Override
        public void afterFailure(Throwable t) {
            processorListenerState.put(ListenerCallback.AFTER_FAILURE, true);
            actualThrowable.getAndSet(t);
            processorListenerFailed.countDown();
        }
    }, mockJobCoordinator, mockContainer);
    final CountDownLatch coordinatorStop = new CountDownLatch(1);
    doAnswer(invocation -> {
        coordinatorStop.countDown();
        return null;
    }).when(mockJobCoordinator).stop();
    doAnswer(invocation -> {
        new Thread(() -> {
            try {
                processor.jobCoordinatorListener.onJobModelExpired();
                processor.jobCoordinatorListener.onNewJobModel("1", getMockJobModel());
                coordinatorStop.await();
                processor.jobCoordinatorListener.onCoordinatorStop();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }).start();
        return null;
    }).when(mockJobCoordinator).start();
    processor.start();
    // This block is required for the mockRunloop is actually started.
    // Otherwise, processor.stop gets triggered before mockRunloop begins to block
    runLoopStartedLatch.await();
    assertTrue("Container failed and processor listener failed was not invoked within timeout!", processorListenerFailed.await(30, TimeUnit.SECONDS));
    assertEquals(expectedThrowable, actualThrowable.get());
    assertTrue(processorListenerState.get(ListenerCallback.BEFORE_START));
    assertTrue(processorListenerState.get(ListenerCallback.AFTER_START));
    Assert.assertFalse(processorListenerState.get(ListenerCallback.AFTER_STOP));
    assertTrue(processorListenerState.get(ListenerCallback.AFTER_FAILURE));
}
Also used : AtomicReference(java.util.concurrent.atomic.AtomicReference) CountDownLatch(java.util.concurrent.CountDownLatch) ProcessorLifecycleListener(org.apache.samza.runtime.ProcessorLifecycleListener) SamzaException(org.apache.samza.SamzaException) StreamTaskFactory(org.apache.samza.task.StreamTaskFactory) SamzaContainer(org.apache.samza.container.SamzaContainer) RunLoop(org.apache.samza.container.RunLoop) JobCoordinator(org.apache.samza.coordinator.JobCoordinator) MapConfig(org.apache.samza.config.MapConfig) StreamTask(org.apache.samza.task.StreamTask) Test(org.junit.Test)

Example 8 with SamzaContainer

use of org.apache.samza.container.SamzaContainer in project samza by apache.

the class TestStreamProcessor method testOnNewJobModelShouldResultInValidStateTransitions.

@Test
public void testOnNewJobModelShouldResultInValidStateTransitions() throws Exception {
    JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
    ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
    SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
    MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
    StreamProcessor streamProcessor = new TestableStreamProcessor(config, new HashMap<>(), null, lifecycleListener, mockJobCoordinator, mockSamzaContainer);
    streamProcessor.state = State.IN_REBALANCE;
    Mockito.doNothing().when(mockSamzaContainer).run();
    streamProcessor.jobCoordinatorListener.onNewJobModel("TestProcessorId", new JobModel(new MapConfig(), new HashMap<>()));
    Mockito.verify(mockSamzaContainer, Mockito.times(1)).setContainerListener(any());
    Mockito.verify(mockSamzaContainer, Mockito.atMost(1)).run();
}
Also used : HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) JobCoordinator(org.apache.samza.coordinator.JobCoordinator) JobModel(org.apache.samza.job.model.JobModel) MapConfig(org.apache.samza.config.MapConfig) ProcessorLifecycleListener(org.apache.samza.runtime.ProcessorLifecycleListener) SamzaContainer(org.apache.samza.container.SamzaContainer) Test(org.junit.Test)

Aggregations

SamzaContainer (org.apache.samza.container.SamzaContainer)8 MapConfig (org.apache.samza.config.MapConfig)5 JobCoordinator (org.apache.samza.coordinator.JobCoordinator)5 ProcessorLifecycleListener (org.apache.samza.runtime.ProcessorLifecycleListener)5 Test (org.junit.Test)5 HashMap (java.util.HashMap)3 CountDownLatch (java.util.concurrent.CountDownLatch)3 SamzaException (org.apache.samza.SamzaException)3 MetadataStore (org.apache.samza.metadatastore.MetadataStore)3 Map (java.util.Map)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 TimeoutException (java.util.concurrent.TimeoutException)2 AtomicReference (java.util.concurrent.atomic.AtomicReference)2 RunLoop (org.apache.samza.container.RunLoop)2 JobModel (org.apache.samza.job.model.JobModel)2 MetricsReporter (org.apache.samza.metrics.MetricsReporter)2 StreamTask (org.apache.samza.task.StreamTask)2 StreamTaskFactory (org.apache.samza.task.StreamTaskFactory)2 TaskFactory (org.apache.samza.task.TaskFactory)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1