Examples with ProcessorLifecycleListener org.apache.samza.runtime.ProcessorLifecycleListener used on opensource projects

Example 1 with ProcessorLifecycleListener

use of org.apache.samza.runtime.ProcessorLifecycleListener in project samza by apache.

the class TestStreamProcessor method testJobModelExpiredDuringAnExistingRebalanceWithContainerInterruptFailed.

@Test
public void testJobModelExpiredDuringAnExistingRebalanceWithContainerInterruptFailed() {
    JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
    ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
    ExecutorService mockExecutorService = Mockito.mock(ExecutorService.class);
    MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
    StreamProcessor streamProcessor = new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, Optional.empty(), Optional.empty(), Optional.empty(), sp -> lifecycleListener, mockJobCoordinator, Mockito.mock(MetadataStore.class));
    runJobModelExpireDuringRebalance(streamProcessor, mockExecutorService, true);
    assertEquals(State.STOPPING, streamProcessor.state);
    assertEquals(mockExecutorService, streamProcessor.containerExecutorService);
    Mockito.verify(mockExecutorService, Mockito.times(1)).shutdownNow();
    Mockito.verify(mockJobCoordinator, Mockito.times(1)).stop();
}

Example 2 with ProcessorLifecycleListener

use of org.apache.samza.runtime.ProcessorLifecycleListener in project samza by apache.

the class TestStreamProcessor method testOnJobModelExpiredShouldMakeCorrectStateTransitions.

@Test
public void testOnJobModelExpiredShouldMakeCorrectStateTransitions() {
    JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
    ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
    SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
    MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
    StreamProcessor streamProcessor = new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, Optional.empty(), Optional.empty(), Optional.empty(), sp -> lifecycleListener, mockJobCoordinator, Mockito.mock(MetadataStore.class));
    /**
     * Without a SamzaContainer running in StreamProcessor and current StreamProcessor state is STARTED,
     * onJobModelExpired should move the state to IN_REBALANCE.
     */
    streamProcessor.start();
    assertEquals(State.STARTED, streamProcessor.getState());
    streamProcessor.jobCoordinatorListener.onJobModelExpired();
    assertEquals(State.IN_REBALANCE, streamProcessor.getState());
    /**
     * When there's initialized SamzaContainer in StreamProcessor and the container shutdown
     * fails in onJobModelExpired. onJobModelExpired should move StreamProcessor to STOPPING
     * state and should shutdown JobCoordinator.
     */
    Mockito.doNothing().when(mockJobCoordinator).start();
    Mockito.doNothing().when(mockJobCoordinator).stop();
    Mockito.doNothing().when(mockSamzaContainer).shutdown();
    Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false);
    Mockito.when(mockSamzaContainer.getStatus()).thenReturn(SamzaContainerStatus.STARTED).thenReturn(SamzaContainerStatus.STOPPED);
    streamProcessor.container = mockSamzaContainer;
    streamProcessor.state = State.STARTED;
    streamProcessor.jobCoordinatorListener.onJobModelExpired();
    assertEquals(State.STOPPING, streamProcessor.getState());
    Mockito.verify(mockSamzaContainer, Mockito.times(1)).shutdown();
    Mockito.verify(mockJobCoordinator, Mockito.times(1)).stop();
}

Example 3 with ProcessorLifecycleListener

use of org.apache.samza.runtime.ProcessorLifecycleListener in project samza by apache.

the class TestStreamProcessor method testJobModelExpiredDuringAnExistingRebalance.

@Test
public void testJobModelExpiredDuringAnExistingRebalance() {
    JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
    ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
    ExecutorService mockExecutorService = Mockito.mock(ExecutorService.class);
    MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
    StreamProcessor streamProcessor = new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, Optional.empty(), Optional.empty(), Optional.empty(), sp -> lifecycleListener, mockJobCoordinator, Mockito.mock(MetadataStore.class));
    runJobModelExpireDuringRebalance(streamProcessor, mockExecutorService, false);
    assertEquals(State.IN_REBALANCE, streamProcessor.state);
    assertNotEquals(mockExecutorService, streamProcessor.containerExecutorService);
    Mockito.verify(mockExecutorService, Mockito.times(1)).shutdownNow();
}

Example 4 with ProcessorLifecycleListener

use of org.apache.samza.runtime.ProcessorLifecycleListener in project samza by apache.

the class TestStreamProcessor method testJobModelExpiredContainerShutdownTimeout.

/**
 * Given that the job model expires, but the container takes too long to stop, a TimeoutException should be propagated
 * to the processor lifecycle listener.
 */
@Test
public void testJobModelExpiredContainerShutdownTimeout() throws InterruptedException {
    JobCoordinator mockJobCoordinator = mock(JobCoordinator.class);
    // use this to store the exception passed to afterFailure for the processor lifecycle listener
    AtomicReference<Throwable> afterFailureException = new AtomicReference<>(null);
    TestableStreamProcessor processor = new TestableStreamProcessor(// set a small shutdown timeout so it triggers faster
    new MapConfig(ImmutableMap.of(TaskConfig.TASK_SHUTDOWN_MS, "1")), new HashMap<>(), mock(StreamTaskFactory.class), new ProcessorLifecycleListener() {

        @Override
        public void beforeStart() {
        }

        @Override
        public void afterStart() {
        }

        @Override
        public void afterFailure(Throwable t) {
            afterFailureException.set(t);
        }

        @Override
        public void afterStop() {
        }
    }, mockJobCoordinator, null, // take an extra second to shut down so that task shutdown timeout gets reached
    Duration.of(1, ChronoUnit.SECONDS));
    Thread jcThread = new Thread(() -> {
        // gets processor into rebalance mode so onNewJobModel creates a new container
        processor.jobCoordinatorListener.onJobModelExpired();
        processor.jobCoordinatorListener.onNewJobModel("1", getMockJobModel());
        try {
            // wait for the run loop to be ready before triggering rebalance
            processor.runLoopStartForMain.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        processor.jobCoordinatorListener.onJobModelExpired();
    });
    doAnswer(invocation -> {
        jcThread.start();
        return null;
    }).when(mockJobCoordinator).start();
    // ensure that the coordinator stop occurred before checking the exception being thrown
    CountDownLatch coordinatorStop = new CountDownLatch(1);
    doAnswer(invocation -> {
        processor.jobCoordinatorListener.onCoordinatorStop();
        coordinatorStop.countDown();
        return null;
    }).when(mockJobCoordinator).stop();
    processor.start();
    // make sure the job model expired callback completed
    assertTrue("Job coordinator stop not called", coordinatorStop.await(10, TimeUnit.SECONDS));
    assertNotNull(afterFailureException.get());
    assertTrue(afterFailureException.get() instanceof TimeoutException);
}

Example 5 with ProcessorLifecycleListener

use of org.apache.samza.runtime.ProcessorLifecycleListener in project samza by apache.

the class TestZkStreamProcessorBase method createStreamProcessor.

protected StreamProcessor createStreamProcessor(final String pId, Map<String, String> map, final CountDownLatch waitStart, final CountDownLatch waitStop) {
    map.put(ApplicationConfig.PROCESSOR_ID, pId);
    Config config = new MapConfig(map);
    String jobCoordinatorFactoryClassName = new JobCoordinatorConfig(config).getJobCoordinatorFactoryClassName();
    JobCoordinator jobCoordinator = ReflectionUtil.getObj(jobCoordinatorFactoryClassName, JobCoordinatorFactory.class).getJobCoordinator(pId, config, new MetricsRegistryMap(), Mockito.mock(CoordinatorStreamStore.class));
    ProcessorLifecycleListener listener = new ProcessorLifecycleListener() {

        @Override
        public void beforeStart() {
        }

        @Override
        public void afterStart() {
            if (waitStart != null) {
                waitStart.countDown();
            }
            LOG.info("onStart is called for pid=" + pId);
        }

        @Override
        public void afterStop() {
            // stopped w/o failure
            if (waitStop != null) {
                waitStop.countDown();
            }
            LOG.info("afterStop is called for pid=" + pId + " with successful shutdown");
        }

        @Override
        public void afterFailure(Throwable t) {
            // stopped w/ failure
            LOG.info("afterStop is called for pid=" + pId + " with failure");
        }
    };
    StreamProcessor processor = new StreamProcessor(pId, config, new HashMap<>(), (StreamTaskFactory) TestStreamTask::new, listener, jobCoordinator);
    return processor;
}