Examples with JobCoordinatorListener org.apache.samza.coordinator.JobCoordinatorListener used on opensource projects

Example 1 with JobCoordinatorListener

use of org.apache.samza.coordinator.JobCoordinatorListener in project samza by apache.

the class StreamProcessor method createJobCoordinatorListener.

JobCoordinatorListener createJobCoordinatorListener() {
    return new JobCoordinatorListener() {

        @Override
        public void onJobModelExpired() {
            if (container != null) {
                SamzaContainerStatus status = container.getStatus();
                if (SamzaContainerStatus.NOT_STARTED.equals(status) || SamzaContainerStatus.STARTED.equals(status)) {
                    boolean shutdownComplete = false;
                    try {
                        LOGGER.info("Shutting down container in onJobModelExpired.");
                        container.pause();
                        shutdownComplete = jcContainerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS);
                    } catch (IllegalContainerStateException icse) {
                        // Ignored since container is not running
                        LOGGER.info("Container was not running.", icse);
                        shutdownComplete = true;
                    } catch (InterruptedException e) {
                        LOGGER.warn("Container shutdown was interrupted!" + container.toString(), e);
                    }
                    if (!shutdownComplete) {
                        LOGGER.warn("Container " + container.toString() + " may not have shutdown successfully. " + "Stopping the processor.");
                        container = null;
                        stop();
                    } else {
                        LOGGER.debug("Container " + container.toString() + " shutdown successfully");
                    }
                } else {
                    LOGGER.debug("Container " + container.toString() + " is not running.");
                }
            } else {
                LOGGER.debug("Container is not instantiated yet.");
            }
        }

        @Override
        public void onNewJobModel(String processorId, JobModel jobModel) {
            if (!jobModel.getContainers().containsKey(processorId)) {
                LOGGER.warn("JobModel does not contain the processorId: " + processorId + ". Stopping the processor.");
                stop();
            } else {
                jcContainerShutdownLatch = new CountDownLatch(1);
                SamzaContainerListener containerListener = new SamzaContainerListener() {

                    @Override
                    public void onContainerStart() {
                        if (!processorOnStartCalled) {
                            // processorListener is called on start only the first time the container starts.
                            // It is not called after every re-balance of partitions among the processors
                            processorOnStartCalled = true;
                            if (processorListener != null) {
                                processorListener.onStart();
                            }
                        } else {
                            LOGGER.debug("StreamProcessorListener was notified of container start previously. Hence, skipping this time.");
                        }
                    }

                    @Override
                    public void onContainerStop(boolean pauseByJm) {
                        if (pauseByJm) {
                            LOGGER.info("Container " + container.toString() + " stopped due to a request from JobCoordinator.");
                            if (jcContainerShutdownLatch != null) {
                                jcContainerShutdownLatch.countDown();
                            }
                        } else {
                            // sp.stop was called or container stopped by itself
                            LOGGER.info("Container " + container.toString() + " stopped.");
                            // this guarantees that stop() doesn't try to stop container again
                            container = null;
                            stop();
                        }
                    }

                    @Override
                    public void onContainerFailed(Throwable t) {
                        if (jcContainerShutdownLatch != null) {
                            jcContainerShutdownLatch.countDown();
                        } else {
                            LOGGER.warn("JobCoordinatorLatch was null. It is possible for some component to be waiting.");
                        }
                        LOGGER.error("Container failed. Stopping the processor.", t);
                        container = null;
                        stop();
                    }
                };
                container = createSamzaContainer(jobModel.getContainers().get(processorId), jobModel.maxChangeLogStreamPartitions);
                container.setContainerListener(containerListener);
                LOGGER.info("Starting container " + container.toString());
                executorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("p-" + processorId + "-container-thread-%d").build());
                executorService.submit(container::run);
            }
        }

        @Override
        public void onCoordinatorStop() {
            if (executorService != null) {
                LOGGER.info("Shutting down the executor service.");
                executorService.shutdownNow();
            }
            if (processorListener != null) {
                processorListener.onShutdown();
            }
        }

        @Override
        public void onCoordinatorFailure(Throwable e) {
            LOGGER.info("Coordinator Failed. Stopping the processor.");
            stop();
            if (processorListener != null) {
                processorListener.onFailure(e);
            }
        }
    };
}