Examples with JobModel org.apache.samza.job.model.JobModel used on opensource projects

Example 11 with JobModel

use of org.apache.samza.job.model.JobModel in project samza by apache.

the class TestStreamProcessor method testStopByProcessor.

/**
   * Tests stop() method when Container AND JobCoordinator are running
   */
@Test
public void testStopByProcessor() {
    JobCoordinator mockJobCoordinator = mock(JobCoordinator.class);
    final CountDownLatch processorListenerStop = new CountDownLatch(1);
    final CountDownLatch processorListenerStart = new CountDownLatch(1);
    TestableStreamProcessor processor = new TestableStreamProcessor(new MapConfig(), new HashMap<>(), mock(StreamTaskFactory.class), new StreamProcessorLifecycleListener() {

        @Override
        public void onStart() {
            processorListenerStart.countDown();
        }

        @Override
        public void onShutdown() {
            processorListenerStop.countDown();
        }

        @Override
        public void onFailure(Throwable t) {
        }
    }, mockJobCoordinator);
    Map containers = mock(Map.class);
    doReturn(true).when(containers).containsKey(anyString());
    when(containers.get(anyString())).thenReturn(mock(ContainerModel.class));
    JobModel mockJobModel = mock(JobModel.class);
    when(mockJobModel.getContainers()).thenReturn(containers);
    final CountDownLatch coordinatorStop = new CountDownLatch(1);
    final Thread jcThread = new Thread(() -> {
        try {
            processor.jobCoordinatorListener.onNewJobModel("1", mockJobModel);
            coordinatorStop.await();
            processor.jobCoordinatorListener.onCoordinatorStop();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    });
    doAnswer(invocation -> {
        coordinatorStop.countDown();
        return null;
    }).when(mockJobCoordinator).stop();
    doAnswer(invocation -> {
        jcThread.start();
        return null;
    }).when(mockJobCoordinator).start();
    try {
        processor.start();
        processorListenerStart.await();
        Assert.assertEquals(SamzaContainerStatus.STARTED, processor.containerReference.getStatus());
        // This block is required for the mockRunloop is actually start.
        // Otherwise, processor.stop gets triggered before mockRunloop begins to block
        processor.runLoopStartForMain.await();
        processor.stop();
        processorListenerStop.await();
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}

Example 12 with JobModel

use of org.apache.samza.job.model.JobModel in project samza by apache.

the class StreamProcessor method createJobCoordinatorListener.

JobCoordinatorListener createJobCoordinatorListener() {
    return new JobCoordinatorListener() {

        @Override
        public void onJobModelExpired() {
            if (container != null) {
                SamzaContainerStatus status = container.getStatus();
                if (SamzaContainerStatus.NOT_STARTED.equals(status) || SamzaContainerStatus.STARTED.equals(status)) {
                    boolean shutdownComplete = false;
                    try {
                        LOGGER.info("Shutting down container in onJobModelExpired.");
                        container.pause();
                        shutdownComplete = jcContainerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS);
                    } catch (IllegalContainerStateException icse) {
                        // Ignored since container is not running
                        LOGGER.info("Container was not running.", icse);
                        shutdownComplete = true;
                    } catch (InterruptedException e) {
                        LOGGER.warn("Container shutdown was interrupted!" + container.toString(), e);
                    }
                    if (!shutdownComplete) {
                        LOGGER.warn("Container " + container.toString() + " may not have shutdown successfully. " + "Stopping the processor.");
                        container = null;
                        stop();
                    } else {
                        LOGGER.debug("Container " + container.toString() + " shutdown successfully");
                    }
                } else {
                    LOGGER.debug("Container " + container.toString() + " is not running.");
                }
            } else {
                LOGGER.debug("Container is not instantiated yet.");
            }
        }

        @Override
        public void onNewJobModel(String processorId, JobModel jobModel) {
            if (!jobModel.getContainers().containsKey(processorId)) {
                LOGGER.warn("JobModel does not contain the processorId: " + processorId + ". Stopping the processor.");
                stop();
            } else {
                jcContainerShutdownLatch = new CountDownLatch(1);
                SamzaContainerListener containerListener = new SamzaContainerListener() {

                    @Override
                    public void onContainerStart() {
                        if (!processorOnStartCalled) {
                            // processorListener is called on start only the first time the container starts.
                            // It is not called after every re-balance of partitions among the processors
                            processorOnStartCalled = true;
                            if (processorListener != null) {
                                processorListener.onStart();
                            }
                        } else {
                            LOGGER.debug("StreamProcessorListener was notified of container start previously. Hence, skipping this time.");
                        }
                    }

                    @Override
                    public void onContainerStop(boolean pauseByJm) {
                        if (pauseByJm) {
                            LOGGER.info("Container " + container.toString() + " stopped due to a request from JobCoordinator.");
                            if (jcContainerShutdownLatch != null) {
                                jcContainerShutdownLatch.countDown();
                            }
                        } else {
                            // sp.stop was called or container stopped by itself
                            LOGGER.info("Container " + container.toString() + " stopped.");
                            // this guarantees that stop() doesn't try to stop container again
                            container = null;
                            stop();
                        }
                    }

                    @Override
                    public void onContainerFailed(Throwable t) {
                        if (jcContainerShutdownLatch != null) {
                            jcContainerShutdownLatch.countDown();
                        } else {
                            LOGGER.warn("JobCoordinatorLatch was null. It is possible for some component to be waiting.");
                        }
                        LOGGER.error("Container failed. Stopping the processor.", t);
                        container = null;
                        stop();
                    }
                };
                container = createSamzaContainer(jobModel.getContainers().get(processorId), jobModel.maxChangeLogStreamPartitions);
                container.setContainerListener(containerListener);
                LOGGER.info("Starting container " + container.toString());
                executorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("p-" + processorId + "-container-thread-%d").build());
                executorService.submit(container::run);
            }
        }

        @Override
        public void onCoordinatorStop() {
            if (executorService != null) {
                LOGGER.info("Shutting down the executor service.");
                executorService.shutdownNow();
            }
            if (processorListener != null) {
                processorListener.onShutdown();
            }
        }

        @Override
        public void onCoordinatorFailure(Throwable e) {
            LOGGER.info("Coordinator Failed. Stopping the processor.");
            stop();
            if (processorListener != null) {
                processorListener.onFailure(e);
            }
        }
    };
}

Example 13 with JobModel

use of org.apache.samza.job.model.JobModel in project samza by apache.

the class StorageRecovery method getContainerModels.

/**
   * build the ContainerModels from job config file and put the results in the
   * map
   */
private void getContainerModels() {
    JobModel jobModel = JobModelManager.apply(jobConfig).jobModel();
    containers = jobModel.getContainers();
}

Example 14 with JobModel

use of org.apache.samza.job.model.JobModel in project samza by apache.

the class SamzaTaskProxy method getTasks.

/**
   * Fetches the complete job model from the coordinator stream based upon the provided {@link JobInstance}
   * param, transforms it to a list of {@link Task} and returns it.
   * {@inheritDoc}
   */
@Override
public List<Task> getTasks(JobInstance jobInstance) throws IOException, InterruptedException {
    Preconditions.checkArgument(installFinder.isInstalled(jobInstance), String.format("Invalid job instance : %s", jobInstance));
    JobModel jobModel = getJobModel(jobInstance);
    StorageConfig storageConfig = new StorageConfig(jobModel.getConfig());
    List<String> storeNames = JavaConverters.seqAsJavaListConverter(storageConfig.getStoreNames()).asJava();
    Map<String, String> containerLocality = jobModel.getAllContainerLocality();
    List<Task> tasks = new ArrayList<>();
    for (ContainerModel containerModel : jobModel.getContainers().values()) {
        String containerId = containerModel.getProcessorId();
        String host = containerLocality.get(containerId);
        for (TaskModel taskModel : containerModel.getTasks().values()) {
            String taskName = taskModel.getTaskName().getTaskName();
            List<Partition> partitions = taskModel.getSystemStreamPartitions().stream().map(Partition::new).collect(Collectors.toList());
            tasks.add(new Task(host, taskName, containerId, partitions, storeNames));
        }
    }
    return tasks;
}

Example 15 with JobModel

use of org.apache.samza.job.model.JobModel in project samza by apache.

the class LocalContainerRunner method main.

public static void main(String[] args) throws Exception {
    Thread.setDefaultUncaughtExceptionHandler(new SamzaContainerExceptionHandler(() -> {
        log.info("Exiting process now.");
        System.exit(1);
    }));
    String containerId = System.getenv(ShellCommandConfig.ENV_CONTAINER_ID());
    log.info(String.format("Got container ID: %s", containerId));
    String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL());
    log.info(String.format("Got coordinator URL: %s", coordinatorUrl));
    int delay = new Random().nextInt(SamzaContainer.DEFAULT_READ_JOBMODEL_DELAY_MS()) + 1;
    JobModel jobModel = SamzaContainer.readJobModel(coordinatorUrl, delay);
    Config config = jobModel.getConfig();
    JobConfig jobConfig = new JobConfig(config);
    if (jobConfig.getName().isEmpty()) {
        throw new SamzaException("can not find the job name");
    }
    String jobName = jobConfig.getName().get();
    String jobId = jobConfig.getJobId().getOrElse(ScalaToJavaUtils.defaultValue("1"));
    MDC.put("containerName", "samza-container-" + containerId);
    MDC.put("jobName", jobName);
    MDC.put("jobId", jobId);
    StreamApplication streamApp = TaskFactoryUtil.createStreamApplication(config);
    LocalContainerRunner localContainerRunner = new LocalContainerRunner(jobModel, containerId);
    localContainerRunner.run(streamApp);
}