Search in sources :

Example 1 with SamzaContainer

use of org.apache.samza.container.SamzaContainer in project samza by apache.

the class ContainerLaunchUtil method run.

@VisibleForTesting
static void run(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc, String jobName, String jobId, String containerId, Optional<String> executionEnvContainerId, Optional<String> samzaEpochId, JobModel jobModel, Config config, Optional<ExternalContext> externalContextOptional) {
    CoordinatorStreamStore coordinatorStreamStore = buildCoordinatorStreamStore(config, new MetricsRegistryMap());
    coordinatorStreamStore.init();
    /*
     * We track the exit code and only trigger exit in the finally block to make sure we are able to execute all the
     * clean up steps. Prior implementation had short circuited exit causing some of the clean up steps to be missed.
     */
    int exitCode = 0;
    try {
        TaskFactory taskFactory = TaskFactoryUtil.getTaskFactory(appDesc);
        LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetContainerHostMapping.TYPE));
        // StartpointManager wraps the coordinatorStreamStore in the namespaces internally
        StartpointManager startpointManager = null;
        if (new JobConfig(config).getStartpointEnabled()) {
            startpointManager = new StartpointManager(coordinatorStreamStore);
        }
        Map<String, MetricsReporter> metricsReporters = loadMetricsReporters(appDesc, containerId, config);
        // Creating diagnostics manager and reporter, and wiring it respectively
        Optional<DiagnosticsManager> diagnosticsManager = DiagnosticsUtil.buildDiagnosticsManager(jobName, jobId, jobModel, containerId, executionEnvContainerId, samzaEpochId, config);
        MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap();
        SamzaContainer container = SamzaContainer$.MODULE$.apply(containerId, jobModel, ScalaJavaUtil.toScalaMap(metricsReporters), metricsRegistryMap, taskFactory, JobContextImpl.fromConfigWithDefaults(config, jobModel), Option.apply(appDesc.getApplicationContainerContextFactory().orElse(null)), Option.apply(appDesc.getApplicationTaskContextFactory().orElse(null)), Option.apply(externalContextOptional.orElse(null)), localityManager, startpointManager, Option.apply(diagnosticsManager.orElse(null)));
        ProcessorLifecycleListener processorLifecycleListener = appDesc.getProcessorLifecycleListenerFactory().createInstance(new ProcessorContext() {
        }, config);
        ClusterBasedProcessorLifecycleListener listener = new ClusterBasedProcessorLifecycleListener(config, processorLifecycleListener, container::shutdown);
        container.setContainerListener(listener);
        ContainerHeartbeatMonitor heartbeatMonitor = createContainerHeartbeatMonitor(container, new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetConfig.TYPE), config);
        if (heartbeatMonitor != null) {
            heartbeatMonitor.start();
        }
        if (new JobConfig(config).getApplicationMasterHighAvailabilityEnabled()) {
            executionEnvContainerId.ifPresent(execEnvContainerId -> {
                ExecutionContainerIdManager executionContainerIdManager = new ExecutionContainerIdManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetExecutionEnvContainerIdMapping.TYPE));
                executionContainerIdManager.writeExecutionEnvironmentContainerIdMapping(containerId, execEnvContainerId);
            });
        }
        container.run();
        if (heartbeatMonitor != null) {
            heartbeatMonitor.stop();
        }
        // overriding the value with what the listener returns
        if (containerRunnerException == null) {
            containerRunnerException = listener.getContainerException();
        }
        if (containerRunnerException != null) {
            log.error("Container stopped with Exception. Exiting process now.", containerRunnerException);
            exitCode = 1;
        }
    } catch (Throwable e) {
        /*
       * Two separate log statements are intended to print the entire stack trace as part of the logs. Using
       * single log statement with custom format requires explicitly fetching stack trace and null checks which makes
       * the code slightly hard to read in comparison with the current choice.
       */
        log.error("Exiting the process due to", e);
        log.error("Container runner exception: ", containerRunnerException);
        exitCode = 1;
    } finally {
        coordinatorStreamStore.close();
        /*
       * Only exit in the scenario of non-zero exit code in order to maintain parity with current implementation where
       * the method completes when no errors are encountered.
       */
        if (exitCode != 0) {
            exitProcess(exitCode);
        }
    }
}
Also used : DiagnosticsManager(org.apache.samza.diagnostics.DiagnosticsManager) ContainerHeartbeatMonitor(org.apache.samza.container.ContainerHeartbeatMonitor) JobConfig(org.apache.samza.config.JobConfig) SamzaContainer(org.apache.samza.container.SamzaContainer) NamespaceAwareCoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore) ExecutionContainerIdManager(org.apache.samza.container.ExecutionContainerIdManager) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) NamespaceAwareCoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore) MetricsReporter(org.apache.samza.metrics.MetricsReporter) TaskFactory(org.apache.samza.task.TaskFactory) StartpointManager(org.apache.samza.startpoint.StartpointManager) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) LocalityManager(org.apache.samza.container.LocalityManager) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 2 with SamzaContainer

use of org.apache.samza.container.SamzaContainer in project samza by apache.

the class TestStreamProcessor method testOnJobModelExpiredShouldMakeCorrectStateTransitions.

@Test
public void testOnJobModelExpiredShouldMakeCorrectStateTransitions() {
    JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
    ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
    SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
    MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
    StreamProcessor streamProcessor = new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, Optional.empty(), Optional.empty(), Optional.empty(), sp -> lifecycleListener, mockJobCoordinator, Mockito.mock(MetadataStore.class));
    /**
     * Without a SamzaContainer running in StreamProcessor and current StreamProcessor state is STARTED,
     * onJobModelExpired should move the state to IN_REBALANCE.
     */
    streamProcessor.start();
    assertEquals(State.STARTED, streamProcessor.getState());
    streamProcessor.jobCoordinatorListener.onJobModelExpired();
    assertEquals(State.IN_REBALANCE, streamProcessor.getState());
    /**
     * When there's initialized SamzaContainer in StreamProcessor and the container shutdown
     * fails in onJobModelExpired. onJobModelExpired should move StreamProcessor to STOPPING
     * state and should shutdown JobCoordinator.
     */
    Mockito.doNothing().when(mockJobCoordinator).start();
    Mockito.doNothing().when(mockJobCoordinator).stop();
    Mockito.doNothing().when(mockSamzaContainer).shutdown();
    Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false);
    Mockito.when(mockSamzaContainer.getStatus()).thenReturn(SamzaContainerStatus.STARTED).thenReturn(SamzaContainerStatus.STOPPED);
    streamProcessor.container = mockSamzaContainer;
    streamProcessor.state = State.STARTED;
    streamProcessor.jobCoordinatorListener.onJobModelExpired();
    assertEquals(State.STOPPING, streamProcessor.getState());
    Mockito.verify(mockSamzaContainer, Mockito.times(1)).shutdown();
    Mockito.verify(mockJobCoordinator, Mockito.times(1)).stop();
}
Also used : MetadataStore(org.apache.samza.metadatastore.MetadataStore) JobCoordinator(org.apache.samza.coordinator.JobCoordinator) MapConfig(org.apache.samza.config.MapConfig) ProcessorLifecycleListener(org.apache.samza.runtime.ProcessorLifecycleListener) SamzaContainer(org.apache.samza.container.SamzaContainer) Test(org.junit.Test)

Example 3 with SamzaContainer

use of org.apache.samza.container.SamzaContainer in project samza by apache.

the class TestStreamProcessor method runJobModelExpireDuringRebalance.

private void runJobModelExpireDuringRebalance(StreamProcessor streamProcessor, ExecutorService executorService, boolean failContainerInterrupt) {
    SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
    CountDownLatch shutdownLatch = new CountDownLatch(1);
    /*
     * When there is an initialized container that hasn't started and the stream processor is still in re-balance phase,
     * subsequent job model expire request should attempt to interrupt the existing container to safely shut it down
     * before proceeding to join the barrier. As part of safe shutdown sequence,  we want to ensure shutdownNow is invoked
     * on the existing executorService to signal interrupt and make sure new executor service is created.
     */
    Mockito.when(executorService.shutdownNow()).thenAnswer(ctx -> {
        if (!failContainerInterrupt) {
            shutdownLatch.countDown();
        }
        return null;
    });
    Mockito.when(executorService.isShutdown()).thenReturn(true);
    streamProcessor.state = State.IN_REBALANCE;
    streamProcessor.container = mockSamzaContainer;
    streamProcessor.containerExecutorService = executorService;
    streamProcessor.containerShutdownLatch = shutdownLatch;
    streamProcessor.jobCoordinatorListener.onJobModelExpired();
}
Also used : CountDownLatch(java.util.concurrent.CountDownLatch) SamzaContainer(org.apache.samza.container.SamzaContainer)

Example 4 with SamzaContainer

use of org.apache.samza.container.SamzaContainer in project samza by apache.

the class WatermarkIntegrationTest method getTaskOperationGraphs.

Map<String, StreamOperatorTask> getTaskOperationGraphs(MockLocalApplicationRunner runner) throws Exception {
    StreamProcessor processor = runner.getProcessors().iterator().next();
    SamzaContainer container = TestStreamProcessorUtil.getContainer(processor);
    Map<TaskName, TaskInstance> taskInstances = JavaConverters.mapAsJavaMapConverter(container.getTaskInstances()).asJava();
    Map<String, StreamOperatorTask> tasks = new HashMap<>();
    for (Map.Entry<TaskName, TaskInstance> entry : taskInstances.entrySet()) {
        StreamOperatorTask task = (StreamOperatorTask) entry.getValue().task();
        tasks.put(entry.getKey().getTaskName(), task);
    }
    return tasks;
}
Also used : TaskInstance(org.apache.samza.container.TaskInstance) StreamProcessor(org.apache.samza.processor.StreamProcessor) TaskName(org.apache.samza.container.TaskName) HashMap(java.util.HashMap) TestStreamOperatorTask(org.apache.samza.task.TestStreamOperatorTask) StreamOperatorTask(org.apache.samza.task.StreamOperatorTask) Map(java.util.Map) HashMap(java.util.HashMap) SamzaContainer(org.apache.samza.container.SamzaContainer)

Example 5 with SamzaContainer

use of org.apache.samza.container.SamzaContainer in project samza by apache.

the class TestStreamProcessor method testCoordinatorFailureShouldStopTheStreamProcessor.

@Test
public void testCoordinatorFailureShouldStopTheStreamProcessor() {
    JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
    ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
    SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
    MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
    StreamProcessor streamProcessor = new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, Optional.empty(), Optional.empty(), Optional.empty(), sp -> lifecycleListener, mockJobCoordinator, Mockito.mock(MetadataStore.class));
    Exception failureException = new Exception("dummy exception");
    streamProcessor.container = mockSamzaContainer;
    streamProcessor.state = State.RUNNING;
    streamProcessor.jobCoordinatorListener.onCoordinatorFailure(failureException);
    Mockito.doNothing().when(mockSamzaContainer).shutdown();
    Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false);
    assertEquals(State.STOPPED, streamProcessor.state);
    Mockito.verify(lifecycleListener).afterFailure(failureException);
    Mockito.verify(mockSamzaContainer).shutdown();
}
Also used : MetadataStore(org.apache.samza.metadatastore.MetadataStore) JobCoordinator(org.apache.samza.coordinator.JobCoordinator) MapConfig(org.apache.samza.config.MapConfig) ProcessorLifecycleListener(org.apache.samza.runtime.ProcessorLifecycleListener) SamzaContainer(org.apache.samza.container.SamzaContainer) TimeoutException(java.util.concurrent.TimeoutException) SamzaException(org.apache.samza.SamzaException) Test(org.junit.Test)

Aggregations

SamzaContainer (org.apache.samza.container.SamzaContainer)8 MapConfig (org.apache.samza.config.MapConfig)5 JobCoordinator (org.apache.samza.coordinator.JobCoordinator)5 ProcessorLifecycleListener (org.apache.samza.runtime.ProcessorLifecycleListener)5 Test (org.junit.Test)5 HashMap (java.util.HashMap)3 CountDownLatch (java.util.concurrent.CountDownLatch)3 SamzaException (org.apache.samza.SamzaException)3 MetadataStore (org.apache.samza.metadatastore.MetadataStore)3 Map (java.util.Map)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 TimeoutException (java.util.concurrent.TimeoutException)2 AtomicReference (java.util.concurrent.atomic.AtomicReference)2 RunLoop (org.apache.samza.container.RunLoop)2 JobModel (org.apache.samza.job.model.JobModel)2 MetricsReporter (org.apache.samza.metrics.MetricsReporter)2 StreamTask (org.apache.samza.task.StreamTask)2 StreamTaskFactory (org.apache.samza.task.StreamTaskFactory)2 TaskFactory (org.apache.samza.task.TaskFactory)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1