use of org.apache.samza.container.SamzaContainer in project samza by apache.
the class ContainerLaunchUtil method run.
@VisibleForTesting
static void run(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc, String jobName, String jobId, String containerId, Optional<String> executionEnvContainerId, Optional<String> samzaEpochId, JobModel jobModel, Config config, Optional<ExternalContext> externalContextOptional) {
CoordinatorStreamStore coordinatorStreamStore = buildCoordinatorStreamStore(config, new MetricsRegistryMap());
coordinatorStreamStore.init();
/*
* We track the exit code and only trigger exit in the finally block to make sure we are able to execute all the
* clean up steps. Prior implementation had short circuited exit causing some of the clean up steps to be missed.
*/
int exitCode = 0;
try {
TaskFactory taskFactory = TaskFactoryUtil.getTaskFactory(appDesc);
LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetContainerHostMapping.TYPE));
// StartpointManager wraps the coordinatorStreamStore in the namespaces internally
StartpointManager startpointManager = null;
if (new JobConfig(config).getStartpointEnabled()) {
startpointManager = new StartpointManager(coordinatorStreamStore);
}
Map<String, MetricsReporter> metricsReporters = loadMetricsReporters(appDesc, containerId, config);
// Creating diagnostics manager and reporter, and wiring it respectively
Optional<DiagnosticsManager> diagnosticsManager = DiagnosticsUtil.buildDiagnosticsManager(jobName, jobId, jobModel, containerId, executionEnvContainerId, samzaEpochId, config);
MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap();
SamzaContainer container = SamzaContainer$.MODULE$.apply(containerId, jobModel, ScalaJavaUtil.toScalaMap(metricsReporters), metricsRegistryMap, taskFactory, JobContextImpl.fromConfigWithDefaults(config, jobModel), Option.apply(appDesc.getApplicationContainerContextFactory().orElse(null)), Option.apply(appDesc.getApplicationTaskContextFactory().orElse(null)), Option.apply(externalContextOptional.orElse(null)), localityManager, startpointManager, Option.apply(diagnosticsManager.orElse(null)));
ProcessorLifecycleListener processorLifecycleListener = appDesc.getProcessorLifecycleListenerFactory().createInstance(new ProcessorContext() {
}, config);
ClusterBasedProcessorLifecycleListener listener = new ClusterBasedProcessorLifecycleListener(config, processorLifecycleListener, container::shutdown);
container.setContainerListener(listener);
ContainerHeartbeatMonitor heartbeatMonitor = createContainerHeartbeatMonitor(container, new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetConfig.TYPE), config);
if (heartbeatMonitor != null) {
heartbeatMonitor.start();
}
if (new JobConfig(config).getApplicationMasterHighAvailabilityEnabled()) {
executionEnvContainerId.ifPresent(execEnvContainerId -> {
ExecutionContainerIdManager executionContainerIdManager = new ExecutionContainerIdManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetExecutionEnvContainerIdMapping.TYPE));
executionContainerIdManager.writeExecutionEnvironmentContainerIdMapping(containerId, execEnvContainerId);
});
}
container.run();
if (heartbeatMonitor != null) {
heartbeatMonitor.stop();
}
// overriding the value with what the listener returns
if (containerRunnerException == null) {
containerRunnerException = listener.getContainerException();
}
if (containerRunnerException != null) {
log.error("Container stopped with Exception. Exiting process now.", containerRunnerException);
exitCode = 1;
}
} catch (Throwable e) {
/*
* Two separate log statements are intended to print the entire stack trace as part of the logs. Using
* single log statement with custom format requires explicitly fetching stack trace and null checks which makes
* the code slightly hard to read in comparison with the current choice.
*/
log.error("Exiting the process due to", e);
log.error("Container runner exception: ", containerRunnerException);
exitCode = 1;
} finally {
coordinatorStreamStore.close();
/*
* Only exit in the scenario of non-zero exit code in order to maintain parity with current implementation where
* the method completes when no errors are encountered.
*/
if (exitCode != 0) {
exitProcess(exitCode);
}
}
}
use of org.apache.samza.container.SamzaContainer in project samza by apache.
the class TestStreamProcessor method testOnJobModelExpiredShouldMakeCorrectStateTransitions.
@Test
public void testOnJobModelExpiredShouldMakeCorrectStateTransitions() {
JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
StreamProcessor streamProcessor = new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, Optional.empty(), Optional.empty(), Optional.empty(), sp -> lifecycleListener, mockJobCoordinator, Mockito.mock(MetadataStore.class));
/**
* Without a SamzaContainer running in StreamProcessor and current StreamProcessor state is STARTED,
* onJobModelExpired should move the state to IN_REBALANCE.
*/
streamProcessor.start();
assertEquals(State.STARTED, streamProcessor.getState());
streamProcessor.jobCoordinatorListener.onJobModelExpired();
assertEquals(State.IN_REBALANCE, streamProcessor.getState());
/**
* When there's initialized SamzaContainer in StreamProcessor and the container shutdown
* fails in onJobModelExpired. onJobModelExpired should move StreamProcessor to STOPPING
* state and should shutdown JobCoordinator.
*/
Mockito.doNothing().when(mockJobCoordinator).start();
Mockito.doNothing().when(mockJobCoordinator).stop();
Mockito.doNothing().when(mockSamzaContainer).shutdown();
Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false);
Mockito.when(mockSamzaContainer.getStatus()).thenReturn(SamzaContainerStatus.STARTED).thenReturn(SamzaContainerStatus.STOPPED);
streamProcessor.container = mockSamzaContainer;
streamProcessor.state = State.STARTED;
streamProcessor.jobCoordinatorListener.onJobModelExpired();
assertEquals(State.STOPPING, streamProcessor.getState());
Mockito.verify(mockSamzaContainer, Mockito.times(1)).shutdown();
Mockito.verify(mockJobCoordinator, Mockito.times(1)).stop();
}
use of org.apache.samza.container.SamzaContainer in project samza by apache.
the class TestStreamProcessor method runJobModelExpireDuringRebalance.
private void runJobModelExpireDuringRebalance(StreamProcessor streamProcessor, ExecutorService executorService, boolean failContainerInterrupt) {
SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
CountDownLatch shutdownLatch = new CountDownLatch(1);
/*
* When there is an initialized container that hasn't started and the stream processor is still in re-balance phase,
* subsequent job model expire request should attempt to interrupt the existing container to safely shut it down
* before proceeding to join the barrier. As part of safe shutdown sequence, we want to ensure shutdownNow is invoked
* on the existing executorService to signal interrupt and make sure new executor service is created.
*/
Mockito.when(executorService.shutdownNow()).thenAnswer(ctx -> {
if (!failContainerInterrupt) {
shutdownLatch.countDown();
}
return null;
});
Mockito.when(executorService.isShutdown()).thenReturn(true);
streamProcessor.state = State.IN_REBALANCE;
streamProcessor.container = mockSamzaContainer;
streamProcessor.containerExecutorService = executorService;
streamProcessor.containerShutdownLatch = shutdownLatch;
streamProcessor.jobCoordinatorListener.onJobModelExpired();
}
use of org.apache.samza.container.SamzaContainer in project samza by apache.
the class WatermarkIntegrationTest method getTaskOperationGraphs.
Map<String, StreamOperatorTask> getTaskOperationGraphs(MockLocalApplicationRunner runner) throws Exception {
StreamProcessor processor = runner.getProcessors().iterator().next();
SamzaContainer container = TestStreamProcessorUtil.getContainer(processor);
Map<TaskName, TaskInstance> taskInstances = JavaConverters.mapAsJavaMapConverter(container.getTaskInstances()).asJava();
Map<String, StreamOperatorTask> tasks = new HashMap<>();
for (Map.Entry<TaskName, TaskInstance> entry : taskInstances.entrySet()) {
StreamOperatorTask task = (StreamOperatorTask) entry.getValue().task();
tasks.put(entry.getKey().getTaskName(), task);
}
return tasks;
}
use of org.apache.samza.container.SamzaContainer in project samza by apache.
the class TestStreamProcessor method testCoordinatorFailureShouldStopTheStreamProcessor.
@Test
public void testCoordinatorFailureShouldStopTheStreamProcessor() {
JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
StreamProcessor streamProcessor = new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, Optional.empty(), Optional.empty(), Optional.empty(), sp -> lifecycleListener, mockJobCoordinator, Mockito.mock(MetadataStore.class));
Exception failureException = new Exception("dummy exception");
streamProcessor.container = mockSamzaContainer;
streamProcessor.state = State.RUNNING;
streamProcessor.jobCoordinatorListener.onCoordinatorFailure(failureException);
Mockito.doNothing().when(mockSamzaContainer).shutdown();
Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false);
assertEquals(State.STOPPED, streamProcessor.state);
Mockito.verify(lifecycleListener).afterFailure(failureException);
Mockito.verify(mockSamzaContainer).shutdown();
}
Aggregations