use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.
the class AuditPublishTest method init.
@BeforeClass
public static void init() throws Exception {
cConf = CConfiguration.create();
cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
Injector injector = AppFabricTestHelper.getInjector(cConf, new AuditModule());
messagingService = injector.getInstance(MessagingService.class);
if (messagingService instanceof Service) {
((Service) messagingService).startAndWait();
}
auditTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.Audit.TOPIC));
}
use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.
the class DefaultPreviewRunner method startUp.
@Override
protected void startUp() throws Exception {
LOG.debug("Starting preview runner service");
StoreDefinition.createAllTables(structuredTableAdmin);
if (messagingService instanceof Service) {
((Service) messagingService).startAndWait();
}
dsOpExecService.startAndWait();
datasetService.startAndWait();
// It is recommended to initialize log appender after datasetService is started,
// since log appender instantiates a dataset.
logAppenderInitializer.initialize();
LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.PREVIEW_HTTP));
Futures.allAsList(applicationLifecycleService.start(), programRuntimeService.start(), metricsCollectionService.start(), programNotificationSubscriberService.start()).get();
Files.createDirectories(previewIdDirPath);
// Reconcile status for abruptly terminated preview runs
try (Stream<Path> paths = Files.walk(Paths.get(previewIdDirPath.toString()))) {
paths.filter(Files::isRegularFile).forEach(path -> {
try (Reader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
ProgramId programId = GSON.fromJson(reader, ProgramId.class);
long submitTimeMillis = RunIds.getTime(programId.getApplication(), TimeUnit.MILLISECONDS);
PreviewStatus status = new PreviewStatus(PreviewStatus.Status.KILLED_BY_EXCEEDING_MEMORY_LIMIT, submitTimeMillis, new BasicThrowable(new Exception("Preview runner container killed possibly because of out of memory. " + "Please try running preview again.")), null, null);
previewTerminated(programId, status);
} catch (IOException e) {
LOG.warn("Error reading file {}. Ignoring", path, e);
}
});
}
}
use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.
the class RuntimeClientServiceTest method testBasicRelay.
@Test
public void testBasicRelay() throws Exception {
// Send some messages to multiple topics in the client side TMS, they should get replicated to the server side TMS.
MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
// the RuntimeClientService will decode it to watch for program termination
if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
// Write a non-terminal state to test basic relaying
programStateWriter.running(PROGRAM_RUN_ID, null);
} else {
messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), entry.getKey(), entry.getKey());
}
}
MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
// Extract the program run status from the Notification
Tasks.waitFor(Collections.singletonList(ProgramRunStatus.RUNNING), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
} else {
Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
}
}
// Writes a program terminate message to unblock stopping of the client service
programStateWriter.completed(PROGRAM_RUN_ID);
}
use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.
the class RuntimeClientServiceTest method testProgramTerminate.
/**
* Test for {@link RuntimeClientService} that will terminate itself when seeing program completed message.
*/
@Test
public void testProgramTerminate() throws Exception {
MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
// Send a terminate program state first, wait for the service sees the state change,
// then publish messages to other topics.
programStateWriter.completed(PROGRAM_RUN_ID);
Tasks.waitFor(true, () -> runtimeClientService.getProgramFinishTime() >= 0, 2, TimeUnit.SECONDS);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
// the RuntimeClientService will decode it to watch for program termination
if (!entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
List<String> payloads = Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey());
messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), StandardCharsets.UTF_8, payloads.iterator());
}
}
// The client service should get stopped by itself.
Tasks.waitFor(Service.State.TERMINATED, () -> runtimeClientService.state(), clientCConf.getLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS) + 2000, TimeUnit.MILLISECONDS);
// All messages should be sent after the runtime client service stopped
MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
// Extract the program run status from the Notification
Tasks.waitFor(Collections.singletonList(ProgramRunStatus.COMPLETED), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
} else {
Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
}
}
}
use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.
the class RuntimeClientServiceTest method beforeTest.
@Before
public void beforeTest() throws Exception {
CConfiguration cConf = CConfiguration.create();
cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
cConf.set(Constants.RuntimeMonitor.TOPICS_CONFIGS, TOPIC_CONFIGS_VALUE);
topicConfigs = RuntimeMonitors.createTopicConfigs(cConf);
InMemoryDiscoveryService discoveryService = new InMemoryDiscoveryService();
// Injector for the server side
Injector injector = Guice.createInjector(new ConfigModule(cConf), new LocalLocationModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), new RuntimeServerModule() {
@Override
protected void bindRequestValidator() {
bind(RuntimeRequestValidator.class).toInstance((programRunId, request) -> new ProgramRunInfo(ProgramRunStatus.COMPLETED, null));
}
@Override
protected void bindLogProcessor() {
bind(RemoteExecutionLogProcessor.class).toInstance(payloads -> {
});
}
}, new AbstractModule() {
@Override
protected void configure() {
bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
bind(DiscoveryService.class).toInstance(discoveryService);
bind(DiscoveryServiceClient.class).toInstance(discoveryService);
}
});
messagingService = injector.getInstance(MessagingService.class);
if (messagingService instanceof Service) {
((Service) messagingService).startAndWait();
}
runtimeServer = injector.getInstance(RuntimeServer.class);
runtimeServer.startAndWait();
// Injector for the client side
clientCConf = CConfiguration.create();
clientCConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
clientCConf.set(Constants.RuntimeMonitor.TOPICS_CONFIGS, TOPIC_CONFIGS_VALUE);
// Shorten the poll delay and grace period to speed up testing of program terminate state handling
clientCConf.setLong(Constants.RuntimeMonitor.POLL_TIME_MS, 200);
clientCConf.setLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS, 3000);
// Use smaller batch size so that fetches is broken into multiple fetches
clientCConf.setInt(Constants.RuntimeMonitor.BATCH_SIZE, 1);
injector = Guice.createInjector(new ConfigModule(clientCConf), RemoteAuthenticatorModules.getNoOpModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), new AbstractModule() {
@Override
protected void configure() {
bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
bind(DiscoveryService.class).toInstance(discoveryService);
bind(DiscoveryServiceClient.class).toInstance(discoveryService);
bind(ProgramRunId.class).toInstance(PROGRAM_RUN_ID);
}
});
clientMessagingService = injector.getInstance(MessagingService.class);
if (clientMessagingService instanceof Service) {
((Service) clientMessagingService).startAndWait();
}
runtimeClientService = injector.getInstance(RuntimeClientService.class);
runtimeClientService.startAndWait();
}
Aggregations