use of io.cdap.cdap.messaging.context.MultiThreadMessagingContext in project cdap by caskdata.
the class RuntimeClientServiceTest method testBasicRelay.
@Test
public void testBasicRelay() throws Exception {
// Send some messages to multiple topics in the client side TMS, they should get replicated to the server side TMS.
MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
// the RuntimeClientService will decode it to watch for program termination
if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
// Write a non-terminal state to test basic relaying
programStateWriter.running(PROGRAM_RUN_ID, null);
} else {
messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), entry.getKey(), entry.getKey());
}
}
MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
// Extract the program run status from the Notification
Tasks.waitFor(Collections.singletonList(ProgramRunStatus.RUNNING), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
} else {
Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
}
}
// Writes a program terminate message to unblock stopping of the client service
programStateWriter.completed(PROGRAM_RUN_ID);
}
use of io.cdap.cdap.messaging.context.MultiThreadMessagingContext in project cdap by caskdata.
the class RuntimeClientServiceTest method testProgramTerminate.
/**
* Test for {@link RuntimeClientService} that will terminate itself when seeing program completed message.
*/
@Test
public void testProgramTerminate() throws Exception {
MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
// Send a terminate program state first, wait for the service sees the state change,
// then publish messages to other topics.
programStateWriter.completed(PROGRAM_RUN_ID);
Tasks.waitFor(true, () -> runtimeClientService.getProgramFinishTime() >= 0, 2, TimeUnit.SECONDS);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
// the RuntimeClientService will decode it to watch for program termination
if (!entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
List<String> payloads = Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey());
messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), StandardCharsets.UTF_8, payloads.iterator());
}
}
// The client service should get stopped by itself.
Tasks.waitFor(Service.State.TERMINATED, () -> runtimeClientService.state(), clientCConf.getLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS) + 2000, TimeUnit.MILLISECONDS);
// All messages should be sent after the runtime client service stopped
MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
// Extract the program run status from the Notification
Tasks.waitFor(Collections.singletonList(ProgramRunStatus.COMPLETED), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
} else {
Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
}
}
}
use of io.cdap.cdap.messaging.context.MultiThreadMessagingContext in project cdap by caskdata.
the class MapReduceRunnerTestBase method getDataNotifications.
/**
* Returns a list of {@link Notification} object fetched from the data event topic in TMS that was published
* starting from the given time.
*/
protected List<Notification> getDataNotifications(long startTime) throws Exception {
// Get data notifications from TMS
List<Notification> notifications = new ArrayList<>();
MessagingContext messagingContext = new MultiThreadMessagingContext(injector.getInstance(MessagingService.class));
try (CloseableIterator<Message> messages = messagingContext.getMessageFetcher().fetch(NamespaceId.SYSTEM.getNamespace(), injector.getInstance(CConfiguration.class).get(Constants.Dataset.DATA_EVENT_TOPIC), 10, startTime)) {
while (messages.hasNext()) {
notifications.add(GSON.fromJson(new String(messages.next().getPayload(), StandardCharsets.UTF_8), Notification.class));
}
}
return notifications;
}
use of io.cdap.cdap.messaging.context.MultiThreadMessagingContext in project cdap by caskdata.
the class TetheringRuntimeJobManagerTest method setUp.
@BeforeClass
public static void setUp() throws IOException, TopicAlreadyExistsException {
CConfiguration cConf = CConfiguration.create();
cConf.set(Constants.Tethering.TOPIC_PREFIX, "prefix-");
cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
Injector injector = Guice.createInjector(new ConfigModule(cConf), new InMemoryDiscoveryModule(), new LocalLocationModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new AbstractModule() {
@Override
protected void configure() {
bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class).in(Scopes.SINGLETON);
}
});
messagingService = injector.getInstance(MessagingService.class);
if (messagingService instanceof Service) {
((Service) messagingService).startAndWait();
}
TetheringConf conf = TetheringConf.fromProperties(PROPERTIES);
topicId = new TopicId(NamespaceId.SYSTEM.getNamespace(), cConf.get(Constants.Tethering.TOPIC_PREFIX) + TETHERED_INSTANCE_NAME);
messagingService.createTopic(new TopicMetadata(topicId, Collections.emptyMap()));
messageFetcher = new MultiThreadMessagingContext(messagingService).getMessageFetcher();
runtimeJobManager = new TetheringRuntimeJobManager(conf, cConf, messagingService);
}
use of io.cdap.cdap.messaging.context.MultiThreadMessagingContext in project cdap by caskdata.
the class MetadataSubscriberServiceTest method testAppDeletionMessage.
@Test
public void testAppDeletionMessage() throws Exception {
Injector injector = getInjector();
// get the alert publisher
CConfiguration cConf = injector.getInstance(CConfiguration.class);
MessagingService messagingService = injector.getInstance(MessagingService.class);
MultiThreadMessagingContext messagingContext = new MultiThreadMessagingContext(messagingService);
AdminEventPublisher publisher = new AdminEventPublisher(cConf, messagingContext);
// get the mds and put some workflow metadata in that, the publish of app deletion message should get the metadata
// deleted
MetadataStorage mds = injector.getInstance(MetadataStorage.class);
// use an app with all program types to get all specification tested
ApplicationId appId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
ProgramId workflowId = appId.workflow(AllProgramsApp.NoOpWorkflow.NAME);
// generate an app spec from the application
ApplicationSpecification appSpec = Specifications.from(new AllProgramsApp());
// need to put metadata on workflow since we currently only set or delete workflow metadata
mds.apply(new MetadataMutation.Update(workflowId.toMetadataEntity(), new Metadata(MetadataScope.SYSTEM, Collections.singletonMap("profile", ProfileId.NATIVE.getScopedName()))), MutationOptions.DEFAULT);
Assert.assertEquals(ProfileId.NATIVE.getScopedName(), getProfileProperty(mds, workflowId));
// publish app deletion message
publisher.publishAppDeletion(appId, appSpec);
// Verify the workflow profile metadata is removed because of the publish app deletion message
Tasks.waitFor(true, () -> mds.read(new Read(workflowId.toMetadataEntity())).isEmpty(), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
}
Aggregations