use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.
the class RuntimeClientServiceTest method testProgramTerminate.
/**
* Test for {@link RuntimeClientService} that will terminate itself when seeing program completed message.
*/
@Test
public void testProgramTerminate() throws Exception {
MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
// Send a terminate program state first, wait for the service sees the state change,
// then publish messages to other topics.
programStateWriter.completed(PROGRAM_RUN_ID);
Tasks.waitFor(true, () -> runtimeClientService.getProgramFinishTime() >= 0, 2, TimeUnit.SECONDS);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
// the RuntimeClientService will decode it to watch for program termination
if (!entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
List<String> payloads = Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey());
messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), StandardCharsets.UTF_8, payloads.iterator());
}
}
// The client service should get stopped by itself.
Tasks.waitFor(Service.State.TERMINATED, () -> runtimeClientService.state(), clientCConf.getLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS) + 2000, TimeUnit.MILLISECONDS);
// All messages should be sent after the runtime client service stopped
MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
// Extract the program run status from the Notification
Tasks.waitFor(Collections.singletonList(ProgramRunStatus.COMPLETED), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
} else {
Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
}
}
}
use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.
the class TestBase method initialize.
@BeforeClass
public static void initialize() throws Exception {
if (nestedStartCount++ > 0) {
return;
}
File localDataDir = TMP_FOLDER.newFolder();
cConf = createCConf(localDataDir);
CConfiguration previewCConf = createPreviewConf(cConf);
LevelDBTableService previewLevelDBTableService = new LevelDBTableService();
previewLevelDBTableService.setConfiguration(previewCConf);
// enable default services
File capabilityFolder = new File(localDataDir.toString(), "capability");
capabilityFolder.mkdir();
cConf.set(Constants.Capability.CONFIG_DIR, capabilityFolder.getAbsolutePath());
cConf.setInt(Constants.Capability.AUTO_INSTALL_THREADS, 5);
org.apache.hadoop.conf.Configuration hConf = new org.apache.hadoop.conf.Configuration();
hConf.addResource("mapred-site-local.xml");
hConf.reloadConfiguration();
hConf.set(Constants.CFG_LOCAL_DATA_DIR, localDataDir.getAbsolutePath());
hConf.set(Constants.AppFabric.OUTPUT_DIR, cConf.get(Constants.AppFabric.OUTPUT_DIR));
hConf.set("hadoop.tmp.dir", new File(localDataDir, cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsolutePath());
// Windows specific requirements
if (OSDetector.isWindows()) {
File tmpDir = TMP_FOLDER.newFolder();
File binDir = new File(tmpDir, "bin");
Assert.assertTrue(binDir.mkdirs());
copyTempFile("hadoop.dll", tmpDir);
copyTempFile("winutils.exe", binDir);
System.setProperty("hadoop.home.dir", tmpDir.getAbsolutePath());
System.load(new File(tmpDir, "hadoop.dll").getAbsolutePath());
}
injector = Guice.createInjector(createDataFabricModule(), new TransactionExecutorModule(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getNoOpModule(), new IOModule(), new LocalLocationModule(), new InMemoryDiscoveryModule(), new AppFabricServiceRuntimeModule(cConf).getInMemoryModules(), new MonitorHandlerModule(false), new AuthenticationContextModules().getMasterModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new ProgramRunnerRuntimeModule().getInMemoryModules(), new SecureStoreServerModule(), new MetadataReaderWriterModules().getInMemoryModules(), new MetadataServiceModule(), new AbstractModule() {
@Override
protected void configure() {
bind(MetricsManager.class).toProvider(MetricsManagerProvider.class);
}
}, new MetricsClientRuntimeModule().getInMemoryModules(), new LocalLogAppenderModule(), new LogReaderRuntimeModules().getInMemoryModules(), new ExploreRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new PreviewConfigModule(cConf, new Configuration(), SConfiguration.create()), new PreviewManagerModule(false), new PreviewRunnerManagerModule().getInMemoryModules(), new SupportBundleServiceModule(), new MockProvisionerModule(), new AbstractModule() {
@Override
protected void configure() {
install(new FactoryModuleBuilder().implement(ApplicationManager.class, DefaultApplicationManager.class).build(ApplicationManagerFactory.class));
install(new FactoryModuleBuilder().implement(ArtifactManager.class, DefaultArtifactManager.class).build(ArtifactManagerFactory.class));
bind(TemporaryFolder.class).toInstance(TMP_FOLDER);
bind(AuthorizationHandler.class).in(Scopes.SINGLETON);
// Needed by MonitorHandlerModuler
bind(TwillRunner.class).to(NoopTwillRunnerService.class);
bind(MetadataSubscriberService.class).in(Scopes.SINGLETON);
}
});
messagingService = injector.getInstance(MessagingService.class);
if (messagingService instanceof Service) {
((Service) messagingService).startAndWait();
}
txService = injector.getInstance(TransactionManager.class);
txService.startAndWait();
metadataSubscriberService = injector.getInstance(MetadataSubscriberService.class);
metadataStorage = injector.getInstance(MetadataStorage.class);
metadataAdmin = injector.getInstance(MetadataAdmin.class);
metadataStorage.createIndex();
metadataService = injector.getInstance(MetadataService.class);
metadataService.startAndWait();
// Define all StructuredTable before starting any services that need StructuredTable
StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
dsOpService = injector.getInstance(DatasetOpExecutorService.class);
dsOpService.startAndWait();
datasetService = injector.getInstance(DatasetService.class);
datasetService.startAndWait();
metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
metricsCollectionService.startAndWait();
if (cConf.getBoolean(Constants.Explore.EXPLORE_ENABLED)) {
exploreExecutorService = injector.getInstance(ExploreExecutorService.class);
exploreExecutorService.startAndWait();
// wait for explore service to be discoverable
DiscoveryServiceClient discoveryService = injector.getInstance(DiscoveryServiceClient.class);
EndpointStrategy endpointStrategy = new RandomEndpointStrategy(() -> discoveryService.discover(Constants.Service.EXPLORE_HTTP_USER_SERVICE));
Preconditions.checkNotNull(endpointStrategy.pick(5, TimeUnit.SECONDS), "%s service is not up after 5 seconds", Constants.Service.EXPLORE_HTTP_USER_SERVICE);
exploreClient = injector.getInstance(ExploreClient.class);
}
programScheduler = injector.getInstance(Scheduler.class);
if (programScheduler instanceof Service) {
((Service) programScheduler).startAndWait();
}
testManager = injector.getInstance(UnitTestManager.class);
metricsManager = injector.getInstance(MetricsManager.class);
accessControllerInstantiator = injector.getInstance(AccessControllerInstantiator.class);
// This is needed so the logged-in user can successfully create the default namespace
if (cConf.getBoolean(Constants.Security.Authorization.ENABLED)) {
String user = System.getProperty("user.name");
SecurityRequestContext.setUserId(user);
InstanceId instance = new InstanceId(cConf.get(Constants.INSTANCE_NAME));
Principal principal = new Principal(user, Principal.PrincipalType.USER);
accessControllerInstantiator.get().grant(Authorizable.fromEntityId(instance), principal, EnumSet.allOf(StandardPermission.class));
accessControllerInstantiator.get().grant(Authorizable.fromEntityId(NamespaceId.DEFAULT), principal, EnumSet.allOf(StandardPermission.class));
}
namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
if (firstInit) {
// only create the default namespace on first test. if multiple tests are run in the same JVM,
// then any time after the first time, the default namespace already exists. That is because
// the namespaceAdmin.delete(Id.Namespace.DEFAULT) in finish() only clears the default namespace
// but does not remove it entirely
namespaceAdmin.create(NamespaceMeta.DEFAULT);
ProfileService profileService = injector.getInstance(ProfileService.class);
profileService.saveProfile(ProfileId.NATIVE, Profile.NATIVE);
}
secureStore = injector.getInstance(SecureStore.class);
secureStoreManager = injector.getInstance(SecureStoreManager.class);
messagingContext = new MultiThreadMessagingContext(messagingService);
firstInit = false;
previewHttpServer = injector.getInstance(PreviewHttpServer.class);
previewHttpServer.startAndWait();
fieldLineageAdmin = injector.getInstance(FieldLineageAdmin.class);
lineageAdmin = injector.getInstance(LineageAdmin.class);
metadataSubscriberService.startAndWait();
previewRunnerManager = injector.getInstance(PreviewRunnerManager.class);
if (previewRunnerManager instanceof Service) {
((Service) previewRunnerManager).startAndWait();
}
appFabricServer = injector.getInstance(AppFabricServer.class);
appFabricServer.startAndWait();
preferencesService = injector.getInstance(PreferencesService.class);
scheduler = injector.getInstance(Scheduler.class);
if (scheduler instanceof Service) {
((Service) scheduler).startAndWait();
}
if (scheduler instanceof CoreSchedulerService) {
((CoreSchedulerService) scheduler).waitUntilFunctional(10, TimeUnit.SECONDS);
}
supportBundleInternalService = injector.getInstance(SupportBundleInternalService.class);
supportBundleInternalService.startAndWait();
appFabricHealthCheckService = injector.getInstance(HealthCheckService.class);
appFabricHealthCheckService.helper(Constants.AppFabricHealthCheck.APP_FABRIC_HEALTH_CHECK_SERVICE, cConf, Constants.Service.MASTER_SERVICES_BIND_ADDRESS);
appFabricHealthCheckService.startAndWait();
}
use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.
the class LeaderElectionMessagingServiceTest method testTransition.
@Test
public void testTransition() throws Throwable {
final TopicId topicId = NamespaceId.SYSTEM.topic("topic");
Injector injector1 = createInjector(0);
Injector injector2 = createInjector(1);
// Start a messaging service, which would becomes leader
ZKClientService zkClient1 = injector1.getInstance(ZKClientService.class);
zkClient1.startAndWait();
final MessagingService firstService = injector1.getInstance(MessagingService.class);
if (firstService instanceof Service) {
((Service) firstService).startAndWait();
}
// Publish a message with the leader
firstService.publish(StoreRequestBuilder.of(topicId).addPayload("Testing1").build());
// Start another messaging service, this one would be follower
ZKClientService zkClient2 = injector2.getInstance(ZKClientService.class);
zkClient2.startAndWait();
final MessagingService secondService = injector2.getInstance(MessagingService.class);
if (secondService instanceof Service) {
((Service) secondService).startAndWait();
}
// Try to call the follower, should get service unavailable.
try {
secondService.listTopics(NamespaceId.SYSTEM);
Assert.fail("Expected service unavailable");
} catch (ServiceUnavailableException e) {
// Expected
}
// Make the ZK session timeout for the leader service. The second one should pickup.
KillZKSession.kill(zkClient1.getZooKeeperSupplier().get(), zkClient1.getConnectString(), 10000);
// Publish one more message and then fetch from the current leader
List<String> messages = Retries.callWithRetries(new Retries.Callable<List<String>, Throwable>() {
@Override
public List<String> call() throws Throwable {
secondService.publish(StoreRequestBuilder.of(topicId).addPayload("Testing2").build());
List<String> messages = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = secondService.prepareFetch(topicId).fetch()) {
while (iterator.hasNext()) {
messages.add(new String(iterator.next().getPayload(), "UTF-8"));
}
}
return messages;
}
}, RetryStrategies.timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS)));
Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages);
// Shutdown the current leader. The session timeout one should becomes leader again.
if (secondService instanceof Service) {
((Service) secondService).stopAndWait();
}
// Try to fetch message from the current leader again.
// Should see two messages (because the cache is cleared and fetch is from the backing store).
messages = Retries.callWithRetries(new Retries.Callable<List<String>, Throwable>() {
@Override
public List<String> call() throws Throwable {
List<String> messages = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = firstService.prepareFetch(topicId).fetch()) {
while (iterator.hasNext()) {
messages.add(new String(iterator.next().getPayload(), "UTF-8"));
}
}
return messages;
}
}, RetryStrategies.timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS)));
Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages);
zkClient1.stopAndWait();
zkClient2.stopAndWait();
}
use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.
the class MessagingServiceMainTest method testMessagingService.
@Test
public void testMessagingService() throws Exception {
// Discover the TMS endpoint
Injector injector = getServiceMainInstance(MessagingServiceMain.class).getInjector();
RemoteClientFactory remoteClientFactory = injector.getInstance(RemoteClientFactory.class);
// Use a separate TMS client to create topic, then publish and then poll some messages
TopicId topicId = NamespaceId.SYSTEM.topic("test");
MessagingService messagingService = new ClientMessagingService(remoteClientFactory, true);
messagingService.createTopic(new TopicMetadata(topicId));
// Publish 10 messages
List<String> messages = new ArrayList<>();
for (int i = 0; i < 10; i++) {
String msg = "Testing Message " + i;
messagingService.publish(StoreRequestBuilder.of(topicId).addPayload(msg).build());
messages.add(msg);
}
try (CloseableIterator<RawMessage> iterator = messagingService.prepareFetch(topicId).setLimit(10).fetch()) {
List<String> received = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, 0), false).map(RawMessage::getPayload).map(ByteBuffer::wrap).map(StandardCharsets.UTF_8::decode).map(CharSequence::toString).collect(Collectors.toList());
Assert.assertEquals(messages, received);
}
}
use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.
the class MapReduceProgramRunner method run.
@Override
public ProgramController run(final Program program, ProgramOptions options) {
// Extract and verify parameters
ApplicationSpecification appSpec = program.getApplicationSpecification();
Preconditions.checkNotNull(appSpec, "Missing application specification.");
ProgramType processorType = program.getType();
Preconditions.checkNotNull(processorType, "Missing processor type.");
Preconditions.checkArgument(processorType == ProgramType.MAPREDUCE, "Only MAPREDUCE process type is supported.");
MapReduceSpecification spec = appSpec.getMapReduce().get(program.getName());
Preconditions.checkNotNull(spec, "Missing MapReduceSpecification for %s", program.getName());
Arguments arguments = options.getArguments();
RunId runId = ProgramRunners.getRunId(options);
WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments);
DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, appSpec);
// Setup dataset framework context, if required
if (programDatasetFramework instanceof ProgramContextAware) {
ProgramId programId = program.getId();
((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
}
MapReduce mapReduce;
try {
mapReduce = new InstantiatorFactory(false).get(TypeToken.of(program.<MapReduce>getMainClass())).create();
} catch (Exception e) {
LOG.error("Failed to instantiate MapReduce class for {}", spec.getClassName(), e);
throw Throwables.propagate(e);
}
// List of all Closeable resources that needs to be cleanup
List<Closeable> closeables = new ArrayList<>();
try {
PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
if (pluginInstantiator != null) {
closeables.add(pluginInstantiator);
}
final BasicMapReduceContext context = new BasicMapReduceContext(program, options, cConf, spec, workflowInfo, discoveryServiceClient, metricsCollectionService, txSystemClient, programDatasetFramework, getPluginArchive(options), pluginInstantiator, secureStore, secureStoreManager, messagingService, metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory);
closeables.add(context);
Reflections.visit(mapReduce, mapReduce.getClass(), new PropertyFieldSetter(context.getSpecification().getProperties()), new MetricsFieldSetter(context.getMetrics()), new DataSetFieldSetter(context));
// note: this sets logging context on the thread level
LoggingContextAccessor.setLoggingContext(context.getLoggingContext());
// Set the job queue to hConf if it is provided
Configuration hConf = new Configuration(this.hConf);
String schedulerQueue = options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE);
if (schedulerQueue != null && !schedulerQueue.isEmpty()) {
hConf.set(JobContext.QUEUE_NAME, schedulerQueue);
}
ClusterMode clusterMode = ProgramRunners.getClusterMode(options);
Service mapReduceRuntimeService = new MapReduceRuntimeService(injector, cConf, hConf, mapReduce, spec, context, program.getJarLocation(), locationFactory, clusterMode, fieldLineageWriter);
mapReduceRuntimeService.addListener(createRuntimeServiceListener(closeables), Threads.SAME_THREAD_EXECUTOR);
ProgramController controller = new MapReduceProgramController(mapReduceRuntimeService, context);
LOG.debug("Starting MapReduce Job: {}", context);
// be running the job, but the data directory will be owned by cdap.
if (MapReduceTaskContextProvider.isLocal(hConf) || UserGroupInformation.isSecurityEnabled()) {
mapReduceRuntimeService.start();
} else {
ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), mapReduceRuntimeService);
}
return controller;
} catch (Exception e) {
closeAllQuietly(closeables);
throw Throwables.propagate(e);
}
}
Aggregations