Search in sources :

Example 31 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class RuntimeClientServiceTest method testProgramTerminate.

/**
 * Test for {@link RuntimeClientService} that will terminate itself when seeing program completed message.
 */
@Test
public void testProgramTerminate() throws Exception {
    MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
    MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
    // Send a terminate program state first, wait for the service sees the state change,
    // then publish messages to other topics.
    programStateWriter.completed(PROGRAM_RUN_ID);
    Tasks.waitFor(true, () -> runtimeClientService.getProgramFinishTime() >= 0, 2, TimeUnit.SECONDS);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        // the RuntimeClientService will decode it to watch for program termination
        if (!entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            List<String> payloads = Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey());
            messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), StandardCharsets.UTF_8, payloads.iterator());
        }
    }
    // The client service should get stopped by itself.
    Tasks.waitFor(Service.State.TERMINATED, () -> runtimeClientService.state(), clientCConf.getLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS) + 2000, TimeUnit.MILLISECONDS);
    // All messages should be sent after the runtime client service stopped
    MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Extract the program run status from the Notification
            Tasks.waitFor(Collections.singletonList(ProgramRunStatus.COMPLETED), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        } else {
            Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        }
    }
}
Also used : MessagingServerRuntimeModule(io.cdap.cdap.messaging.guice.MessagingServerRuntimeModule) Arrays(java.util.Arrays) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Notification(io.cdap.cdap.proto.Notification) Spliterators(java.util.Spliterators) TimeoutException(java.util.concurrent.TimeoutException) MessageFetcher(io.cdap.cdap.api.messaging.MessageFetcher) ProgramStateWriter(io.cdap.cdap.app.runtime.ProgramStateWriter) Gson(com.google.gson.Gson) RuntimeServerModule(io.cdap.cdap.app.guice.RuntimeServerModule) After(org.junit.After) Map(java.util.Map) ClassRule(org.junit.ClassRule) NoOpMetricsCollectionService(io.cdap.cdap.common.metrics.NoOpMetricsCollectionService) Tasks(io.cdap.cdap.common.utils.Tasks) MessagingService(io.cdap.cdap.messaging.MessagingService) ProgramRunStatus(io.cdap.cdap.proto.ProgramRunStatus) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) List(java.util.List) MultiThreadMessagingContext(io.cdap.cdap.messaging.context.MultiThreadMessagingContext) Constants(io.cdap.cdap.common.conf.Constants) MessagingContext(io.cdap.cdap.api.messaging.MessagingContext) ProgramOptionConstants(io.cdap.cdap.internal.app.runtime.ProgramOptionConstants) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) InMemoryDiscoveryService(org.apache.twill.discovery.InMemoryDiscoveryService) LocalLocationModule(io.cdap.cdap.common.guice.LocalLocationModule) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) RemoteAuthenticatorModules(io.cdap.cdap.common.guice.RemoteAuthenticatorModules) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) StreamSupport(java.util.stream.StreamSupport) Nullable(javax.annotation.Nullable) Before(org.junit.Before) DiscoveryService(org.apache.twill.discovery.DiscoveryService) Message(io.cdap.cdap.api.messaging.Message) RunIds(io.cdap.cdap.common.app.RunIds) Test(org.junit.Test) IOException(java.io.IOException) CloseableIterator(io.cdap.cdap.api.dataset.lib.CloseableIterator) TopicNotFoundException(io.cdap.cdap.api.messaging.TopicNotFoundException) Service(com.google.common.util.concurrent.Service) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) Injector(com.google.inject.Injector) TimeUnit(java.util.concurrent.TimeUnit) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) MessagePublisher(io.cdap.cdap.api.messaging.MessagePublisher) Guice(com.google.inject.Guice) Assert(org.junit.Assert) Collections(java.util.Collections) AuthenticationContextModules(io.cdap.cdap.security.auth.context.AuthenticationContextModules) TemporaryFolder(org.junit.rules.TemporaryFolder) AbstractModule(com.google.inject.AbstractModule) Message(io.cdap.cdap.api.messaging.Message) MessagePublisher(io.cdap.cdap.api.messaging.MessagePublisher) MultiThreadMessagingContext(io.cdap.cdap.messaging.context.MultiThreadMessagingContext) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) Notification(io.cdap.cdap.proto.Notification) ProgramRunStatus(io.cdap.cdap.proto.ProgramRunStatus) ProgramStateWriter(io.cdap.cdap.app.runtime.ProgramStateWriter) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) MultiThreadMessagingContext(io.cdap.cdap.messaging.context.MultiThreadMessagingContext) MessagingContext(io.cdap.cdap.api.messaging.MessagingContext) Map(java.util.Map) Test(org.junit.Test)

Example 32 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.

the class TestBase method initialize.

@BeforeClass
public static void initialize() throws Exception {
    if (nestedStartCount++ > 0) {
        return;
    }
    File localDataDir = TMP_FOLDER.newFolder();
    cConf = createCConf(localDataDir);
    CConfiguration previewCConf = createPreviewConf(cConf);
    LevelDBTableService previewLevelDBTableService = new LevelDBTableService();
    previewLevelDBTableService.setConfiguration(previewCConf);
    // enable default services
    File capabilityFolder = new File(localDataDir.toString(), "capability");
    capabilityFolder.mkdir();
    cConf.set(Constants.Capability.CONFIG_DIR, capabilityFolder.getAbsolutePath());
    cConf.setInt(Constants.Capability.AUTO_INSTALL_THREADS, 5);
    org.apache.hadoop.conf.Configuration hConf = new org.apache.hadoop.conf.Configuration();
    hConf.addResource("mapred-site-local.xml");
    hConf.reloadConfiguration();
    hConf.set(Constants.CFG_LOCAL_DATA_DIR, localDataDir.getAbsolutePath());
    hConf.set(Constants.AppFabric.OUTPUT_DIR, cConf.get(Constants.AppFabric.OUTPUT_DIR));
    hConf.set("hadoop.tmp.dir", new File(localDataDir, cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsolutePath());
    // Windows specific requirements
    if (OSDetector.isWindows()) {
        File tmpDir = TMP_FOLDER.newFolder();
        File binDir = new File(tmpDir, "bin");
        Assert.assertTrue(binDir.mkdirs());
        copyTempFile("hadoop.dll", tmpDir);
        copyTempFile("winutils.exe", binDir);
        System.setProperty("hadoop.home.dir", tmpDir.getAbsolutePath());
        System.load(new File(tmpDir, "hadoop.dll").getAbsolutePath());
    }
    injector = Guice.createInjector(createDataFabricModule(), new TransactionExecutorModule(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getNoOpModule(), new IOModule(), new LocalLocationModule(), new InMemoryDiscoveryModule(), new AppFabricServiceRuntimeModule(cConf).getInMemoryModules(), new MonitorHandlerModule(false), new AuthenticationContextModules().getMasterModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new ProgramRunnerRuntimeModule().getInMemoryModules(), new SecureStoreServerModule(), new MetadataReaderWriterModules().getInMemoryModules(), new MetadataServiceModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsManager.class).toProvider(MetricsManagerProvider.class);
        }
    }, new MetricsClientRuntimeModule().getInMemoryModules(), new LocalLogAppenderModule(), new LogReaderRuntimeModules().getInMemoryModules(), new ExploreRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new PreviewConfigModule(cConf, new Configuration(), SConfiguration.create()), new PreviewManagerModule(false), new PreviewRunnerManagerModule().getInMemoryModules(), new SupportBundleServiceModule(), new MockProvisionerModule(), new AbstractModule() {

        @Override
        protected void configure() {
            install(new FactoryModuleBuilder().implement(ApplicationManager.class, DefaultApplicationManager.class).build(ApplicationManagerFactory.class));
            install(new FactoryModuleBuilder().implement(ArtifactManager.class, DefaultArtifactManager.class).build(ArtifactManagerFactory.class));
            bind(TemporaryFolder.class).toInstance(TMP_FOLDER);
            bind(AuthorizationHandler.class).in(Scopes.SINGLETON);
            // Needed by MonitorHandlerModuler
            bind(TwillRunner.class).to(NoopTwillRunnerService.class);
            bind(MetadataSubscriberService.class).in(Scopes.SINGLETON);
        }
    });
    messagingService = injector.getInstance(MessagingService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    txService = injector.getInstance(TransactionManager.class);
    txService.startAndWait();
    metadataSubscriberService = injector.getInstance(MetadataSubscriberService.class);
    metadataStorage = injector.getInstance(MetadataStorage.class);
    metadataAdmin = injector.getInstance(MetadataAdmin.class);
    metadataStorage.createIndex();
    metadataService = injector.getInstance(MetadataService.class);
    metadataService.startAndWait();
    // Define all StructuredTable before starting any services that need StructuredTable
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    dsOpService = injector.getInstance(DatasetOpExecutorService.class);
    dsOpService.startAndWait();
    datasetService = injector.getInstance(DatasetService.class);
    datasetService.startAndWait();
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsCollectionService.startAndWait();
    if (cConf.getBoolean(Constants.Explore.EXPLORE_ENABLED)) {
        exploreExecutorService = injector.getInstance(ExploreExecutorService.class);
        exploreExecutorService.startAndWait();
        // wait for explore service to be discoverable
        DiscoveryServiceClient discoveryService = injector.getInstance(DiscoveryServiceClient.class);
        EndpointStrategy endpointStrategy = new RandomEndpointStrategy(() -> discoveryService.discover(Constants.Service.EXPLORE_HTTP_USER_SERVICE));
        Preconditions.checkNotNull(endpointStrategy.pick(5, TimeUnit.SECONDS), "%s service is not up after 5 seconds", Constants.Service.EXPLORE_HTTP_USER_SERVICE);
        exploreClient = injector.getInstance(ExploreClient.class);
    }
    programScheduler = injector.getInstance(Scheduler.class);
    if (programScheduler instanceof Service) {
        ((Service) programScheduler).startAndWait();
    }
    testManager = injector.getInstance(UnitTestManager.class);
    metricsManager = injector.getInstance(MetricsManager.class);
    accessControllerInstantiator = injector.getInstance(AccessControllerInstantiator.class);
    // This is needed so the logged-in user can successfully create the default namespace
    if (cConf.getBoolean(Constants.Security.Authorization.ENABLED)) {
        String user = System.getProperty("user.name");
        SecurityRequestContext.setUserId(user);
        InstanceId instance = new InstanceId(cConf.get(Constants.INSTANCE_NAME));
        Principal principal = new Principal(user, Principal.PrincipalType.USER);
        accessControllerInstantiator.get().grant(Authorizable.fromEntityId(instance), principal, EnumSet.allOf(StandardPermission.class));
        accessControllerInstantiator.get().grant(Authorizable.fromEntityId(NamespaceId.DEFAULT), principal, EnumSet.allOf(StandardPermission.class));
    }
    namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
    if (firstInit) {
        // only create the default namespace on first test. if multiple tests are run in the same JVM,
        // then any time after the first time, the default namespace already exists. That is because
        // the namespaceAdmin.delete(Id.Namespace.DEFAULT) in finish() only clears the default namespace
        // but does not remove it entirely
        namespaceAdmin.create(NamespaceMeta.DEFAULT);
        ProfileService profileService = injector.getInstance(ProfileService.class);
        profileService.saveProfile(ProfileId.NATIVE, Profile.NATIVE);
    }
    secureStore = injector.getInstance(SecureStore.class);
    secureStoreManager = injector.getInstance(SecureStoreManager.class);
    messagingContext = new MultiThreadMessagingContext(messagingService);
    firstInit = false;
    previewHttpServer = injector.getInstance(PreviewHttpServer.class);
    previewHttpServer.startAndWait();
    fieldLineageAdmin = injector.getInstance(FieldLineageAdmin.class);
    lineageAdmin = injector.getInstance(LineageAdmin.class);
    metadataSubscriberService.startAndWait();
    previewRunnerManager = injector.getInstance(PreviewRunnerManager.class);
    if (previewRunnerManager instanceof Service) {
        ((Service) previewRunnerManager).startAndWait();
    }
    appFabricServer = injector.getInstance(AppFabricServer.class);
    appFabricServer.startAndWait();
    preferencesService = injector.getInstance(PreferencesService.class);
    scheduler = injector.getInstance(Scheduler.class);
    if (scheduler instanceof Service) {
        ((Service) scheduler).startAndWait();
    }
    if (scheduler instanceof CoreSchedulerService) {
        ((CoreSchedulerService) scheduler).waitUntilFunctional(10, TimeUnit.SECONDS);
    }
    supportBundleInternalService = injector.getInstance(SupportBundleInternalService.class);
    supportBundleInternalService.startAndWait();
    appFabricHealthCheckService = injector.getInstance(HealthCheckService.class);
    appFabricHealthCheckService.helper(Constants.AppFabricHealthCheck.APP_FABRIC_HEALTH_CHECK_SERVICE, cConf, Constants.Service.MASTER_SERVICES_BIND_ADDRESS);
    appFabricHealthCheckService.startAndWait();
}
Also used : DataSetServiceModules(io.cdap.cdap.data.runtime.DataSetServiceModules) PreviewManagerModule(io.cdap.cdap.app.preview.PreviewManagerModule) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) Configuration(org.apache.hadoop.conf.Configuration) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) TwillRunner(org.apache.twill.api.TwillRunner) DatasetService(io.cdap.cdap.data2.datafabric.dataset.service.DatasetService) MetricsClientRuntimeModule(io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule) LineageAdmin(io.cdap.cdap.metadata.LineageAdmin) FieldLineageAdmin(io.cdap.cdap.metadata.FieldLineageAdmin) StandardPermission(io.cdap.cdap.proto.security.StandardPermission) PreviewConfigModule(io.cdap.cdap.app.preview.PreviewConfigModule) RandomEndpointStrategy(io.cdap.cdap.common.discovery.RandomEndpointStrategy) EndpointStrategy(io.cdap.cdap.common.discovery.EndpointStrategy) CoreSchedulerService(io.cdap.cdap.scheduler.CoreSchedulerService) Configuration(org.apache.hadoop.conf.Configuration) SecureStoreManager(io.cdap.cdap.api.security.store.SecureStoreManager) SupportBundleServiceModule(io.cdap.cdap.support.app.guice.SupportBundleServiceModule) PreviewRunnerManagerModule(io.cdap.cdap.app.preview.PreviewRunnerManagerModule) PreviewRunnerManager(io.cdap.cdap.app.preview.PreviewRunnerManager) AuthorizationModule(io.cdap.cdap.app.guice.AuthorizationModule) HealthCheckService(io.cdap.cdap.common.service.HealthCheckService) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) InstanceId(io.cdap.cdap.proto.id.InstanceId) AuthenticationContextModules(io.cdap.cdap.security.auth.context.AuthenticationContextModules) MetadataServiceModule(io.cdap.cdap.metadata.MetadataServiceModule) NamespaceAdmin(io.cdap.cdap.common.namespace.NamespaceAdmin) ExploreRuntimeModule(io.cdap.cdap.explore.guice.ExploreRuntimeModule) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) SecureStore(io.cdap.cdap.api.security.store.SecureStore) MetadataReaderWriterModules(io.cdap.cdap.metadata.MetadataReaderWriterModules) SupportBundleInternalService(io.cdap.cdap.support.internal.app.services.SupportBundleInternalService) ExploreClientModule(io.cdap.cdap.explore.guice.ExploreClientModule) MetadataSubscriberService(io.cdap.cdap.metadata.MetadataSubscriberService) TransactionManager(org.apache.tephra.TransactionManager) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) File(java.io.File) AppFabricServiceRuntimeModule(io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule) AuthorizationEnforcementModule(io.cdap.cdap.security.authorization.AuthorizationEnforcementModule) FieldLineageAdmin(io.cdap.cdap.metadata.FieldLineageAdmin) IOModule(io.cdap.cdap.common.guice.IOModule) AuthorizationHandler(io.cdap.cdap.gateway.handlers.AuthorizationHandler) ExploreClient(io.cdap.cdap.explore.client.ExploreClient) InMemoryDiscoveryModule(io.cdap.cdap.common.guice.InMemoryDiscoveryModule) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) PreviewConfigModule(io.cdap.cdap.app.preview.PreviewConfigModule) FactoryModuleBuilder(com.google.inject.assistedinject.FactoryModuleBuilder) ArtifactManagerFactory(io.cdap.cdap.test.internal.ArtifactManagerFactory) StructuredTableAdmin(io.cdap.cdap.spi.data.StructuredTableAdmin) Scheduler(io.cdap.cdap.scheduler.Scheduler) AppFabricServer(io.cdap.cdap.internal.app.services.AppFabricServer) MessagingServerRuntimeModule(io.cdap.cdap.messaging.guice.MessagingServerRuntimeModule) MultiThreadMessagingContext(io.cdap.cdap.messaging.context.MultiThreadMessagingContext) MetadataService(io.cdap.cdap.metadata.MetadataService) MonitorHandlerModule(io.cdap.cdap.app.guice.MonitorHandlerModule) PreferencesService(io.cdap.cdap.config.PreferencesService) TransactionExecutorModule(io.cdap.cdap.data.runtime.TransactionExecutorModule) LocalLocationModule(io.cdap.cdap.common.guice.LocalLocationModule) TemporaryFolder(org.junit.rules.TemporaryFolder) MockProvisionerModule(io.cdap.cdap.internal.provision.MockProvisionerModule) MetadataAdmin(io.cdap.cdap.metadata.MetadataAdmin) ApplicationManagerFactory(io.cdap.cdap.test.internal.ApplicationManagerFactory) LogReaderRuntimeModules(io.cdap.cdap.logging.guice.LogReaderRuntimeModules) DataSetsModules(io.cdap.cdap.data.runtime.DataSetsModules) NoopTwillRunnerService(io.cdap.cdap.common.twill.NoopTwillRunnerService) PreferencesService(io.cdap.cdap.config.PreferencesService) MetadataSubscriberService(io.cdap.cdap.metadata.MetadataSubscriberService) ExploreExecutorService(io.cdap.cdap.explore.executor.ExploreExecutorService) LevelDBTableService(io.cdap.cdap.data2.dataset2.lib.table.leveldb.LevelDBTableService) DatasetOpExecutorService(io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutorService) ProfileService(io.cdap.cdap.internal.profile.ProfileService) Service(com.google.common.util.concurrent.Service) CapabilityManagementService(io.cdap.cdap.internal.capability.CapabilityManagementService) MetadataService(io.cdap.cdap.metadata.MetadataService) HealthCheckService(io.cdap.cdap.common.service.HealthCheckService) MessagingService(io.cdap.cdap.messaging.MessagingService) SupportBundleInternalService(io.cdap.cdap.support.internal.app.services.SupportBundleInternalService) DatasetService(io.cdap.cdap.data2.datafabric.dataset.service.DatasetService) CoreSchedulerService(io.cdap.cdap.scheduler.CoreSchedulerService) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) AccessControllerInstantiator(io.cdap.cdap.security.authorization.AccessControllerInstantiator) AbstractModule(com.google.inject.AbstractModule) MessagingService(io.cdap.cdap.messaging.MessagingService) ProgramRunnerRuntimeModule(io.cdap.cdap.app.guice.ProgramRunnerRuntimeModule) ProfileService(io.cdap.cdap.internal.profile.ProfileService) LocalLogAppenderModule(io.cdap.cdap.logging.guice.LocalLogAppenderModule) LevelDBTableService(io.cdap.cdap.data2.dataset2.lib.table.leveldb.LevelDBTableService) DatasetOpExecutorService(io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutorService) ExploreExecutorService(io.cdap.cdap.explore.executor.ExploreExecutorService) SecureStoreServerModule(io.cdap.cdap.security.guice.SecureStoreServerModule) Principal(io.cdap.cdap.proto.security.Principal) RandomEndpointStrategy(io.cdap.cdap.common.discovery.RandomEndpointStrategy) PreviewHttpServer(io.cdap.cdap.app.preview.PreviewHttpServer) BeforeClass(org.junit.BeforeClass)

Example 33 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.

the class LeaderElectionMessagingServiceTest method testTransition.

@Test
public void testTransition() throws Throwable {
    final TopicId topicId = NamespaceId.SYSTEM.topic("topic");
    Injector injector1 = createInjector(0);
    Injector injector2 = createInjector(1);
    // Start a messaging service, which would becomes leader
    ZKClientService zkClient1 = injector1.getInstance(ZKClientService.class);
    zkClient1.startAndWait();
    final MessagingService firstService = injector1.getInstance(MessagingService.class);
    if (firstService instanceof Service) {
        ((Service) firstService).startAndWait();
    }
    // Publish a message with the leader
    firstService.publish(StoreRequestBuilder.of(topicId).addPayload("Testing1").build());
    // Start another messaging service, this one would be follower
    ZKClientService zkClient2 = injector2.getInstance(ZKClientService.class);
    zkClient2.startAndWait();
    final MessagingService secondService = injector2.getInstance(MessagingService.class);
    if (secondService instanceof Service) {
        ((Service) secondService).startAndWait();
    }
    // Try to call the follower, should get service unavailable.
    try {
        secondService.listTopics(NamespaceId.SYSTEM);
        Assert.fail("Expected service unavailable");
    } catch (ServiceUnavailableException e) {
    // Expected
    }
    // Make the ZK session timeout for the leader service. The second one should pickup.
    KillZKSession.kill(zkClient1.getZooKeeperSupplier().get(), zkClient1.getConnectString(), 10000);
    // Publish one more message and then fetch from the current leader
    List<String> messages = Retries.callWithRetries(new Retries.Callable<List<String>, Throwable>() {

        @Override
        public List<String> call() throws Throwable {
            secondService.publish(StoreRequestBuilder.of(topicId).addPayload("Testing2").build());
            List<String> messages = new ArrayList<>();
            try (CloseableIterator<RawMessage> iterator = secondService.prepareFetch(topicId).fetch()) {
                while (iterator.hasNext()) {
                    messages.add(new String(iterator.next().getPayload(), "UTF-8"));
                }
            }
            return messages;
        }
    }, RetryStrategies.timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS)));
    Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages);
    // Shutdown the current leader. The session timeout one should becomes leader again.
    if (secondService instanceof Service) {
        ((Service) secondService).stopAndWait();
    }
    // Try to fetch message from the current leader again.
    // Should see two messages (because the cache is cleared and fetch is from the backing store).
    messages = Retries.callWithRetries(new Retries.Callable<List<String>, Throwable>() {

        @Override
        public List<String> call() throws Throwable {
            List<String> messages = new ArrayList<>();
            try (CloseableIterator<RawMessage> iterator = firstService.prepareFetch(topicId).fetch()) {
                while (iterator.hasNext()) {
                    messages.add(new String(iterator.next().getPayload(), "UTF-8"));
                }
            }
            return messages;
        }
    }, RetryStrategies.timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS)));
    Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages);
    zkClient1.stopAndWait();
    zkClient2.stopAndWait();
}
Also used : CloseableIterator(io.cdap.cdap.api.dataset.lib.CloseableIterator) ArrayList(java.util.ArrayList) NoOpMetricsCollectionService(io.cdap.cdap.common.metrics.NoOpMetricsCollectionService) MessagingService(io.cdap.cdap.messaging.MessagingService) ZKClientService(org.apache.twill.zookeeper.ZKClientService) Service(com.google.common.util.concurrent.Service) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) ServiceUnavailableException(io.cdap.cdap.common.ServiceUnavailableException) Callable(java.util.concurrent.Callable) MessagingService(io.cdap.cdap.messaging.MessagingService) ZKClientService(org.apache.twill.zookeeper.ZKClientService) Injector(com.google.inject.Injector) Retries(io.cdap.cdap.common.service.Retries) TopicId(io.cdap.cdap.proto.id.TopicId) List(java.util.List) ArrayList(java.util.ArrayList) RawMessage(io.cdap.cdap.messaging.data.RawMessage) Test(org.junit.Test)

Example 34 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.

the class MessagingServiceMainTest method testMessagingService.

@Test
public void testMessagingService() throws Exception {
    // Discover the TMS endpoint
    Injector injector = getServiceMainInstance(MessagingServiceMain.class).getInjector();
    RemoteClientFactory remoteClientFactory = injector.getInstance(RemoteClientFactory.class);
    // Use a separate TMS client to create topic, then publish and then poll some messages
    TopicId topicId = NamespaceId.SYSTEM.topic("test");
    MessagingService messagingService = new ClientMessagingService(remoteClientFactory, true);
    messagingService.createTopic(new TopicMetadata(topicId));
    // Publish 10 messages
    List<String> messages = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msg = "Testing Message " + i;
        messagingService.publish(StoreRequestBuilder.of(topicId).addPayload(msg).build());
        messages.add(msg);
    }
    try (CloseableIterator<RawMessage> iterator = messagingService.prepareFetch(topicId).setLimit(10).fetch()) {
        List<String> received = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, 0), false).map(RawMessage::getPayload).map(ByteBuffer::wrap).map(StandardCharsets.UTF_8::decode).map(CharSequence::toString).collect(Collectors.toList());
        Assert.assertEquals(messages, received);
    }
}
Also used : RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) ArrayList(java.util.ArrayList) MessagingService(io.cdap.cdap.messaging.MessagingService) ClientMessagingService(io.cdap.cdap.messaging.client.ClientMessagingService) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) Injector(com.google.inject.Injector) TopicId(io.cdap.cdap.proto.id.TopicId) ClientMessagingService(io.cdap.cdap.messaging.client.ClientMessagingService) RawMessage(io.cdap.cdap.messaging.data.RawMessage) Test(org.junit.Test)

Example 35 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by caskdata.

the class MapReduceProgramRunner method run.

@Override
public ProgramController run(final Program program, ProgramOptions options) {
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.MAPREDUCE, "Only MAPREDUCE process type is supported.");
    MapReduceSpecification spec = appSpec.getMapReduce().get(program.getName());
    Preconditions.checkNotNull(spec, "Missing MapReduceSpecification for %s", program.getName());
    Arguments arguments = options.getArguments();
    RunId runId = ProgramRunners.getRunId(options);
    WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments);
    DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, appSpec);
    // Setup dataset framework context, if required
    if (programDatasetFramework instanceof ProgramContextAware) {
        ProgramId programId = program.getId();
        ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
    }
    MapReduce mapReduce;
    try {
        mapReduce = new InstantiatorFactory(false).get(TypeToken.of(program.<MapReduce>getMainClass())).create();
    } catch (Exception e) {
        LOG.error("Failed to instantiate MapReduce class for {}", spec.getClassName(), e);
        throw Throwables.propagate(e);
    }
    // List of all Closeable resources that needs to be cleanup
    List<Closeable> closeables = new ArrayList<>();
    try {
        PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
        if (pluginInstantiator != null) {
            closeables.add(pluginInstantiator);
        }
        final BasicMapReduceContext context = new BasicMapReduceContext(program, options, cConf, spec, workflowInfo, discoveryServiceClient, metricsCollectionService, txSystemClient, programDatasetFramework, getPluginArchive(options), pluginInstantiator, secureStore, secureStoreManager, messagingService, metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory);
        closeables.add(context);
        Reflections.visit(mapReduce, mapReduce.getClass(), new PropertyFieldSetter(context.getSpecification().getProperties()), new MetricsFieldSetter(context.getMetrics()), new DataSetFieldSetter(context));
        // note: this sets logging context on the thread level
        LoggingContextAccessor.setLoggingContext(context.getLoggingContext());
        // Set the job queue to hConf if it is provided
        Configuration hConf = new Configuration(this.hConf);
        String schedulerQueue = options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE);
        if (schedulerQueue != null && !schedulerQueue.isEmpty()) {
            hConf.set(JobContext.QUEUE_NAME, schedulerQueue);
        }
        ClusterMode clusterMode = ProgramRunners.getClusterMode(options);
        Service mapReduceRuntimeService = new MapReduceRuntimeService(injector, cConf, hConf, mapReduce, spec, context, program.getJarLocation(), locationFactory, clusterMode, fieldLineageWriter);
        mapReduceRuntimeService.addListener(createRuntimeServiceListener(closeables), Threads.SAME_THREAD_EXECUTOR);
        ProgramController controller = new MapReduceProgramController(mapReduceRuntimeService, context);
        LOG.debug("Starting MapReduce Job: {}", context);
        // be running the job, but the data directory will be owned by cdap.
        if (MapReduceTaskContextProvider.isLocal(hConf) || UserGroupInformation.isSecurityEnabled()) {
            mapReduceRuntimeService.start();
        } else {
            ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), mapReduceRuntimeService);
        }
        return controller;
    } catch (Exception e) {
        closeAllQuietly(closeables);
        throw Throwables.propagate(e);
    }
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ClusterMode(io.cdap.cdap.app.guice.ClusterMode) Closeable(java.io.Closeable) ArrayList(java.util.ArrayList) MapReduce(io.cdap.cdap.api.mapreduce.MapReduce) DatasetFramework(io.cdap.cdap.data2.dataset2.DatasetFramework) NameMappedDatasetFramework(io.cdap.cdap.internal.app.runtime.workflow.NameMappedDatasetFramework) InstantiatorFactory(io.cdap.cdap.common.lang.InstantiatorFactory) MetricsFieldSetter(io.cdap.cdap.internal.app.runtime.MetricsFieldSetter) ProgramType(io.cdap.cdap.proto.ProgramType) RunId(org.apache.twill.api.RunId) ProgramController(io.cdap.cdap.app.runtime.ProgramController) MapReduceSpecification(io.cdap.cdap.api.mapreduce.MapReduceSpecification) Arguments(io.cdap.cdap.app.runtime.Arguments) MessagingService(io.cdap.cdap.messaging.MessagingService) Service(com.google.common.util.concurrent.Service) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) ProgramId(io.cdap.cdap.proto.id.ProgramId) BasicProgramContext(io.cdap.cdap.internal.app.runtime.BasicProgramContext) DataSetFieldSetter(io.cdap.cdap.internal.app.runtime.DataSetFieldSetter) PropertyFieldSetter(io.cdap.cdap.common.lang.PropertyFieldSetter) WorkflowProgramInfo(io.cdap.cdap.internal.app.runtime.workflow.WorkflowProgramInfo) PluginInstantiator(io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator) ProgramContextAware(io.cdap.cdap.data.ProgramContextAware)

Aggregations

MessagingService (io.cdap.cdap.messaging.MessagingService)64 Service (com.google.common.util.concurrent.Service)52 MetricsCollectionService (io.cdap.cdap.api.metrics.MetricsCollectionService)48 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)34 Injector (com.google.inject.Injector)24 ConfigModule (io.cdap.cdap.common.guice.ConfigModule)20 NoOpMetricsCollectionService (io.cdap.cdap.common.metrics.NoOpMetricsCollectionService)20 DatasetService (io.cdap.cdap.data2.datafabric.dataset.service.DatasetService)20 AbstractModule (com.google.inject.AbstractModule)18 DatasetOpExecutorService (io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutorService)18 MessagingServerRuntimeModule (io.cdap.cdap.messaging.guice.MessagingServerRuntimeModule)18 IOException (java.io.IOException)18 LocalLocationModule (io.cdap.cdap.common.guice.LocalLocationModule)16 AuthenticationContextModules (io.cdap.cdap.security.auth.context.AuthenticationContextModules)16 Test (org.junit.Test)16 MultiThreadMessagingContext (io.cdap.cdap.messaging.context.MultiThreadMessagingContext)14 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)14 DiscoveryServiceClient (org.apache.twill.discovery.DiscoveryServiceClient)14 Before (org.junit.Before)14 MetadataService (io.cdap.cdap.metadata.MetadataService)12