Examples with MessagingService io.cdap.cdap.messaging.MessagingService used on opensource projects

Example 21 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class LeaderElectionMessagingServiceTest method testTransition.

@Test
public void testTransition() throws Throwable {
    final TopicId topicId = NamespaceId.SYSTEM.topic("topic");
    Injector injector1 = createInjector(0);
    Injector injector2 = createInjector(1);
    // Start a messaging service, which would becomes leader
    ZKClientService zkClient1 = injector1.getInstance(ZKClientService.class);
    zkClient1.startAndWait();
    final MessagingService firstService = injector1.getInstance(MessagingService.class);
    if (firstService instanceof Service) {
        ((Service) firstService).startAndWait();
    }
    // Publish a message with the leader
    firstService.publish(StoreRequestBuilder.of(topicId).addPayload("Testing1").build());
    // Start another messaging service, this one would be follower
    ZKClientService zkClient2 = injector2.getInstance(ZKClientService.class);
    zkClient2.startAndWait();
    final MessagingService secondService = injector2.getInstance(MessagingService.class);
    if (secondService instanceof Service) {
        ((Service) secondService).startAndWait();
    }
    // Try to call the follower, should get service unavailable.
    try {
        secondService.listTopics(NamespaceId.SYSTEM);
        Assert.fail("Expected service unavailable");
    } catch (ServiceUnavailableException e) {
    // Expected
    }
    // Make the ZK session timeout for the leader service. The second one should pickup.
    KillZKSession.kill(zkClient1.getZooKeeperSupplier().get(), zkClient1.getConnectString(), 10000);
    // Publish one more message and then fetch from the current leader
    List<String> messages = Retries.callWithRetries(new Retries.Callable<List<String>, Throwable>() {

        @Override
        public List<String> call() throws Throwable {
            secondService.publish(StoreRequestBuilder.of(topicId).addPayload("Testing2").build());
            List<String> messages = new ArrayList<>();
            try (CloseableIterator<RawMessage> iterator = secondService.prepareFetch(topicId).fetch()) {
                while (iterator.hasNext()) {
                    messages.add(new String(iterator.next().getPayload(), "UTF-8"));
                }
            }
            return messages;
        }
    }, RetryStrategies.timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS)));
    Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages);
    // Shutdown the current leader. The session timeout one should becomes leader again.
    if (secondService instanceof Service) {
        ((Service) secondService).stopAndWait();
    }
    // Try to fetch message from the current leader again.
    // Should see two messages (because the cache is cleared and fetch is from the backing store).
    messages = Retries.callWithRetries(new Retries.Callable<List<String>, Throwable>() {

        @Override
        public List<String> call() throws Throwable {
            List<String> messages = new ArrayList<>();
            try (CloseableIterator<RawMessage> iterator = firstService.prepareFetch(topicId).fetch()) {
                while (iterator.hasNext()) {
                    messages.add(new String(iterator.next().getPayload(), "UTF-8"));
                }
            }
            return messages;
        }
    }, RetryStrategies.timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS)));
    Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages);
    zkClient1.stopAndWait();
    zkClient2.stopAndWait();
}

Example 22 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class MetricsAdminSubscriberServiceTest method init.

@BeforeClass
public static void init() throws IOException {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    // Shorten delay to speed up test
    cConf.setLong(Constants.Metrics.ADMIN_POLL_DELAY_MILLIS, 100L);
    injector = Guice.createInjector(new ConfigModule(cConf), new IOModule(), new InMemoryDiscoveryModule(), new MessagingServerRuntimeModule().getStandaloneModules(), new SystemDatasetRuntimeModule().getStandaloneModules(), // distributed mode. It requires bindings that are too cumbersome to construct them one by one.
    new PrivateModule() {

        @Override
        protected void configure() {
            install(new MetricsHandlerModule());
            expose(MetricsQueryService.class);
            install(new MetricsStoreModule());
            bind(MetricsCollectionService.class).to(LocalMetricsCollectionService.class).in(Scopes.SINGLETON);
            expose(MetricsCollectionService.class);
            // Bind the RemoteMetricsSystemClient for testing.
            bind(MetricsSystemClient.class).to(DirectMetricsSystemClient.class);
            expose(MetricsSystemClient.class);
            // Bind the admin subscriber
            bind(MetricsAdminSubscriberService.class).in(Scopes.SINGLETON);
            expose(MetricsAdminSubscriberService.class);
        }
    });
    messagingService = injector.getInstance(MessagingService.class);
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsQueryService = injector.getInstance(MetricsQueryService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    metricsCollectionService.startAndWait();
    metricsQueryService.startAndWait();
}

Example 23 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class MetricsTestBase method init.

@Before
public void init() throws IOException, UnsupportedTypeException {
    cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.Metrics.TOPIC_PREFIX, TOPIC_PREFIX);
    cConf.setInt(Constants.Metrics.MESSAGING_TOPIC_NUM, 10);
    cConf.setInt(Constants.Metrics.QUEUE_SIZE, 1000);
    // Set it to really short delay for faster test
    cConf.setLong(Constants.Metrics.PROCESSOR_MAX_DELAY_MS, 5);
    injector = Guice.createInjector(getModules());
    messagingService = injector.getInstance(MessagingService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    metricValueType = TypeToken.of(MetricValues.class);
    schema = new ReflectionSchemaGenerator().generate(metricValueType.getType());
    recordWriter = new ASMDatumWriterFactory(new ASMFieldAccessorFactory()).create(metricValueType, schema);
}

Example 24 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class AppFabricTestBase method initializeAndStartServices.

protected static void initializeAndStartServices(CConfiguration cConf, Module overrides) throws Exception {
    injector = Guice.createInjector(Modules.override(new AppFabricTestModule(cConf, null)).with(overrides));
    int connectionTimeout = cConf.getInt(Constants.HTTP_CLIENT_CONNECTION_TIMEOUT_MS);
    int readTimeout = cConf.getInt(Constants.HTTP_CLIENT_READ_TIMEOUT_MS);
    httpRequestConfig = new HttpRequestConfig(connectionTimeout, readTimeout, false);
    messagingService = injector.getInstance(MessagingService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    // Define all StructuredTable before starting any services that need StructuredTable
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    metadataStorage = injector.getInstance(MetadataStorage.class);
    metadataStorage.createIndex();
    dsOpService = injector.getInstance(DatasetOpExecutorService.class);
    dsOpService.startAndWait();
    datasetService = injector.getInstance(DatasetService.class);
    datasetService.startAndWait();
    appFabricServer = injector.getInstance(AppFabricServer.class);
    appFabricServer.startAndWait();
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    appFabricEndpointStrategy = new RandomEndpointStrategy(() -> discoveryClient.discover(Constants.Service.APP_FABRIC_HTTP));
    txClient = injector.getInstance(TransactionSystemClient.class);
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsCollectionService.startAndWait();
    serviceStore = injector.getInstance(ServiceStore.class);
    serviceStore.startAndWait();
    metadataService = injector.getInstance(MetadataService.class);
    metadataService.startAndWait();
    metadataSubscriberService = injector.getInstance(MetadataSubscriberService.class);
    metadataSubscriberService.startAndWait();
    logQueryService = injector.getInstance(LogQueryService.class);
    logQueryService.startAndWait();
    locationFactory = getInjector().getInstance(LocationFactory.class);
    datasetClient = new DatasetClient(getClientConfig(discoveryClient, Constants.Service.DATASET_MANAGER));
    remoteClientFactory = new RemoteClientFactory(discoveryClient, new DefaultInternalAuthenticator(new AuthenticationTestContext()));
    metadataClient = new MetadataClient(getClientConfig(discoveryClient, Constants.Service.METADATA_SERVICE));
    metadataServiceClient = new DefaultMetadataServiceClient(remoteClientFactory);
    metricStore = injector.getInstance(MetricStore.class);
    Scheduler programScheduler = injector.getInstance(Scheduler.class);
    // Wait for the scheduler to be functional.
    if (programScheduler instanceof CoreSchedulerService) {
        try {
            ((CoreSchedulerService) programScheduler).waitUntilFunctional(10, TimeUnit.SECONDS);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    createNamespaces();
}

Example 25 with MessagingService

use of io.cdap.cdap.messaging.MessagingService in project cdap by cdapio.

the class MessagingServiceMainTest method testMessagingService.

@Test
public void testMessagingService() throws Exception {
    // Discover the TMS endpoint
    Injector injector = getServiceMainInstance(MessagingServiceMain.class).getInjector();
    RemoteClientFactory remoteClientFactory = injector.getInstance(RemoteClientFactory.class);
    // Use a separate TMS client to create topic, then publish and then poll some messages
    TopicId topicId = NamespaceId.SYSTEM.topic("test");
    MessagingService messagingService = new ClientMessagingService(remoteClientFactory, true);
    messagingService.createTopic(new TopicMetadata(topicId));
    // Publish 10 messages
    List<String> messages = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msg = "Testing Message " + i;
        messagingService.publish(StoreRequestBuilder.of(topicId).addPayload(msg).build());
        messages.add(msg);
    }
    try (CloseableIterator<RawMessage> iterator = messagingService.prepareFetch(topicId).setLimit(10).fetch()) {
        List<String> received = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, 0), false).map(RawMessage::getPayload).map(ByteBuffer::wrap).map(StandardCharsets.UTF_8::decode).map(CharSequence::toString).collect(Collectors.toList());
        Assert.assertEquals(messages, received);
    }
}