Examples with RawMessage io.cdap.cdap.messaging.data.RawMessage used on opensource projects

Example 11 with RawMessage

use of io.cdap.cdap.messaging.data.RawMessage in project cdap by cdapio.

the class TestTMSLogging method testTmsLogAppender.

@Test
public void testTmsLogAppender() throws Exception {
    // setup TMSLogAppender and log messages to it
    LogAppenderInitializer logAppenderInitializer = new LogAppenderInitializer(tmsLogAppender);
    logAppenderInitializer.initialize("TestTMSLogging");
    Logger logger = LoggerFactory.getLogger("TestTMSLogging");
    LoggingTester loggingTester = new LoggingTester();
    LoggingContext loggingContext = new MapReduceLoggingContext("TKL_NS_1", "APP_1", "MR_1", "RUN1");
    loggingTester.generateLogs(logger, loggingContext);
    logAppenderInitializer.close();
    // fetch and deserialize all the logs from TMS
    LoggingEventSerializer loggingEventSerializer = new LoggingEventSerializer();
    Map<Integer, List<ILoggingEvent>> partitionedFetchedLogs = new HashMap<>();
    int totalFetchedLogs = 0;
    for (Map.Entry<Integer, TopicId> topicId : topicIds.entrySet()) {
        List<ILoggingEvent> fetchedLogs = new ArrayList<>();
        MessageFetcher messageFetcher = client.prepareFetch(topicId.getValue());
        try (CloseableIterator<RawMessage> messages = messageFetcher.fetch()) {
            while (messages.hasNext()) {
                RawMessage message = messages.next();
                ILoggingEvent iLoggingEvent = loggingEventSerializer.fromBytes(ByteBuffer.wrap(message.getPayload()));
                fetchedLogs.add(iLoggingEvent);
            }
        }
        totalFetchedLogs += fetchedLogs.size();
        partitionedFetchedLogs.put(topicId.getKey(), fetchedLogs);
    }
    // LoggingTester emits 240 logs in total
    Assert.assertEquals(240, totalFetchedLogs);
    // Read the partition that our LoggingContext maps to and filter the logs in there to the logs that correspond
    // to our LoggingContext.
    LogPartitionType logPartitionType = LogPartitionType.valueOf(cConf.get(Constants.Logging.LOG_PUBLISH_PARTITION_KEY).toUpperCase());
    String partitionKey = logPartitionType.getPartitionKey(loggingContext);
    int partition = TMSLogAppender.partition(partitionKey, cConf.getInt(Constants.Logging.NUM_PARTITIONS));
    Filter logFilter = LoggingContextHelper.createFilter(loggingContext);
    List<ILoggingEvent> filteredLogs = partitionedFetchedLogs.get(partition).stream().filter(logFilter::match).collect(Collectors.toList());
    // LoggingTester emits 60 logs with the given LoggingContext
    Assert.assertEquals(60, filteredLogs.size());
    for (int i = 0; i < filteredLogs.size(); i++) {
        ILoggingEvent loggingEvent = filteredLogs.get(i);
        Assert.assertEquals(String.format("Test log message %s arg1 arg2", i), loggingEvent.getFormattedMessage());
    }
}

Example 12 with RawMessage

use of io.cdap.cdap.messaging.data.RawMessage in project cdap by cdapio.

the class MessagingServiceMainTest method testMessagingService.

@Test
public void testMessagingService() throws Exception {
    // Discover the TMS endpoint
    Injector injector = getServiceMainInstance(MessagingServiceMain.class).getInjector();
    RemoteClientFactory remoteClientFactory = injector.getInstance(RemoteClientFactory.class);
    // Use a separate TMS client to create topic, then publish and then poll some messages
    TopicId topicId = NamespaceId.SYSTEM.topic("test");
    MessagingService messagingService = new ClientMessagingService(remoteClientFactory, true);
    messagingService.createTopic(new TopicMetadata(topicId));
    // Publish 10 messages
    List<String> messages = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String msg = "Testing Message " + i;
        messagingService.publish(StoreRequestBuilder.of(topicId).addPayload(msg).build());
        messages.add(msg);
    }
    try (CloseableIterator<RawMessage> iterator = messagingService.prepareFetch(topicId).setLimit(10).fetch()) {
        List<String> received = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, 0), false).map(RawMessage::getPayload).map(ByteBuffer::wrap).map(StandardCharsets.UTF_8::decode).map(CharSequence::toString).collect(Collectors.toList());
        Assert.assertEquals(messages, received);
    }
}

Example 13 with RawMessage

use of io.cdap.cdap.messaging.data.RawMessage in project cdap by cdapio.

the class MessagingHttpServiceTest method testPayloadTable.

@Test
public void testPayloadTable() throws Exception {
    // This test is to verify storing transaction messages to the payload table
    TopicId topicId = new NamespaceId("ns1").topic("testPayloadTable");
    client.createTopic(new TopicMetadata(topicId));
    // Try to store to Payload table with empty iterator, expected failure
    try {
        client.storePayload(StoreRequestBuilder.of(topicId).setTransaction(1L).build());
        Assert.fail("Expected IllegalArgumentException");
    } catch (IllegalArgumentException e) {
    // Expected
    }
    // Store 20 payloads to the payload table, with 2 payloads per request
    for (int i = 0; i < 10; i++) {
        String payload = Integer.toString(i);
        client.storePayload(StoreRequestBuilder.of(topicId).addPayload(payload).addPayload(payload).setTransaction(1L).build());
    }
    // Try to consume and there should be no messages
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Publish an empty payload message to the message table. This simulates a tx commit.
    client.publish(StoreRequestBuilder.of(topicId).setTransaction(1L).build());
    // Consume again and there should be 20 messages
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(20, messages.size());
    for (int i = 0; i < 20; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString(i / 2), payload1);
    }
    // Should get the last 10 messages
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(10).getId(), true).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(10, messages.size());
    for (int i = 0; i < 10; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString((i + 10) / 2), payload1);
    }
    // We start with the 12th message id as offset, hence should get 8 messages.
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(1).getId(), false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(8, messages.size());
    for (int i = 0; i < 8; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString((i + 12) / 2), payload1);
    }
    // Fetch with the last message id in the payload table, exclusively. Should get an empty iterator
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(messages.size() - 1).getId(), false).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Consume with a limit
    messages.clear();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(6).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(6, messages.size());
    for (int i = 0; i < 6; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString(i / 2), payload1);
    }
    // Store and publish two more payloads
    String payload = Integer.toString(10);
    client.storePayload(StoreRequestBuilder.of(topicId).addPayload(payload).addPayload(payload).setTransaction(2L).build());
    client.publish(StoreRequestBuilder.of(topicId).setTransaction(2L).build());
    // Should get 22 messages
    messages.clear();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(22, messages.size());
    for (int i = 0; i < 22; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString(i / 2), payload1);
    }
    // Should get 2 messages back
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(19).getId(), false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    String payload1 = Bytes.toString(messages.get(0).getPayload());
    String payload2 = Bytes.toString(messages.get(1).getPayload());
    Assert.assertEquals(payload1, payload2);
    Assert.assertEquals(Integer.toString(10), payload1);
    client.deleteTopic(topicId);
}

Example 14 with RawMessage

use of io.cdap.cdap.messaging.data.RawMessage in project cdap by cdapio.

the class MessagingHttpServiceTest method testTxMaxLifeTime.

@Test
public void testTxMaxLifeTime() throws Exception {
    NamespaceId nsId = new NamespaceId("txCheck");
    TopicId topic1 = nsId.topic("t1");
    // Create a topic
    client.createTopic(new TopicMetadata(topic1));
    final RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topic1).setTransaction(1L).addPayload("a").addPayload("b").build());
    try {
        client.publish(StoreRequestBuilder.of(topic1).setTransaction(-Long.MAX_VALUE).addPayload("c").addPayload("d").build());
        Assert.fail("Expected IOException");
    } catch (IOException ex) {
    // expected
    }
    Set<String> msgs = new HashSet<>();
    CloseableIterator<RawMessage> messages = client.prepareFetch(topic1).fetch();
    while (messages.hasNext()) {
        RawMessage message = messages.next();
        msgs.add(Bytes.toString(message.getPayload()));
    }
    Assert.assertEquals(2, msgs.size());
    Assert.assertTrue(msgs.contains("a"));
    Assert.assertTrue(msgs.contains("b"));
    messages.close();
    client.rollback(topic1, rollbackDetail);
    client.deleteTopic(topic1);
}

Example 15 with RawMessage

use of io.cdap.cdap.messaging.data.RawMessage in project cdap by cdapio.

the class MessagingHttpServiceTest method testBasicPubSub.

@Test
public void testBasicPubSub() throws Exception {
    TopicId topicId = new NamespaceId("ns1").topic("testBasicPubSub");
    // Publish to a non-existing topic should get not found exception
    try {
        client.publish(StoreRequestBuilder.of(topicId).addPayload("a").build());
        Assert.fail("Expected TopicNotFoundException");
    } catch (TopicNotFoundException e) {
    // Expected
    }
    // Consume from a non-existing topic should get not found exception
    try {
        client.prepareFetch(topicId).fetch();
        Assert.fail("Expected TopicNotFoundException");
    } catch (TopicNotFoundException e) {
    // Expected
    }
    client.createTopic(new TopicMetadata(topicId));
    // Publish a non-transactional message with empty payload should result in failure
    try {
        client.publish(StoreRequestBuilder.of(topicId).build());
        Assert.fail("Expected IllegalArgumentException");
    } catch (IllegalArgumentException e) {
    // Expected
    }
    // Publish a non-tx message, no RollbackDetail is returned
    Assert.assertNull(client.publish(StoreRequestBuilder.of(topicId).addPayload("m0").addPayload("m1").build()));
    // Publish a transactional message, a RollbackDetail should be returned
    RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topicId).addPayload("m2").setTransaction(1L).build());
    Assert.assertNotNull(rollbackDetail);
    // Rollback the published message
    client.rollback(topicId, rollbackDetail);
    // Fetch messages non-transactionally (should be able to read all the messages since rolled back messages
    // are still visible until ttl kicks in)
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(3, messages.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Consume transactionally. It should get only m0 and m1 since m2 has been rolled back
    List<RawMessage> txMessages = new ArrayList<>();
    Transaction transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(0).setTransaction(transaction).fetch()) {
        Iterators.addAll(txMessages, iterator);
    }
    Assert.assertEquals(2, txMessages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Fetch again from a given message offset exclusively.
    // Expects one message to be fetched
    byte[] startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        // It should have only one message (m2)
        Assert.assertTrue(iterator.hasNext());
        RawMessage msg = iterator.next();
        Assert.assertEquals("m2", Bytes.toString(msg.getPayload()));
    }
    // Fetch again from the last message offset exclusively
    // Expects no message to be fetched
    startMessageId = messages.get(2).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Fetch with start time. It should get both m0 and m1 since they are published in the same request, hence
    // having the same publish time
    startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(new MessageId(startMessageId).getPublishTimestamp()).setLimit(2).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Publish 2 messages, one transactionally, one without transaction
    client.publish(StoreRequestBuilder.of(topicId).addPayload("m3").setTransaction(2L).build());
    client.publish(StoreRequestBuilder.of(topicId).addPayload("m4").build());
    // Consume without transactional, it should see m2, m3 and m4
    startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(3, messages.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals("m" + (i + 2), Bytes.toString(messages.get(i).getPayload()));
    }
    // Consume using a transaction that doesn't have tx = 2L visible. It should get no message as it should block on m3
    transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Consume using a transaction that has tx = 2L in the invalid list. It should skip m3 and got m4
    transaction = new Transaction(3L, 3L, new long[] { 2L }, new long[0], 0L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(1, messages.size());
    Assert.assertEquals("m4", Bytes.toString(messages.get(0).getPayload()));
    // Consume using a transaction that has tx = 2L committed. It should get m3 and m4
    transaction = new Transaction(3L, 3L, new long[0], new long[0], 0L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + (i + 3), Bytes.toString(messages.get(i).getPayload()));
    }
    client.deleteTopic(topicId);
}