Search in sources :

Example 1 with RawMessage

use of io.cdap.cdap.messaging.data.RawMessage in project cdap by caskdata.

the class AuditPublishTest method fetchAuditMessages.

private List<AuditMessage> fetchAuditMessages() throws TopicNotFoundException, IOException {
    List<AuditMessage> result = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = messagingService.prepareFetch(auditTopic).fetch()) {
        while (iterator.hasNext()) {
            RawMessage message = iterator.next();
            result.add(GSON.fromJson(new String(message.getPayload(), StandardCharsets.UTF_8), AuditMessage.class));
        }
    }
    return result;
}
Also used : AuditMessage(io.cdap.cdap.proto.audit.AuditMessage) ArrayList(java.util.ArrayList) RawMessage(io.cdap.cdap.messaging.data.RawMessage)

Example 2 with RawMessage

use of io.cdap.cdap.messaging.data.RawMessage in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testMultiMaxSequence.

@Test
public void testMultiMaxSequence() throws IOException, InterruptedException {
    TopicId topicId = new NamespaceId("ns1").topic("t1");
    final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
    // This test the case when multiple StoreRequests combined exceeding the 65536 payload.
    // See testMaxSequence() for more details when it matters
    // Generate 3 StoreRequests, each with 43690 messages
    int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT / 3 * 2;
    int requestCount = 3;
    List<StoreRequest> requests = new ArrayList<>();
    for (int i = 0; i < requestCount; i++) {
        List<String> payloads = new ArrayList<>(msgCount);
        for (int j = 0; j < msgCount; j++) {
            payloads.add(Integer.toString(j));
        }
        requests.add(new TestStoreRequest(topicId, payloads));
    }
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    // We use a custom metrics collector here to make all the persist calls reached the same latch,
    // since we know that the ConcurrentMessageWriter will emit a metrics "persist.requested" after enqueued but
    // before flushing.
    // This will make all requests batched together
    final CountDownLatch latch = new CountDownLatch(requestCount);
    final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter, new MetricsCollector() {

        @Override
        public void increment(String metricName, long value) {
            if ("persist.requested".equals(metricName)) {
                latch.countDown();
                Uninterruptibles.awaitUninterruptibly(latch);
            }
        }

        @Override
        public void gauge(String metricName, long value) {
            LOG.info("MetricsContext.gauge: {} = {}", metricName, value);
        }
    });
    ExecutorService executor = Executors.newFixedThreadPool(3);
    for (final StoreRequest request : requests) {
        executor.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    writer.persist(request, metadata);
                } catch (IOException e) {
                    LOG.error("Failed to persist", e);
                }
            }
        });
    }
    executor.shutdown();
    Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
    // Validates all messages are being written
    List<RawMessage> messages = testWriter.getMessages().get(topicId);
    Assert.assertEquals(requestCount * msgCount, messages.size());
    // We expect the payload is in repeated sequence of [0..msgCount-1]
    int expectedPayload = 0;
    // The sequenceId should be (i % SEQUENCE_ID_LIMIT)
    for (int i = 0; i < messages.size(); i++) {
        RawMessage message = messages.get(i);
        MessageId messageId = new MessageId(message.getId());
        Assert.assertEquals(i / StoreRequestWriter.SEQUENCE_ID_LIMIT, messageId.getPublishTimestamp());
        Assert.assertEquals((short) (i % StoreRequestWriter.SEQUENCE_ID_LIMIT), messageId.getSequenceId());
        Assert.assertEquals(expectedPayload, Integer.parseInt(Bytes.toString(message.getPayload())));
        expectedPayload = (expectedPayload + 1) % msgCount;
    }
}
Also used : ArrayList(java.util.ArrayList) TopicId(io.cdap.cdap.proto.id.TopicId) RawMessage(io.cdap.cdap.messaging.data.RawMessage) MetricsCollector(io.cdap.cdap.api.metrics.MetricsCollector) TimeProvider(io.cdap.cdap.common.utils.TimeProvider) StoreRequest(io.cdap.cdap.messaging.StoreRequest) IOException(java.io.IOException) CountDownLatch(java.util.concurrent.CountDownLatch) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) ExecutorService(java.util.concurrent.ExecutorService) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Example 3 with RawMessage

use of io.cdap.cdap.messaging.data.RawMessage in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testBasic.

@Test
public void testBasic() throws IOException {
    TopicId topicId1 = new NamespaceId("ns1").topic("t1");
    TopicId topicId2 = new NamespaceId("ns2").topic("t2");
    TopicMetadata metadata1 = new TopicMetadata(topicId1, new HashMap<String, String>(), 1);
    TopicMetadata metadata2 = new TopicMetadata(topicId2, new HashMap<String, String>(), 1);
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter);
    writer.persist(new TestStoreRequest(topicId1, Arrays.asList("1", "2", "3")), metadata1);
    // There should be 3 messages being written
    List<RawMessage> messages = testWriter.getMessages().get(topicId1);
    Assert.assertEquals(3, messages.size());
    // All messages should be written with timestamp 0
    List<String> payloads = new ArrayList<>();
    for (RawMessage message : messages) {
        Assert.assertEquals(0L, new MessageId(message.getId()).getPublishTimestamp());
        payloads.add(Bytes.toString(message.getPayload()));
    }
    Assert.assertEquals(Arrays.asList("1", "2", "3"), payloads);
    // Write to another topic
    writer.persist(new TestStoreRequest(topicId2, Arrays.asList("a", "b", "c")), metadata2);
    // There should be 3 messages being written to topic2
    messages = testWriter.getMessages().get(topicId2);
    Assert.assertEquals(3, messages.size());
    // All messages should be written with timestamp 1
    payloads.clear();
    for (RawMessage message : messages) {
        Assert.assertEquals(1L, new MessageId(message.getId()).getPublishTimestamp());
        payloads.add(Bytes.toString(message.getPayload()));
    }
    Assert.assertEquals(Arrays.asList("a", "b", "c"), payloads);
}
Also used : TimeProvider(io.cdap.cdap.common.utils.TimeProvider) ArrayList(java.util.ArrayList) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) TopicId(io.cdap.cdap.proto.id.TopicId) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) RawMessage(io.cdap.cdap.messaging.data.RawMessage) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Example 4 with RawMessage

use of io.cdap.cdap.messaging.data.RawMessage in project cdap by caskdata.

the class MessagingHttpServiceTest method testLargePublish.

@Test
public void testLargePublish() throws IOException, TopicAlreadyExistsException, TopicNotFoundException, UnauthorizedException {
    // A 5MB message, which is larger than the 1MB buffer.
    String message = Strings.repeat("01234", 1024 * 1024);
    TopicId topicId = new NamespaceId("ns1").topic("testLargePublish");
    client.createTopic(new TopicMetadata(topicId));
    StoreRequest request = StoreRequestBuilder.of(topicId).addPayload(message).build();
    client.publish(request);
    // Read it back
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(10).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(1, messages.size());
    Assert.assertEquals(Collections.singletonList(message), messages.stream().map(RawMessage::getPayload).map(Bytes::toString).collect(Collectors.toList()));
}
Also used : Bytes(io.cdap.cdap.api.common.Bytes) StoreRequest(io.cdap.cdap.messaging.StoreRequest) ArrayList(java.util.ArrayList) TopicId(io.cdap.cdap.proto.id.TopicId) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) RawMessage(io.cdap.cdap.messaging.data.RawMessage) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) Test(org.junit.Test)

Example 5 with RawMessage

use of io.cdap.cdap.messaging.data.RawMessage in project cdap by caskdata.

the class MessagingHttpServiceTest method testChunkConsume.

@Test
public void testChunkConsume() throws Exception {
    // This test is to verify the message fetching body producer works correctly
    TopicId topicId = new NamespaceId("ns1").topic("testChunkConsume");
    client.createTopic(new TopicMetadata(topicId));
    // Publish 10 messages, each payload is half the size of the chunk size
    int payloadSize = cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_CONSUME_CHUNK_SIZE) / 2;
    for (int i = 0; i < 10; i++) {
        String payload = Strings.repeat(Integer.toString(i), payloadSize);
        client.publish(StoreRequestBuilder.of(topicId).addPayload(payload).build());
    }
    // Fetch messages. All of them should be fetched correctly
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(10, messages.size());
    for (int i = 0; i < 10; i++) {
        RawMessage message = messages.get(i);
        Assert.assertEquals(payloadSize, message.getPayload().length);
        String payload = Strings.repeat(Integer.toString(i), payloadSize);
        Assert.assertEquals(payload, Bytes.toString(message.getPayload()));
    }
    client.deleteTopic(topicId);
}
Also used : ArrayList(java.util.ArrayList) TopicId(io.cdap.cdap.proto.id.TopicId) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) RawMessage(io.cdap.cdap.messaging.data.RawMessage) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) Test(org.junit.Test)

Aggregations

RawMessage (io.cdap.cdap.messaging.data.RawMessage)17 TopicId (io.cdap.cdap.proto.id.TopicId)16 ArrayList (java.util.ArrayList)14 Test (org.junit.Test)14 TopicMetadata (io.cdap.cdap.messaging.TopicMetadata)12 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)11 TimeProvider (io.cdap.cdap.common.utils.TimeProvider)4 MessageId (io.cdap.cdap.messaging.data.MessageId)4 IOException (java.io.IOException)4 StoreRequest (io.cdap.cdap.messaging.StoreRequest)3 Injector (com.google.inject.Injector)2 Bytes (io.cdap.cdap.api.common.Bytes)2 MessagingService (io.cdap.cdap.messaging.MessagingService)2 RollbackDetail (io.cdap.cdap.messaging.RollbackDetail)2 HashSet (java.util.HashSet)2 List (java.util.List)2 Map (java.util.Map)2 ILoggingEvent (ch.qos.logback.classic.spi.ILoggingEvent)1 Stopwatch (com.google.common.base.Stopwatch)1 ImmutableMap (com.google.common.collect.ImmutableMap)1