Examples with RawMessage co.cask.cdap.messaging.data.RawMessage used on opensource projects

Example 11 with RawMessage

use of co.cask.cdap.messaging.data.RawMessage in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testConcurrentWrites.

@Test
public void testConcurrentWrites() throws InterruptedException, BrokenBarrierException {
    int payloadsPerRequest = 200;
    int threadCount = 20;
    final int requestPerThread = 20;
    long writeLatencyMillis = 50L;
    final TopicId topicId = NamespaceId.DEFAULT.topic("t");
    final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider(), writeLatencyMillis);
    final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter);
    final List<String> payload = new ArrayList<>(payloadsPerRequest);
    for (int i = 0; i < payloadsPerRequest; i++) {
        payload.add(Integer.toString(i));
    }
    ExecutorService executor = Executors.newFixedThreadPool(threadCount);
    final CyclicBarrier barrier = new CyclicBarrier(threadCount + 1);
    for (int i = 0; i < threadCount; i++) {
        final int threadId = i;
        executor.submit(new Runnable() {

            @Override
            public void run() {
                Stopwatch stopwatch = new Stopwatch();
                try {
                    barrier.await();
                    stopwatch.start();
                    for (int i = 0; i < requestPerThread; i++) {
                        writer.persist(new TestStoreRequest(topicId, payload), metadata);
                    }
                    LOG.info("Complete time for thread {} is {} ms", threadId, stopwatch.elapsedMillis());
                } catch (Exception e) {
                    LOG.error("Exception raised when persisting.", e);
                }
            }
        });
    }
    Stopwatch stopwatch = new Stopwatch();
    barrier.await();
    stopwatch.start();
    executor.shutdown();
    Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
    LOG.info("Total time passed: {} ms", stopwatch.elapsedMillis());
    // Validate that the total number of messages written is correct
    List<RawMessage> messages = testWriter.getMessages().get(topicId);
    Assert.assertEquals(payloadsPerRequest * threadCount * requestPerThread, messages.size());
    // The message id must be sorted
    RawMessage lastMessage = null;
    for (RawMessage message : messages) {
        if (lastMessage != null) {
            Assert.assertTrue(Bytes.compareTo(lastMessage.getId(), message.getId()) < 0);
        }
        lastMessage = message;
    }
}

Example 12 with RawMessage

use of co.cask.cdap.messaging.data.RawMessage in project cdap by caskdata.

the class MessagingHttpServiceTest method testChunkConsume.

@Test
public void testChunkConsume() throws Exception {
    // This test is to verify the message fetching body producer works correctly
    TopicId topicId = new NamespaceId("ns1").topic("testChunkConsume");
    client.createTopic(new TopicMetadata(topicId));
    // Publish 10 messages, each payload is half the size of the chunk size
    int payloadSize = cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_CONSUME_CHUNK_SIZE) / 2;
    for (int i = 0; i < 10; i++) {
        String payload = Strings.repeat(Integer.toString(i), payloadSize);
        client.publish(StoreRequestBuilder.of(topicId).addPayloads(payload).build());
    }
    // Fetch messages. All of them should be fetched correctly
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(10, messages.size());
    for (int i = 0; i < 10; i++) {
        RawMessage message = messages.get(i);
        Assert.assertEquals(payloadSize, message.getPayload().length);
        String payload = Strings.repeat(Integer.toString(i), payloadSize);
        Assert.assertEquals(payload, Bytes.toString(message.getPayload()));
    }
    client.deleteTopic(topicId);
}

Example 13 with RawMessage

use of co.cask.cdap.messaging.data.RawMessage in project cdap by caskdata.

the class MessagingHttpServiceTest method testPayloadTable.

@Test
public void testPayloadTable() throws Exception {
    // This test is to verify storing transaction messages to the payload table
    TopicId topicId = new NamespaceId("ns1").topic("testPayloadTable");
    client.createTopic(new TopicMetadata(topicId));
    // Try to store to Payload table with empty iterator, expected failure
    try {
        client.storePayload(StoreRequestBuilder.of(topicId).setTransaction(1L).build());
        Assert.fail("Expected IllegalArgumentException");
    } catch (IllegalArgumentException e) {
    // Expected
    }
    // Store 20 payloads to the payload table, with 2 payloads per request
    for (int i = 0; i < 10; i++) {
        String payload = Integer.toString(i);
        client.storePayload(StoreRequestBuilder.of(topicId).addPayloads(payload, payload).setTransaction(1L).build());
    }
    // Try to consume and there should be no messages
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Publish an empty payload message to the message table. This simulates a tx commit.
    client.publish(StoreRequestBuilder.of(topicId).setTransaction(1L).build());
    // Consume again and there should be 20 messages
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(20, messages.size());
    for (int i = 0; i < 20; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString(i / 2), payload1);
    }
    // Should get the last 10 messages
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(10).getId(), true).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(10, messages.size());
    for (int i = 0; i < 10; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString((i + 10) / 2), payload1);
    }
    // We start with the 12th message id as offset, hence should get 8 messages.
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(1).getId(), false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(8, messages.size());
    for (int i = 0; i < 8; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString((i + 12) / 2), payload1);
    }
    // Fetch with the last message id in the payload table, exclusively. Should get an empty iterator
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(messages.size() - 1).getId(), false).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Consume with a limit
    messages.clear();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(6).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(6, messages.size());
    for (int i = 0; i < 6; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString(i / 2), payload1);
    }
    // Store and publish two more payloads
    String payload = Integer.toString(10);
    client.storePayload(StoreRequestBuilder.of(topicId).addPayloads(payload, payload).setTransaction(2L).build());
    client.publish(StoreRequestBuilder.of(topicId).setTransaction(2L).build());
    // Should get 22 messages
    messages.clear();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(22, messages.size());
    for (int i = 0; i < 22; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString(i / 2), payload1);
    }
    // Should get 2 messages back
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(19).getId(), false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    String payload1 = Bytes.toString(messages.get(0).getPayload());
    String payload2 = Bytes.toString(messages.get(1).getPayload());
    Assert.assertEquals(payload1, payload2);
    Assert.assertEquals(Integer.toString(10), payload1);
    client.deleteTopic(topicId);
}

Example 14 with RawMessage

use of co.cask.cdap.messaging.data.RawMessage in project cdap by caskdata.

the class FetchHandler method poll.

@POST
@Path("poll")
public void poll(FullHttpRequest request, HttpResponder responder, @PathParam("namespace") String namespace, @PathParam("topic") String topic) throws Exception {
    TopicId topicId = new NamespaceId(namespace).topic(topic);
    // Currently only support avro
    if (!"avro/binary".equals(request.headers().get(HttpHeaderNames.CONTENT_TYPE))) {
        throw new BadRequestException("Only avro/binary content type is supported.");
    }
    // Decode the poll request
    Decoder decoder = DecoderFactory.get().directBinaryDecoder(new ByteBufInputStream(request.content()), null);
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(Schemas.V1.ConsumeRequest.SCHEMA);
    // Fetch the messages
    CloseableIterator<RawMessage> iterator = fetchMessages(datumReader.read(null, decoder), topicId);
    try {
        responder.sendContent(HttpResponseStatus.OK, new MessagesBodyProducer(iterator, messageChunkSize), new DefaultHttpHeaders().set(HttpHeaderNames.CONTENT_TYPE, "avro/binary"));
    } catch (Throwable t) {
        iterator.close();
        throw t;
    }
}

Example 15 with RawMessage

use of co.cask.cdap.messaging.data.RawMessage in project cdap by caskdata.

the class AuditPublishTest method fetchAuditMessages.

private List<AuditMessage> fetchAuditMessages() throws TopicNotFoundException, IOException {
    List<AuditMessage> result = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = messagingService.prepareFetch(auditTopic).fetch()) {
        while (iterator.hasNext()) {
            RawMessage message = iterator.next();
            result.add(GSON.fromJson(new String(message.getPayload(), StandardCharsets.UTF_8), AuditMessage.class));
        }
    }
    return result;
}