Examples with TopicId io.cdap.cdap.proto.id.TopicId used on opensource projects

Example 16 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class MetadataTableTest method testCRUD.

@Test
public void testCRUD() throws Exception {
    try (MetadataTable table = createMetadataTable()) {
        TopicId topicId = NamespaceId.DEFAULT.topic("topic");
        // Update a non-existing topic should fail.
        try {
            table.updateTopic(new TopicMetadata(topicId, "ttl", 10));
            Assert.fail("Expected TopicNotFoundException");
        } catch (TopicNotFoundException e) {
        // Expected
        }
        // Create a topic and validate
        table.createTopic(new TopicMetadata(topicId, "ttl", 10));
        Assert.assertEquals(10, table.getMetadata(topicId).getTTL());
        // Update the property and validate
        table.updateTopic(new TopicMetadata(topicId, "ttl", 30));
        Assert.assertEquals(30, table.getMetadata(topicId).getTTL());
        // Create the same topic again should fail
        try {
            table.createTopic(new TopicMetadata(topicId, "ttl", 10));
            Assert.fail("Expected TopicAlreadyExistsException");
        } catch (TopicAlreadyExistsException e) {
        // Expected
        }
        // It shouldn't affect the topic at all if creation failed
        Assert.assertEquals(30, table.getMetadata(topicId).getTTL());
        // Delete the topic
        table.deleteTopic(topicId);
        try {
            table.getMetadata(topicId);
            Assert.fail("Expected TopicNotFoundException");
        } catch (TopicNotFoundException e) {
        // Expected
        }
        // Delete again should raise a TopicNotFoundException
        try {
            table.deleteTopic(topicId);
            Assert.fail("Expected TopicNotFoundException");
        } catch (TopicNotFoundException e) {
        // Expected
        }
    }
}

Example 17 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class MessagingHttpServiceTest method testLargePublish.

@Test
public void testLargePublish() throws IOException, TopicAlreadyExistsException, TopicNotFoundException, UnauthorizedException {
    // A 5MB message, which is larger than the 1MB buffer.
    String message = Strings.repeat("01234", 1024 * 1024);
    TopicId topicId = new NamespaceId("ns1").topic("testLargePublish");
    client.createTopic(new TopicMetadata(topicId));
    StoreRequest request = StoreRequestBuilder.of(topicId).addPayload(message).build();
    client.publish(request);
    // Read it back
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(10).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(1, messages.size());
    Assert.assertEquals(Collections.singletonList(message), messages.stream().map(RawMessage::getPayload).map(Bytes::toString).collect(Collectors.toList()));
}

Example 18 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class MessagingHttpServiceTest method testChunkConsume.

@Test
public void testChunkConsume() throws Exception {
    // This test is to verify the message fetching body producer works correctly
    TopicId topicId = new NamespaceId("ns1").topic("testChunkConsume");
    client.createTopic(new TopicMetadata(topicId));
    // Publish 10 messages, each payload is half the size of the chunk size
    int payloadSize = cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_CONSUME_CHUNK_SIZE) / 2;
    for (int i = 0; i < 10; i++) {
        String payload = Strings.repeat(Integer.toString(i), payloadSize);
        client.publish(StoreRequestBuilder.of(topicId).addPayload(payload).build());
    }
    // Fetch messages. All of them should be fetched correctly
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(10, messages.size());
    for (int i = 0; i < 10; i++) {
        RawMessage message = messages.get(i);
        Assert.assertEquals(payloadSize, message.getPayload().length);
        String payload = Strings.repeat(Integer.toString(i), payloadSize);
        Assert.assertEquals(payload, Bytes.toString(message.getPayload()));
    }
    client.deleteTopic(topicId);
}

Example 19 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class MessagingHttpServiceTest method testBasicPubSub.

@Test
public void testBasicPubSub() throws Exception {
    TopicId topicId = new NamespaceId("ns1").topic("testBasicPubSub");
    // Publish to a non-existing topic should get not found exception
    try {
        client.publish(StoreRequestBuilder.of(topicId).addPayload("a").build());
        Assert.fail("Expected TopicNotFoundException");
    } catch (TopicNotFoundException e) {
    // Expected
    }
    // Consume from a non-existing topic should get not found exception
    try {
        client.prepareFetch(topicId).fetch();
        Assert.fail("Expected TopicNotFoundException");
    } catch (TopicNotFoundException e) {
    // Expected
    }
    client.createTopic(new TopicMetadata(topicId));
    // Publish a non-transactional message with empty payload should result in failure
    try {
        client.publish(StoreRequestBuilder.of(topicId).build());
        Assert.fail("Expected IllegalArgumentException");
    } catch (IllegalArgumentException e) {
    // Expected
    }
    // Publish a non-tx message, no RollbackDetail is returned
    Assert.assertNull(client.publish(StoreRequestBuilder.of(topicId).addPayload("m0").addPayload("m1").build()));
    // Publish a transactional message, a RollbackDetail should be returned
    RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topicId).addPayload("m2").setTransaction(1L).build());
    Assert.assertNotNull(rollbackDetail);
    // Rollback the published message
    client.rollback(topicId, rollbackDetail);
    // Fetch messages non-transactionally (should be able to read all the messages since rolled back messages
    // are still visible until ttl kicks in)
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(3, messages.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Consume transactionally. It should get only m0 and m1 since m2 has been rolled back
    List<RawMessage> txMessages = new ArrayList<>();
    Transaction transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(0).setTransaction(transaction).fetch()) {
        Iterators.addAll(txMessages, iterator);
    }
    Assert.assertEquals(2, txMessages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Fetch again from a given message offset exclusively.
    // Expects one message to be fetched
    byte[] startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        // It should have only one message (m2)
        Assert.assertTrue(iterator.hasNext());
        RawMessage msg = iterator.next();
        Assert.assertEquals("m2", Bytes.toString(msg.getPayload()));
    }
    // Fetch again from the last message offset exclusively
    // Expects no message to be fetched
    startMessageId = messages.get(2).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Fetch with start time. It should get both m0 and m1 since they are published in the same request, hence
    // having the same publish time
    startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(new MessageId(startMessageId).getPublishTimestamp()).setLimit(2).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Publish 2 messages, one transactionally, one without transaction
    client.publish(StoreRequestBuilder.of(topicId).addPayload("m3").setTransaction(2L).build());
    client.publish(StoreRequestBuilder.of(topicId).addPayload("m4").build());
    // Consume without transactional, it should see m2, m3 and m4
    startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(3, messages.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals("m" + (i + 2), Bytes.toString(messages.get(i).getPayload()));
    }
    // Consume using a transaction that doesn't have tx = 2L visible. It should get no message as it should block on m3
    transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Consume using a transaction that has tx = 2L in the invalid list. It should skip m3 and got m4
    transaction = new Transaction(3L, 3L, new long[] { 2L }, new long[0], 0L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(1, messages.size());
    Assert.assertEquals("m4", Bytes.toString(messages.get(0).getPayload()));
    // Consume using a transaction that has tx = 2L committed. It should get m3 and m4
    transaction = new Transaction(3L, 3L, new long[0], new long[0], 0L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + (i + 3), Bytes.toString(messages.get(i).getPayload()));
    }
    client.deleteTopic(topicId);
}

Example 20 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class MessagingHttpServiceTest method testReuseRequest.

@Test
public void testReuseRequest() throws IOException, TopicAlreadyExistsException, TopicNotFoundException, UnauthorizedException {
    // This test a StoreRequest object can be reused.
    // This test is to verify storing transaction messages to the payload table
    TopicId topicId = new NamespaceId("ns1").topic("testReuseRequest");
    client.createTopic(new TopicMetadata(topicId));
    StoreRequest request = StoreRequestBuilder.of(topicId).addPayload("m1").addPayload("m2").build();
    // Publish the request twice
    client.publish(request);
    client.publish(request);
    // Expects four messages
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(10).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(4, messages.size());
    List<String> expected = Arrays.asList("m1", "m2", "m1", "m2");
    Assert.assertEquals(expected, messages.stream().map(RawMessage::getPayload).map(Bytes::toString).collect(Collectors.toList()));
}