Examples with TopicMetadata io.cdap.cdap.messaging.TopicMetadata used on opensource projects

Example 6 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class CoreMessagingService method rollback.

@Override
public void rollback(TopicId topicId, RollbackDetail rollbackDetail) throws TopicNotFoundException, IOException {
    TopicMetadata metadata = getTopic(topicId);
    Exception failure = null;
    try (MessageTable messageTable = createMessageTable(metadata)) {
        messageTable.rollback(metadata, rollbackDetail);
    } catch (Exception e) {
        failure = e;
    }
    // Throw if there is any failure in rollback.
    if (failure != null) {
        Throwables.propagateIfPossible(failure, TopicNotFoundException.class, IOException.class);
        throw Throwables.propagate(failure);
    }
}

Example 7 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class ClientMessagingService method getTopic.

@Override
public TopicMetadata getTopic(TopicId topicId) throws TopicNotFoundException, IOException, UnauthorizedException {
    HttpRequest request = remoteClient.requestBuilder(HttpMethod.GET, createTopicPath(topicId)).build();
    HttpResponse response = remoteClient.execute(request);
    if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
        throw new TopicNotFoundException(topicId.getNamespace(), topicId.getTopic());
    }
    handleError(response, "Failed to update topic " + topicId);
    Map<String, String> properties = GSON.fromJson(response.getResponseBodyAsString(), TOPIC_PROPERTY_TYPE);
    return new TopicMetadata(topicId, properties);
}

Example 8 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class MessagingHttpServiceTest method testChunkConsume.

@Test
public void testChunkConsume() throws Exception {
    // This test is to verify the message fetching body producer works correctly
    TopicId topicId = new NamespaceId("ns1").topic("testChunkConsume");
    client.createTopic(new TopicMetadata(topicId));
    // Publish 10 messages, each payload is half the size of the chunk size
    int payloadSize = cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_CONSUME_CHUNK_SIZE) / 2;
    for (int i = 0; i < 10; i++) {
        String payload = Strings.repeat(Integer.toString(i), payloadSize);
        client.publish(StoreRequestBuilder.of(topicId).addPayload(payload).build());
    }
    // Fetch messages. All of them should be fetched correctly
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(10, messages.size());
    for (int i = 0; i < 10; i++) {
        RawMessage message = messages.get(i);
        Assert.assertEquals(payloadSize, message.getPayload().length);
        String payload = Strings.repeat(Integer.toString(i), payloadSize);
        Assert.assertEquals(payload, Bytes.toString(message.getPayload()));
    }
    client.deleteTopic(topicId);
}

Example 9 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class MessagingHttpServiceTest method testLargePublish.

@Test
public void testLargePublish() throws IOException, TopicAlreadyExistsException, TopicNotFoundException, UnauthorizedException {
    // A 5MB message, which is larger than the 1MB buffer.
    String message = Strings.repeat("01234", 1024 * 1024);
    TopicId topicId = new NamespaceId("ns1").topic("testLargePublish");
    client.createTopic(new TopicMetadata(topicId));
    StoreRequest request = StoreRequestBuilder.of(topicId).addPayload(message).build();
    client.publish(request);
    // Read it back
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(10).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(1, messages.size());
    Assert.assertEquals(Collections.singletonList(message), messages.stream().map(RawMessage::getPayload).map(Bytes::toString).collect(Collectors.toList()));
}

Example 10 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class MessagingHttpServiceTest method testBasicPubSub.

@Test
public void testBasicPubSub() throws Exception {
    TopicId topicId = new NamespaceId("ns1").topic("testBasicPubSub");
    // Publish to a non-existing topic should get not found exception
    try {
        client.publish(StoreRequestBuilder.of(topicId).addPayload("a").build());
        Assert.fail("Expected TopicNotFoundException");
    } catch (TopicNotFoundException e) {
    // Expected
    }
    // Consume from a non-existing topic should get not found exception
    try {
        client.prepareFetch(topicId).fetch();
        Assert.fail("Expected TopicNotFoundException");
    } catch (TopicNotFoundException e) {
    // Expected
    }
    client.createTopic(new TopicMetadata(topicId));
    // Publish a non-transactional message with empty payload should result in failure
    try {
        client.publish(StoreRequestBuilder.of(topicId).build());
        Assert.fail("Expected IllegalArgumentException");
    } catch (IllegalArgumentException e) {
    // Expected
    }
    // Publish a non-tx message, no RollbackDetail is returned
    Assert.assertNull(client.publish(StoreRequestBuilder.of(topicId).addPayload("m0").addPayload("m1").build()));
    // Publish a transactional message, a RollbackDetail should be returned
    RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topicId).addPayload("m2").setTransaction(1L).build());
    Assert.assertNotNull(rollbackDetail);
    // Rollback the published message
    client.rollback(topicId, rollbackDetail);
    // Fetch messages non-transactionally (should be able to read all the messages since rolled back messages
    // are still visible until ttl kicks in)
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(3, messages.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Consume transactionally. It should get only m0 and m1 since m2 has been rolled back
    List<RawMessage> txMessages = new ArrayList<>();
    Transaction transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(0).setTransaction(transaction).fetch()) {
        Iterators.addAll(txMessages, iterator);
    }
    Assert.assertEquals(2, txMessages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Fetch again from a given message offset exclusively.
    // Expects one message to be fetched
    byte[] startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        // It should have only one message (m2)
        Assert.assertTrue(iterator.hasNext());
        RawMessage msg = iterator.next();
        Assert.assertEquals("m2", Bytes.toString(msg.getPayload()));
    }
    // Fetch again from the last message offset exclusively
    // Expects no message to be fetched
    startMessageId = messages.get(2).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Fetch with start time. It should get both m0 and m1 since they are published in the same request, hence
    // having the same publish time
    startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(new MessageId(startMessageId).getPublishTimestamp()).setLimit(2).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Publish 2 messages, one transactionally, one without transaction
    client.publish(StoreRequestBuilder.of(topicId).addPayload("m3").setTransaction(2L).build());
    client.publish(StoreRequestBuilder.of(topicId).addPayload("m4").build());
    // Consume without transactional, it should see m2, m3 and m4
    startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(3, messages.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals("m" + (i + 2), Bytes.toString(messages.get(i).getPayload()));
    }
    // Consume using a transaction that doesn't have tx = 2L visible. It should get no message as it should block on m3
    transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Consume using a transaction that has tx = 2L in the invalid list. It should skip m3 and got m4
    transaction = new Transaction(3L, 3L, new long[] { 2L }, new long[0], 0L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(1, messages.size());
    Assert.assertEquals("m4", Bytes.toString(messages.get(0).getPayload()));
    // Consume using a transaction that has tx = 2L committed. It should get m3 and m4
    transaction = new Transaction(3L, 3L, new long[0], new long[0], 0L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + (i + 3), Bytes.toString(messages.get(i).getPayload()));
    }
    client.deleteTopic(topicId);
}