Examples with RollbackDetail io.cdap.cdap.messaging.RollbackDetail used on opensource projects

Example 1 with RollbackDetail

use of io.cdap.cdap.messaging.RollbackDetail in project cdap by caskdata.

the class MessageTableTest method testNonTxAndTxConsumption.

@Test
public void testNonTxAndTxConsumption() throws Exception {
    try (MessageTable table1 = getMessageTable(M1);
        MessageTable table2 = getMessageTable(M2);
        MetadataTable metadataTable = getMetadataTable()) {
        metadataTable.createTopic(M1);
        metadataTable.createTopic(M2);
        List<MessageTable.Entry> entryList1 = new ArrayList<>();
        List<MessageTable.Entry> entryList2 = new ArrayList<>();
        Map<Long, Short> startSequenceIds = new HashMap<>();
        Map<Long, Short> endSequenceIds = new HashMap<>();
        long publishTimestamp1 = populateList(entryList1, T1, 123, Arrays.asList(100L, 101L, 102L), startSequenceIds, endSequenceIds);
        long publishTimestamp2 = populateList(entryList2, T2, 321, Arrays.asList(100L, 101L, 102L), startSequenceIds, endSequenceIds);
        table1.store(entryList1.iterator());
        table2.store(entryList2.iterator());
        try (CloseableIterator<MessageTable.Entry> iterator = table1.fetch(M1, 0, Integer.MAX_VALUE, null)) {
            checkPointerCount(iterator, 123, ImmutableSet.of(100L, 101L, 102L), 150);
        }
        // Read with 85 items limit
        try (CloseableIterator<MessageTable.Entry> iterator = table1.fetch(M1, 0, 85, null)) {
            checkPointerCount(iterator, 123, ImmutableSet.of(100L, 101L, 102L), 85);
        }
        // Read with all messages visible
        Transaction tx = new Transaction(200, 200, new long[0], new long[0], -1);
        try (CloseableIterator<MessageTable.Entry> iterator = table1.fetch(M1, 0, Integer.MAX_VALUE, tx)) {
            checkPointerCount(iterator, 123, ImmutableSet.of(100L, 101L, 102L), 150);
        }
        // Read with 101 as invalid transaction
        tx = new Transaction(200, 200, new long[] { 101 }, new long[0], -1);
        try (CloseableIterator<MessageTable.Entry> iterator = table1.fetch(M1, 0, Integer.MAX_VALUE, tx)) {
            checkPointerCount(iterator, 123, ImmutableSet.of(100L, 102L), 100);
        }
        // Mark 101 as in progress transaction, then we shouldn't read past committed transaction which is 100.
        tx = new Transaction(100, 100, new long[] {}, new long[] { 101 }, -1);
        try (CloseableIterator<MessageTable.Entry> iterator = table1.fetch(M1, 0, Integer.MAX_VALUE, tx)) {
            checkPointerCount(iterator, 123, ImmutableSet.of(100L), 50);
        }
        // Same read as above but with limit of 10 elements
        try (CloseableIterator<MessageTable.Entry> iterator = table1.fetch(M1, 0, 10, tx)) {
            checkPointerCount(iterator, 123, ImmutableSet.of(100L), 10);
        }
        // Reading non-tx from t2 should provide 150 items
        try (CloseableIterator<MessageTable.Entry> iterator = table2.fetch(M2, 0, Integer.MAX_VALUE, null)) {
            checkPointerCount(iterator, 321, ImmutableSet.of(100L, 101L, 102L), 150);
        }
        // Delete txPtr entries for 101, and then try fetching again for that
        RollbackDetail rollbackDetail = new TestRollbackDetail(101L, publishTimestamp1, startSequenceIds.get(101L), publishTimestamp1, endSequenceIds.get(101L));
        table1.rollback(M1, rollbackDetail);
        try (CloseableIterator<MessageTable.Entry> iterator = table1.fetch(M1, 0, Integer.MAX_VALUE, null)) {
            checkPointerCount(iterator, 123, ImmutableSet.of(100L, 101L, 102L), 150);
        }
        // Delete txPtr entries for 100, and then try fetching transactionally all data
        rollbackDetail = new TestRollbackDetail(100L, publishTimestamp1, startSequenceIds.get(100L), publishTimestamp1, endSequenceIds.get(100L));
        table1.rollback(M1, rollbackDetail);
        tx = new Transaction(200, 200, new long[0], new long[0], -1);
        try (CloseableIterator<MessageTable.Entry> iterator = table1.fetch(M1, 0, Integer.MAX_VALUE, tx)) {
            checkPointerCount(iterator, 123, ImmutableSet.of(102L), 50);
        }
        // Use the above tx and read from t2 and it should give all entries
        try (CloseableIterator<MessageTable.Entry> iterator = table2.fetch(M2, 0, Integer.MAX_VALUE, tx)) {
            checkPointerCount(iterator, 321, ImmutableSet.of(100L, 101L, 102L), 150);
        }
    }
}

Example 2 with RollbackDetail

use of io.cdap.cdap.messaging.RollbackDetail in project cdap by caskdata.

the class MessageTableTest method testSingleMessage.

@Test
public void testSingleMessage() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("singleMessage");
    TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
    String payload = "data";
    long txWritePtr = 123L;
    try (MessageTable table = getMessageTable(metadata);
        MetadataTable metadataTable = getMetadataTable()) {
        metadataTable.createTopic(metadata);
        List<MessageTable.Entry> entryList = new ArrayList<>();
        entryList.add(new TestMessageEntry(topicId, GENERATION, 0L, 0, txWritePtr, Bytes.toBytes(payload)));
        table.store(entryList.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), false, 50, null)) {
            // Fetch not including the first message, expect empty
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
            // Fetch including the first message, should get the message
            Assert.assertTrue(iterator.hasNext());
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0, 50, null)) {
            // Fetch by time, should get the entry
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        RollbackDetail rollbackDetail = new TestRollbackDetail(123L, 0, (short) 0, 0L, (short) 0);
        table.rollback(metadata, rollbackDetail);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
            // Fetching the message non-tx should provide a result even after deletion
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        Transaction tx = new Transaction(200, 200, new long[0], new long[0], -1);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, tx)) {
            // Fetching messages transactionally should not return any entry
            Assert.assertFalse(iterator.hasNext());
        }
    }
}

Example 3 with RollbackDetail

use of io.cdap.cdap.messaging.RollbackDetail in project cdap by caskdata.

the class MessagingHttpServiceTest method testBasicPubSub.

@Test
public void testBasicPubSub() throws Exception {
    TopicId topicId = new NamespaceId("ns1").topic("testBasicPubSub");
    // Publish to a non-existing topic should get not found exception
    try {
        client.publish(StoreRequestBuilder.of(topicId).addPayload("a").build());
        Assert.fail("Expected TopicNotFoundException");
    } catch (TopicNotFoundException e) {
    // Expected
    }
    // Consume from a non-existing topic should get not found exception
    try {
        client.prepareFetch(topicId).fetch();
        Assert.fail("Expected TopicNotFoundException");
    } catch (TopicNotFoundException e) {
    // Expected
    }
    client.createTopic(new TopicMetadata(topicId));
    // Publish a non-transactional message with empty payload should result in failure
    try {
        client.publish(StoreRequestBuilder.of(topicId).build());
        Assert.fail("Expected IllegalArgumentException");
    } catch (IllegalArgumentException e) {
    // Expected
    }
    // Publish a non-tx message, no RollbackDetail is returned
    Assert.assertNull(client.publish(StoreRequestBuilder.of(topicId).addPayload("m0").addPayload("m1").build()));
    // Publish a transactional message, a RollbackDetail should be returned
    RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topicId).addPayload("m2").setTransaction(1L).build());
    Assert.assertNotNull(rollbackDetail);
    // Rollback the published message
    client.rollback(topicId, rollbackDetail);
    // Fetch messages non-transactionally (should be able to read all the messages since rolled back messages
    // are still visible until ttl kicks in)
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(3, messages.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Consume transactionally. It should get only m0 and m1 since m2 has been rolled back
    List<RawMessage> txMessages = new ArrayList<>();
    Transaction transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(0).setTransaction(transaction).fetch()) {
        Iterators.addAll(txMessages, iterator);
    }
    Assert.assertEquals(2, txMessages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Fetch again from a given message offset exclusively.
    // Expects one message to be fetched
    byte[] startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        // It should have only one message (m2)
        Assert.assertTrue(iterator.hasNext());
        RawMessage msg = iterator.next();
        Assert.assertEquals("m2", Bytes.toString(msg.getPayload()));
    }
    // Fetch again from the last message offset exclusively
    // Expects no message to be fetched
    startMessageId = messages.get(2).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Fetch with start time. It should get both m0 and m1 since they are published in the same request, hence
    // having the same publish time
    startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(new MessageId(startMessageId).getPublishTimestamp()).setLimit(2).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Publish 2 messages, one transactionally, one without transaction
    client.publish(StoreRequestBuilder.of(topicId).addPayload("m3").setTransaction(2L).build());
    client.publish(StoreRequestBuilder.of(topicId).addPayload("m4").build());
    // Consume without transactional, it should see m2, m3 and m4
    startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(3, messages.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals("m" + (i + 2), Bytes.toString(messages.get(i).getPayload()));
    }
    // Consume using a transaction that doesn't have tx = 2L visible. It should get no message as it should block on m3
    transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Consume using a transaction that has tx = 2L in the invalid list. It should skip m3 and got m4
    transaction = new Transaction(3L, 3L, new long[] { 2L }, new long[0], 0L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(1, messages.size());
    Assert.assertEquals("m4", Bytes.toString(messages.get(0).getPayload()));
    // Consume using a transaction that has tx = 2L committed. It should get m3 and m4
    transaction = new Transaction(3L, 3L, new long[0], new long[0], 0L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + (i + 3), Bytes.toString(messages.get(i).getPayload()));
    }
    client.deleteTopic(topicId);
}

Example 4 with RollbackDetail

use of io.cdap.cdap.messaging.RollbackDetail in project cdap by caskdata.

the class MessagingHttpServiceTest method testTxMaxLifeTime.

@Test
public void testTxMaxLifeTime() throws Exception {
    NamespaceId nsId = new NamespaceId("txCheck");
    TopicId topic1 = nsId.topic("t1");
    // Create a topic
    client.createTopic(new TopicMetadata(topic1));
    final RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topic1).setTransaction(1L).addPayload("a").addPayload("b").build());
    try {
        client.publish(StoreRequestBuilder.of(topic1).setTransaction(-Long.MAX_VALUE).addPayload("c").addPayload("d").build());
        Assert.fail("Expected IOException");
    } catch (IOException ex) {
    // expected
    }
    Set<String> msgs = new HashSet<>();
    CloseableIterator<RawMessage> messages = client.prepareFetch(topic1).fetch();
    while (messages.hasNext()) {
        RawMessage message = messages.next();
        msgs.add(Bytes.toString(message.getPayload()));
    }
    Assert.assertEquals(2, msgs.size());
    Assert.assertTrue(msgs.contains("a"));
    Assert.assertTrue(msgs.contains("b"));
    messages.close();
    client.rollback(topic1, rollbackDetail);
    client.deleteTopic(topic1);
}

Example 5 with RollbackDetail

use of io.cdap.cdap.messaging.RollbackDetail in project cdap by caskdata.

the class StoreHandler method publish.

@POST
@Path("/publish")
public BodyConsumer publish(HttpRequest request, HttpResponder responder, @PathParam("namespace") String namespace, @PathParam("topic") String topic) throws Exception {
    TopicId topicId = new NamespaceId(namespace).topic(topic);
    return new SpillableBodyConsumer(Files.createTempFile(tempDir, "tms.publish", ".tmp"), bufferSize) {

        @Override
        protected void processInput(InputStream inputStream, HttpResponder responder) throws Exception {
            StoreRequest storeRequest = createStoreRequest(topicId, request, inputStream);
            // Empty payload is only allowed for transactional publish
            if (!storeRequest.isTransactional() && !storeRequest.hasPayload()) {
                throw new BadRequestException("Empty payload is only allowed for publishing transactional message. Topic: " + topicId);
            }
            // Publish the message and response with the rollback information
            RollbackDetail rollbackInfo = messagingService.publish(storeRequest);
            if (rollbackInfo == null) {
                // Non-tx publish doesn't have rollback info.
                responder.sendStatus(HttpResponseStatus.OK);
            } else {
                ByteBuf response = encodeRollbackDetail(rollbackInfo);
                responder.sendContent(HttpResponseStatus.OK, response, new DefaultHttpHeaders().set(HttpHeaderNames.CONTENT_TYPE, "avro/binary"));
            }
        }
    };
}