Search in sources :

Example 1 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testMultiMaxSequence.

@Test
public void testMultiMaxSequence() throws IOException, InterruptedException {
    TopicId topicId = new NamespaceId("ns1").topic("t1");
    final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
    // This test the case when multiple StoreRequests combined exceeding the 65536 payload.
    // See testMaxSequence() for more details when it matters
    // Generate 3 StoreRequests, each with 43690 messages
    int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT / 3 * 2;
    int requestCount = 3;
    List<StoreRequest> requests = new ArrayList<>();
    for (int i = 0; i < requestCount; i++) {
        List<String> payloads = new ArrayList<>(msgCount);
        for (int j = 0; j < msgCount; j++) {
            payloads.add(Integer.toString(j));
        }
        requests.add(new TestStoreRequest(topicId, payloads));
    }
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    // We use a custom metrics collector here to make all the persist calls reached the same latch,
    // since we know that the ConcurrentMessageWriter will emit a metrics "persist.requested" after enqueued but
    // before flushing.
    // This will make all requests batched together
    final CountDownLatch latch = new CountDownLatch(requestCount);
    final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter, new MetricsCollector() {

        @Override
        public void increment(String metricName, long value) {
            if ("persist.requested".equals(metricName)) {
                latch.countDown();
                Uninterruptibles.awaitUninterruptibly(latch);
            }
        }

        @Override
        public void gauge(String metricName, long value) {
            LOG.info("MetricsContext.gauge: {} = {}", metricName, value);
        }
    });
    ExecutorService executor = Executors.newFixedThreadPool(3);
    for (final StoreRequest request : requests) {
        executor.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    writer.persist(request, metadata);
                } catch (IOException e) {
                    LOG.error("Failed to persist", e);
                }
            }
        });
    }
    executor.shutdown();
    Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
    // Validates all messages are being written
    List<RawMessage> messages = testWriter.getMessages().get(topicId);
    Assert.assertEquals(requestCount * msgCount, messages.size());
    // We expect the payload is in repeated sequence of [0..msgCount-1]
    int expectedPayload = 0;
    // The sequenceId should be (i % SEQUENCE_ID_LIMIT)
    for (int i = 0; i < messages.size(); i++) {
        RawMessage message = messages.get(i);
        MessageId messageId = new MessageId(message.getId());
        Assert.assertEquals(i / StoreRequestWriter.SEQUENCE_ID_LIMIT, messageId.getPublishTimestamp());
        Assert.assertEquals((short) (i % StoreRequestWriter.SEQUENCE_ID_LIMIT), messageId.getSequenceId());
        Assert.assertEquals(expectedPayload, Integer.parseInt(Bytes.toString(message.getPayload())));
        expectedPayload = (expectedPayload + 1) % msgCount;
    }
}
Also used : ArrayList(java.util.ArrayList) TopicId(io.cdap.cdap.proto.id.TopicId) RawMessage(io.cdap.cdap.messaging.data.RawMessage) MetricsCollector(io.cdap.cdap.api.metrics.MetricsCollector) TimeProvider(io.cdap.cdap.common.utils.TimeProvider) StoreRequest(io.cdap.cdap.messaging.StoreRequest) IOException(java.io.IOException) CountDownLatch(java.util.concurrent.CountDownLatch) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) ExecutorService(java.util.concurrent.ExecutorService) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Example 2 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testBasic.

@Test
public void testBasic() throws IOException {
    TopicId topicId1 = new NamespaceId("ns1").topic("t1");
    TopicId topicId2 = new NamespaceId("ns2").topic("t2");
    TopicMetadata metadata1 = new TopicMetadata(topicId1, new HashMap<String, String>(), 1);
    TopicMetadata metadata2 = new TopicMetadata(topicId2, new HashMap<String, String>(), 1);
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter);
    writer.persist(new TestStoreRequest(topicId1, Arrays.asList("1", "2", "3")), metadata1);
    // There should be 3 messages being written
    List<RawMessage> messages = testWriter.getMessages().get(topicId1);
    Assert.assertEquals(3, messages.size());
    // All messages should be written with timestamp 0
    List<String> payloads = new ArrayList<>();
    for (RawMessage message : messages) {
        Assert.assertEquals(0L, new MessageId(message.getId()).getPublishTimestamp());
        payloads.add(Bytes.toString(message.getPayload()));
    }
    Assert.assertEquals(Arrays.asList("1", "2", "3"), payloads);
    // Write to another topic
    writer.persist(new TestStoreRequest(topicId2, Arrays.asList("a", "b", "c")), metadata2);
    // There should be 3 messages being written to topic2
    messages = testWriter.getMessages().get(topicId2);
    Assert.assertEquals(3, messages.size());
    // All messages should be written with timestamp 1
    payloads.clear();
    for (RawMessage message : messages) {
        Assert.assertEquals(1L, new MessageId(message.getId()).getPublishTimestamp());
        payloads.add(Bytes.toString(message.getPayload()));
    }
    Assert.assertEquals(Arrays.asList("a", "b", "c"), payloads);
}
Also used : TimeProvider(io.cdap.cdap.common.utils.TimeProvider) ArrayList(java.util.ArrayList) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) TopicId(io.cdap.cdap.proto.id.TopicId) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) RawMessage(io.cdap.cdap.messaging.data.RawMessage) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Example 3 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class MessageTableTest method testSingleMessage.

@Test
public void testSingleMessage() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("singleMessage");
    TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
    String payload = "data";
    long txWritePtr = 123L;
    try (MessageTable table = getMessageTable(metadata);
        MetadataTable metadataTable = getMetadataTable()) {
        metadataTable.createTopic(metadata);
        List<MessageTable.Entry> entryList = new ArrayList<>();
        entryList.add(new TestMessageEntry(topicId, GENERATION, 0L, 0, txWritePtr, Bytes.toBytes(payload)));
        table.store(entryList.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), false, 50, null)) {
            // Fetch not including the first message, expect empty
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
            // Fetch including the first message, should get the message
            Assert.assertTrue(iterator.hasNext());
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0, 50, null)) {
            // Fetch by time, should get the entry
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        RollbackDetail rollbackDetail = new TestRollbackDetail(123L, 0, (short) 0, 0L, (short) 0);
        table.rollback(metadata, rollbackDetail);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
            // Fetching the message non-tx should provide a result even after deletion
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        Transaction tx = new Transaction(200, 200, new long[0], new long[0], -1);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, tx)) {
            // Fetching messages transactionally should not return any entry
            Assert.assertFalse(iterator.hasNext());
        }
    }
}
Also used : RollbackDetail(io.cdap.cdap.messaging.RollbackDetail) ArrayList(java.util.ArrayList) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) Transaction(org.apache.tephra.Transaction) TopicId(io.cdap.cdap.proto.id.TopicId) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Example 4 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class MessagingHttpServiceTest method testBasicPubSub.

@Test
public void testBasicPubSub() throws Exception {
    TopicId topicId = new NamespaceId("ns1").topic("testBasicPubSub");
    // Publish to a non-existing topic should get not found exception
    try {
        client.publish(StoreRequestBuilder.of(topicId).addPayload("a").build());
        Assert.fail("Expected TopicNotFoundException");
    } catch (TopicNotFoundException e) {
    // Expected
    }
    // Consume from a non-existing topic should get not found exception
    try {
        client.prepareFetch(topicId).fetch();
        Assert.fail("Expected TopicNotFoundException");
    } catch (TopicNotFoundException e) {
    // Expected
    }
    client.createTopic(new TopicMetadata(topicId));
    // Publish a non-transactional message with empty payload should result in failure
    try {
        client.publish(StoreRequestBuilder.of(topicId).build());
        Assert.fail("Expected IllegalArgumentException");
    } catch (IllegalArgumentException e) {
    // Expected
    }
    // Publish a non-tx message, no RollbackDetail is returned
    Assert.assertNull(client.publish(StoreRequestBuilder.of(topicId).addPayload("m0").addPayload("m1").build()));
    // Publish a transactional message, a RollbackDetail should be returned
    RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topicId).addPayload("m2").setTransaction(1L).build());
    Assert.assertNotNull(rollbackDetail);
    // Rollback the published message
    client.rollback(topicId, rollbackDetail);
    // Fetch messages non-transactionally (should be able to read all the messages since rolled back messages
    // are still visible until ttl kicks in)
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(3, messages.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Consume transactionally. It should get only m0 and m1 since m2 has been rolled back
    List<RawMessage> txMessages = new ArrayList<>();
    Transaction transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(0).setTransaction(transaction).fetch()) {
        Iterators.addAll(txMessages, iterator);
    }
    Assert.assertEquals(2, txMessages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Fetch again from a given message offset exclusively.
    // Expects one message to be fetched
    byte[] startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        // It should have only one message (m2)
        Assert.assertTrue(iterator.hasNext());
        RawMessage msg = iterator.next();
        Assert.assertEquals("m2", Bytes.toString(msg.getPayload()));
    }
    // Fetch again from the last message offset exclusively
    // Expects no message to be fetched
    startMessageId = messages.get(2).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Fetch with start time. It should get both m0 and m1 since they are published in the same request, hence
    // having the same publish time
    startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(new MessageId(startMessageId).getPublishTimestamp()).setLimit(2).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
    }
    // Publish 2 messages, one transactionally, one without transaction
    client.publish(StoreRequestBuilder.of(topicId).addPayload("m3").setTransaction(2L).build());
    client.publish(StoreRequestBuilder.of(topicId).addPayload("m4").build());
    // Consume without transactional, it should see m2, m3 and m4
    startMessageId = messages.get(1).getId();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(3, messages.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals("m" + (i + 2), Bytes.toString(messages.get(i).getPayload()));
    }
    // Consume using a transaction that doesn't have tx = 2L visible. It should get no message as it should block on m3
    transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Consume using a transaction that has tx = 2L in the invalid list. It should skip m3 and got m4
    transaction = new Transaction(3L, 3L, new long[] { 2L }, new long[0], 0L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(1, messages.size());
    Assert.assertEquals("m4", Bytes.toString(messages.get(0).getPayload()));
    // Consume using a transaction that has tx = 2L committed. It should get m3 and m4
    transaction = new Transaction(3L, 3L, new long[0], new long[0], 0L);
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    for (int i = 0; i < 2; i++) {
        Assert.assertEquals("m" + (i + 3), Bytes.toString(messages.get(i).getPayload()));
    }
    client.deleteTopic(topicId);
}
Also used : RollbackDetail(io.cdap.cdap.messaging.RollbackDetail) TopicNotFoundException(io.cdap.cdap.api.messaging.TopicNotFoundException) ArrayList(java.util.ArrayList) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) Transaction(org.apache.tephra.Transaction) TopicId(io.cdap.cdap.proto.id.TopicId) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) RawMessage(io.cdap.cdap.messaging.data.RawMessage) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Example 5 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class DataCleanupTest method testPayloadTTLCleanup.

@Test
public void testPayloadTTLCleanup() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("t2");
    TopicMetadata topic = new TopicMetadata(topicId, "ttl", "3", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
    try (MetadataTable metadataTable = getMetadataTable();
        PayloadTable payloadTable = getPayloadTable(topic);
        MessageTable messageTable = getMessageTable(topic)) {
        Assert.assertNotNull(messageTable);
        metadataTable.createTopic(topic);
        List<PayloadTable.Entry> entries = new ArrayList<>();
        entries.add(new TestPayloadEntry(topicId, GENERATION, "payloaddata", 101, (short) 0));
        payloadTable.store(entries.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
        Assert.assertTrue(iterator.hasNext());
        PayloadTable.Entry entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
        Assert.assertEquals(101L, entry.getTransactionWritePointer());
        iterator.close();
        forceFlushAndCompact(Table.PAYLOAD);
        // Entry should still be there since ttl has not expired
        iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
        Assert.assertTrue(iterator.hasNext());
        entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
        Assert.assertEquals(101L, entry.getTransactionWritePointer());
        iterator.close();
        TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
        forceFlushAndCompact(Table.PAYLOAD);
        iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
        Assert.assertFalse(iterator.hasNext());
        iterator.close();
        metadataTable.deleteTopic(topicId);
    }
}
Also used : ArrayList(java.util.ArrayList) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) TopicId(io.cdap.cdap.proto.id.TopicId) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Aggregations

MessageId (io.cdap.cdap.messaging.data.MessageId)15 Test (org.junit.Test)13 TopicMetadata (io.cdap.cdap.messaging.TopicMetadata)12 TopicId (io.cdap.cdap.proto.id.TopicId)12 ArrayList (java.util.ArrayList)11 TimeProvider (io.cdap.cdap.common.utils.TimeProvider)4 RawMessage (io.cdap.cdap.messaging.data.RawMessage)4 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)4 MessageTable (io.cdap.cdap.messaging.store.MessageTable)3 HashMap (java.util.HashMap)3 Transaction (org.apache.tephra.Transaction)3 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)2 RollbackDetail (io.cdap.cdap.messaging.RollbackDetail)2 MessageTableTest (io.cdap.cdap.messaging.store.MessageTableTest)2 TestMessageEntry (io.cdap.cdap.messaging.store.TestMessageEntry)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 AbstractIterator (com.google.common.collect.AbstractIterator)1 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)1 TopicNotFoundException (io.cdap.cdap.api.messaging.TopicNotFoundException)1 MetricsCollector (io.cdap.cdap.api.metrics.MetricsCollector)1