Examples with TopicId co.cask.cdap.proto.id.TopicId used on opensource projects

Example 41 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class MessageTableTest method testEmptyPayload.

@Test
public void testEmptyPayload() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("testEmptyPayload");
    TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
    // stores only a reference to the payload table
    try (MessageTable table = getMessageTable();
        MetadataTable metadataTable = getMetadataTable()) {
        metadataTable.createTopic(metadata);
        try {
            table.store(Collections.singleton(new TestMessageEntry(topicId, GENERATION, 1L, 0, null, null)).iterator());
            Assert.fail("Expected IllegalArgumentException");
        } catch (IllegalArgumentException e) {
        // Expected as non-transactional message cannot have null payload
        }
        // For transactional message, ok to have null payload
        table.store(Collections.singleton(new TestMessageEntry(topicId, GENERATION, 1L, 0, 2L, null)).iterator());
        // Fetch the entry to validate
        List<MessageTable.Entry> entries = new ArrayList<>();
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0L, Integer.MAX_VALUE, null)) {
            Iterators.addAll(entries, iterator);
        }
        Assert.assertEquals(1, entries.size());
        MessageTable.Entry entry = entries.get(0);
        Assert.assertEquals(1L, entry.getPublishTimestamp());
        Assert.assertEquals(0, entry.getSequenceId());
        Assert.assertTrue(entry.isTransactional());
        Assert.assertEquals(2L, entry.getTransactionWritePointer());
        Assert.assertNull(entry.getPayload());
        Assert.assertTrue(entry.isPayloadReference());
    }
}

Example 42 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class LevelDBMetadataTable method scanTopics.

private CloseableIterator<TopicMetadata> scanTopics(@Nullable byte[] startKey, @Nullable byte[] stopKey) throws IOException {
    final CloseableIterator<Map.Entry<byte[], byte[]>> iterator = new DBScanIterator(levelDB, startKey, stopKey);
    return new AbstractCloseableIterator<TopicMetadata>() {

        private boolean closed = false;

        @Override
        protected TopicMetadata computeNext() {
            if (closed || (!iterator.hasNext())) {
                return endOfData();
            }
            Map.Entry<byte[], byte[]> entry = iterator.next();
            TopicId topicId = MessagingUtils.toTopicId(entry.getKey());
            Map<String, String> properties = GSON.fromJson(Bytes.toString(entry.getValue()), MAP_TYPE);
            return new TopicMetadata(topicId, properties);
        }

        @Override
        public void close() {
            try {
                iterator.close();
            } finally {
                endOfData();
                closed = true;
            }
        }
    };
}

Example 43 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class MessagingHttpServiceTest method testChunkConsume.

@Test
public void testChunkConsume() throws Exception {
    // This test is to verify the message fetching body producer works correctly
    TopicId topicId = new NamespaceId("ns1").topic("testChunkConsume");
    client.createTopic(new TopicMetadata(topicId));
    // Publish 10 messages, each payload is half the size of the chunk size
    int payloadSize = cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_CONSUME_CHUNK_SIZE) / 2;
    for (int i = 0; i < 10; i++) {
        String payload = Strings.repeat(Integer.toString(i), payloadSize);
        client.publish(StoreRequestBuilder.of(topicId).addPayloads(payload).build());
    }
    // Fetch messages. All of them should be fetched correctly
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(10, messages.size());
    for (int i = 0; i < 10; i++) {
        RawMessage message = messages.get(i);
        Assert.assertEquals(payloadSize, message.getPayload().length);
        String payload = Strings.repeat(Integer.toString(i), payloadSize);
        Assert.assertEquals(payload, Bytes.toString(message.getPayload()));
    }
    client.deleteTopic(topicId);
}

Example 44 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class MessagingHttpServiceTest method testPayloadTable.

@Test
public void testPayloadTable() throws Exception {
    // This test is to verify storing transaction messages to the payload table
    TopicId topicId = new NamespaceId("ns1").topic("testPayloadTable");
    client.createTopic(new TopicMetadata(topicId));
    // Try to store to Payload table with empty iterator, expected failure
    try {
        client.storePayload(StoreRequestBuilder.of(topicId).setTransaction(1L).build());
        Assert.fail("Expected IllegalArgumentException");
    } catch (IllegalArgumentException e) {
    // Expected
    }
    // Store 20 payloads to the payload table, with 2 payloads per request
    for (int i = 0; i < 10; i++) {
        String payload = Integer.toString(i);
        client.storePayload(StoreRequestBuilder.of(topicId).addPayloads(payload, payload).setTransaction(1L).build());
    }
    // Try to consume and there should be no messages
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Publish an empty payload message to the message table. This simulates a tx commit.
    client.publish(StoreRequestBuilder.of(topicId).setTransaction(1L).build());
    // Consume again and there should be 20 messages
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(20, messages.size());
    for (int i = 0; i < 20; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString(i / 2), payload1);
    }
    // Should get the last 10 messages
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(10).getId(), true).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(10, messages.size());
    for (int i = 0; i < 10; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString((i + 10) / 2), payload1);
    }
    // We start with the 12th message id as offset, hence should get 8 messages.
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(1).getId(), false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(8, messages.size());
    for (int i = 0; i < 8; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString((i + 12) / 2), payload1);
    }
    // Fetch with the last message id in the payload table, exclusively. Should get an empty iterator
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(messages.size() - 1).getId(), false).fetch()) {
        Assert.assertFalse(iterator.hasNext());
    }
    // Consume with a limit
    messages.clear();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(6).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(6, messages.size());
    for (int i = 0; i < 6; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString(i / 2), payload1);
    }
    // Store and publish two more payloads
    String payload = Integer.toString(10);
    client.storePayload(StoreRequestBuilder.of(topicId).addPayloads(payload, payload).setTransaction(2L).build());
    client.publish(StoreRequestBuilder.of(topicId).setTransaction(2L).build());
    // Should get 22 messages
    messages.clear();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(22, messages.size());
    for (int i = 0; i < 22; i += 2) {
        String payload1 = Bytes.toString(messages.get(i).getPayload());
        String payload2 = Bytes.toString(messages.get(i + 1).getPayload());
        Assert.assertEquals(payload1, payload2);
        Assert.assertEquals(Integer.toString(i / 2), payload1);
    }
    // Should get 2 messages back
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(messages.get(19).getId(), false).fetch()) {
        messages.clear();
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(2, messages.size());
    String payload1 = Bytes.toString(messages.get(0).getPayload());
    String payload2 = Bytes.toString(messages.get(1).getPayload());
    Assert.assertEquals(payload1, payload2);
    Assert.assertEquals(Integer.toString(10), payload1);
    client.deleteTopic(topicId);
}

Example 45 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class DataCleanupTest method testMessageTTLCleanup.

@Test
public void testMessageTTLCleanup() throws Exception {
    try (MetadataTable metadataTable = getMetadataTable();
        MessageTable messageTable = getMessageTable();
        PayloadTable payloadTable = getPayloadTable()) {
        Assert.assertNotNull(payloadTable);
        TopicId topicId = NamespaceId.DEFAULT.topic("t1");
        TopicMetadata topic = new TopicMetadata(topicId, "ttl", "3", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
        metadataTable.createTopic(topic);
        List<MessageTable.Entry> entries = new ArrayList<>();
        entries.add(new TestMessageEntry(topicId, GENERATION, "data", 100, (short) 0));
        messageTable.store(entries.iterator());
        // Fetch the entries and make sure we are able to read it
        CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
        Assert.assertTrue(iterator.hasNext());
        MessageTable.Entry entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals(100, entry.getTransactionWritePointer());
        Assert.assertEquals("data", Bytes.toString(entry.getPayload()));
        iterator.close();
        forceFlushAndCompact(Table.MESSAGE);
        // Entry should still be there since the ttl has not expired
        iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
        entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals(100, entry.getTransactionWritePointer());
        Assert.assertEquals("data", Bytes.toString(entry.getPayload()));
        iterator.close();
        TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
        forceFlushAndCompact(Table.MESSAGE);
        iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
        Assert.assertFalse(iterator.hasNext());
        iterator.close();
    }
}