Examples with TopicMetadata io.cdap.cdap.messaging.TopicMetadata used on opensource projects

Example 16 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class DataCleanupTest method testPayloadTTLCleanup.

@Test
public void testPayloadTTLCleanup() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("t2");
    TopicMetadata topic = new TopicMetadata(topicId, "ttl", "3", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
    try (MetadataTable metadataTable = getMetadataTable();
        PayloadTable payloadTable = getPayloadTable(topic);
        MessageTable messageTable = getMessageTable(topic)) {
        Assert.assertNotNull(messageTable);
        metadataTable.createTopic(topic);
        List<PayloadTable.Entry> entries = new ArrayList<>();
        entries.add(new TestPayloadEntry(topicId, GENERATION, "payloaddata", 101, (short) 0));
        payloadTable.store(entries.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
        Assert.assertTrue(iterator.hasNext());
        PayloadTable.Entry entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
        Assert.assertEquals(101L, entry.getTransactionWritePointer());
        iterator.close();
        forceFlushAndCompact(Table.PAYLOAD);
        // Entry should still be there since ttl has not expired
        iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
        Assert.assertTrue(iterator.hasNext());
        entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
        Assert.assertEquals(101L, entry.getTransactionWritePointer());
        iterator.close();
        TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
        forceFlushAndCompact(Table.PAYLOAD);
        iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
        Assert.assertFalse(iterator.hasNext());
        iterator.close();
        metadataTable.deleteTopic(topicId);
    }
}

Example 17 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class MessageTableTest method testEmptyPayload.

@Test
public void testEmptyPayload() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("testEmptyPayload");
    TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
    // stores only a reference to the payload table
    try (MessageTable table = getMessageTable(metadata);
        MetadataTable metadataTable = getMetadataTable()) {
        metadataTable.createTopic(metadata);
        try {
            table.store(Collections.singleton(new TestMessageEntry(topicId, GENERATION, 1L, 0, null, null)).iterator());
            Assert.fail("Expected IllegalArgumentException");
        } catch (IllegalArgumentException e) {
        // Expected as non-transactional message cannot have null payload
        }
        // For transactional message, ok to have null payload
        table.store(Collections.singleton(new TestMessageEntry(topicId, GENERATION, 1L, 0, 2L, null)).iterator());
        // Fetch the entry to validate
        List<MessageTable.Entry> entries = new ArrayList<>();
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0L, Integer.MAX_VALUE, null)) {
            Iterators.addAll(entries, iterator);
        }
        Assert.assertEquals(1, entries.size());
        MessageTable.Entry entry = entries.get(0);
        Assert.assertEquals(1L, entry.getPublishTimestamp());
        Assert.assertEquals(0, entry.getSequenceId());
        Assert.assertTrue(entry.isTransactional());
        Assert.assertEquals(2L, entry.getTransactionWritePointer());
        Assert.assertNull(entry.getPayload());
        Assert.assertTrue(entry.isPayloadReference());
    }
}

Example 18 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class MessageTableTest method testSingleMessage.

@Test
public void testSingleMessage() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("singleMessage");
    TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
    String payload = "data";
    long txWritePtr = 123L;
    try (MessageTable table = getMessageTable(metadata);
        MetadataTable metadataTable = getMetadataTable()) {
        metadataTable.createTopic(metadata);
        List<MessageTable.Entry> entryList = new ArrayList<>();
        entryList.add(new TestMessageEntry(topicId, GENERATION, 0L, 0, txWritePtr, Bytes.toBytes(payload)));
        table.store(entryList.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), false, 50, null)) {
            // Fetch not including the first message, expect empty
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
            // Fetch including the first message, should get the message
            Assert.assertTrue(iterator.hasNext());
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0, 50, null)) {
            // Fetch by time, should get the entry
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        RollbackDetail rollbackDetail = new TestRollbackDetail(123L, 0, (short) 0, 0L, (short) 0);
        table.rollback(metadata, rollbackDetail);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
            // Fetching the message non-tx should provide a result even after deletion
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        Transaction tx = new Transaction(200, 200, new long[0], new long[0], -1);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, tx)) {
            // Fetching messages transactionally should not return any entry
            Assert.assertFalse(iterator.hasNext());
        }
    }
}

Example 19 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class MetadataTableTest method testCRUD.

@Test
public void testCRUD() throws Exception {
    try (MetadataTable table = createMetadataTable()) {
        TopicId topicId = NamespaceId.DEFAULT.topic("topic");
        // Update a non-existing topic should fail.
        try {
            table.updateTopic(new TopicMetadata(topicId, "ttl", 10));
            Assert.fail("Expected TopicNotFoundException");
        } catch (TopicNotFoundException e) {
        // Expected
        }
        // Create a topic and validate
        table.createTopic(new TopicMetadata(topicId, "ttl", 10));
        Assert.assertEquals(10, table.getMetadata(topicId).getTTL());
        // Update the property and validate
        table.updateTopic(new TopicMetadata(topicId, "ttl", 30));
        Assert.assertEquals(30, table.getMetadata(topicId).getTTL());
        // Create the same topic again should fail
        try {
            table.createTopic(new TopicMetadata(topicId, "ttl", 10));
            Assert.fail("Expected TopicAlreadyExistsException");
        } catch (TopicAlreadyExistsException e) {
        // Expected
        }
        // It shouldn't affect the topic at all if creation failed
        Assert.assertEquals(30, table.getMetadata(topicId).getTTL());
        // Delete the topic
        table.deleteTopic(topicId);
        try {
            table.getMetadata(topicId);
            Assert.fail("Expected TopicNotFoundException");
        } catch (TopicNotFoundException e) {
        // Expected
        }
        // Delete again should raise a TopicNotFoundException
        try {
            table.deleteTopic(topicId);
            Assert.fail("Expected TopicNotFoundException");
        } catch (TopicNotFoundException e) {
        // Expected
        }
    }
}

Example 20 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class LevelDBMetadataTable method createTopic.

@Override
public void createTopic(TopicMetadata topicMetadata) throws TopicAlreadyExistsException, IOException {
    try {
        TopicId topicId = topicMetadata.getTopicId();
        byte[] key = MessagingUtils.toMetadataRowKey(topicId);
        TreeMap<String, String> properties = new TreeMap<>(topicMetadata.getProperties());
        properties.put(TopicMetadata.GENERATION_KEY, MessagingUtils.Constants.DEFAULT_GENERATION);
        synchronized (this) {
            byte[] tableValue = levelDB.get(key);
            if (tableValue != null) {
                Map<String, String> oldProperties = GSON.fromJson(Bytes.toString(tableValue), MAP_TYPE);
                TopicMetadata metadata = new TopicMetadata(topicId, oldProperties);
                if (metadata.exists()) {
                    throw new TopicAlreadyExistsException(topicId.getNamespace(), topicId.getTopic());
                }
                int newGenerationId = (metadata.getGeneration() * -1) + 1;
                properties.put(TopicMetadata.GENERATION_KEY, Integer.toString(newGenerationId));
            }
            byte[] value = Bytes.toBytes(GSON.toJson(properties, MAP_TYPE));
            levelDB.put(key, value, WRITE_OPTIONS);
        }
    } catch (DBException e) {
        throw new IOException(e);
    }
}