use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class DataCleanupTest method testPayloadTTLCleanup.
@Test
public void testPayloadTTLCleanup() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("t2");
TopicMetadata topic = new TopicMetadata(topicId, "ttl", "3", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
try (MetadataTable metadataTable = getMetadataTable();
PayloadTable payloadTable = getPayloadTable(topic);
MessageTable messageTable = getMessageTable(topic)) {
Assert.assertNotNull(messageTable);
metadataTable.createTopic(topic);
List<PayloadTable.Entry> entries = new ArrayList<>();
entries.add(new TestPayloadEntry(topicId, GENERATION, "payloaddata", 101, (short) 0));
payloadTable.store(entries.iterator());
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
Assert.assertTrue(iterator.hasNext());
PayloadTable.Entry entry = iterator.next();
Assert.assertFalse(iterator.hasNext());
Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
Assert.assertEquals(101L, entry.getTransactionWritePointer());
iterator.close();
forceFlushAndCompact(Table.PAYLOAD);
// Entry should still be there since ttl has not expired
iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
Assert.assertTrue(iterator.hasNext());
entry = iterator.next();
Assert.assertFalse(iterator.hasNext());
Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
Assert.assertEquals(101L, entry.getTransactionWritePointer());
iterator.close();
TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
forceFlushAndCompact(Table.PAYLOAD);
iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
Assert.assertFalse(iterator.hasNext());
iterator.close();
metadataTable.deleteTopic(topicId);
}
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class MessageTableTest method testEmptyPayload.
@Test
public void testEmptyPayload() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("testEmptyPayload");
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
// stores only a reference to the payload table
try (MessageTable table = getMessageTable(metadata);
MetadataTable metadataTable = getMetadataTable()) {
metadataTable.createTopic(metadata);
try {
table.store(Collections.singleton(new TestMessageEntry(topicId, GENERATION, 1L, 0, null, null)).iterator());
Assert.fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
// Expected as non-transactional message cannot have null payload
}
// For transactional message, ok to have null payload
table.store(Collections.singleton(new TestMessageEntry(topicId, GENERATION, 1L, 0, 2L, null)).iterator());
// Fetch the entry to validate
List<MessageTable.Entry> entries = new ArrayList<>();
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0L, Integer.MAX_VALUE, null)) {
Iterators.addAll(entries, iterator);
}
Assert.assertEquals(1, entries.size());
MessageTable.Entry entry = entries.get(0);
Assert.assertEquals(1L, entry.getPublishTimestamp());
Assert.assertEquals(0, entry.getSequenceId());
Assert.assertTrue(entry.isTransactional());
Assert.assertEquals(2L, entry.getTransactionWritePointer());
Assert.assertNull(entry.getPayload());
Assert.assertTrue(entry.isPayloadReference());
}
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class MessageTableTest method testSingleMessage.
@Test
public void testSingleMessage() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("singleMessage");
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
String payload = "data";
long txWritePtr = 123L;
try (MessageTable table = getMessageTable(metadata);
MetadataTable metadataTable = getMetadataTable()) {
metadataTable.createTopic(metadata);
List<MessageTable.Entry> entryList = new ArrayList<>();
entryList.add(new TestMessageEntry(topicId, GENERATION, 0L, 0, txWritePtr, Bytes.toBytes(payload)));
table.store(entryList.iterator());
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), false, 50, null)) {
// Fetch not including the first message, expect empty
Assert.assertFalse(iterator.hasNext());
}
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
// Fetch including the first message, should get the message
Assert.assertTrue(iterator.hasNext());
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0, 50, null)) {
// Fetch by time, should get the entry
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
RollbackDetail rollbackDetail = new TestRollbackDetail(123L, 0, (short) 0, 0L, (short) 0);
table.rollback(metadata, rollbackDetail);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
// Fetching the message non-tx should provide a result even after deletion
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
Transaction tx = new Transaction(200, 200, new long[0], new long[0], -1);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, tx)) {
// Fetching messages transactionally should not return any entry
Assert.assertFalse(iterator.hasNext());
}
}
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class MetadataTableTest method testCRUD.
@Test
public void testCRUD() throws Exception {
try (MetadataTable table = createMetadataTable()) {
TopicId topicId = NamespaceId.DEFAULT.topic("topic");
// Update a non-existing topic should fail.
try {
table.updateTopic(new TopicMetadata(topicId, "ttl", 10));
Assert.fail("Expected TopicNotFoundException");
} catch (TopicNotFoundException e) {
// Expected
}
// Create a topic and validate
table.createTopic(new TopicMetadata(topicId, "ttl", 10));
Assert.assertEquals(10, table.getMetadata(topicId).getTTL());
// Update the property and validate
table.updateTopic(new TopicMetadata(topicId, "ttl", 30));
Assert.assertEquals(30, table.getMetadata(topicId).getTTL());
// Create the same topic again should fail
try {
table.createTopic(new TopicMetadata(topicId, "ttl", 10));
Assert.fail("Expected TopicAlreadyExistsException");
} catch (TopicAlreadyExistsException e) {
// Expected
}
// It shouldn't affect the topic at all if creation failed
Assert.assertEquals(30, table.getMetadata(topicId).getTTL());
// Delete the topic
table.deleteTopic(topicId);
try {
table.getMetadata(topicId);
Assert.fail("Expected TopicNotFoundException");
} catch (TopicNotFoundException e) {
// Expected
}
// Delete again should raise a TopicNotFoundException
try {
table.deleteTopic(topicId);
Assert.fail("Expected TopicNotFoundException");
} catch (TopicNotFoundException e) {
// Expected
}
}
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class LevelDBMetadataTable method createTopic.
@Override
public void createTopic(TopicMetadata topicMetadata) throws TopicAlreadyExistsException, IOException {
try {
TopicId topicId = topicMetadata.getTopicId();
byte[] key = MessagingUtils.toMetadataRowKey(topicId);
TreeMap<String, String> properties = new TreeMap<>(topicMetadata.getProperties());
properties.put(TopicMetadata.GENERATION_KEY, MessagingUtils.Constants.DEFAULT_GENERATION);
synchronized (this) {
byte[] tableValue = levelDB.get(key);
if (tableValue != null) {
Map<String, String> oldProperties = GSON.fromJson(Bytes.toString(tableValue), MAP_TYPE);
TopicMetadata metadata = new TopicMetadata(topicId, oldProperties);
if (metadata.exists()) {
throw new TopicAlreadyExistsException(topicId.getNamespace(), topicId.getTopic());
}
int newGenerationId = (metadata.getGeneration() * -1) + 1;
properties.put(TopicMetadata.GENERATION_KEY, Integer.toString(newGenerationId));
}
byte[] value = Bytes.toBytes(GSON.toJson(properties, MAP_TYPE));
levelDB.put(key, value, WRITE_OPTIONS);
}
} catch (DBException e) {
throw new IOException(e);
}
}
Aggregations