use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class DataCleanupTest method testPayloadTTLCleanup.
@Test
public void testPayloadTTLCleanup() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("t2");
TopicMetadata topic = new TopicMetadata(topicId, "ttl", "3", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
try (MetadataTable metadataTable = getMetadataTable();
PayloadTable payloadTable = getPayloadTable(topic);
MessageTable messageTable = getMessageTable(topic)) {
Assert.assertNotNull(messageTable);
metadataTable.createTopic(topic);
List<PayloadTable.Entry> entries = new ArrayList<>();
entries.add(new TestPayloadEntry(topicId, GENERATION, "payloaddata", 101, (short) 0));
payloadTable.store(entries.iterator());
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
Assert.assertTrue(iterator.hasNext());
PayloadTable.Entry entry = iterator.next();
Assert.assertFalse(iterator.hasNext());
Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
Assert.assertEquals(101L, entry.getTransactionWritePointer());
iterator.close();
forceFlushAndCompact(Table.PAYLOAD);
// Entry should still be there since ttl has not expired
iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
Assert.assertTrue(iterator.hasNext());
entry = iterator.next();
Assert.assertFalse(iterator.hasNext());
Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
Assert.assertEquals(101L, entry.getTransactionWritePointer());
iterator.close();
TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
forceFlushAndCompact(Table.PAYLOAD);
iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
Assert.assertFalse(iterator.hasNext());
iterator.close();
metadataTable.deleteTopic(topicId);
}
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class DataCleanupTest method testOldGenCleanup.
@Test
public void testOldGenCleanup() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("oldGenCleanup");
TopicMetadata topic = new TopicMetadata(topicId, TopicMetadata.TTL_KEY, "100000", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
try (MetadataTable metadataTable = getMetadataTable()) {
int txWritePtr = 100;
metadataTable.createTopic(topic);
List<MessageTable.Entry> entries = new ArrayList<>();
List<PayloadTable.Entry> pentries = new ArrayList<>();
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
entries.add(new TestMessageEntry(topicId, GENERATION, "data", txWritePtr, (short) 0));
pentries.add(new TestPayloadEntry(topicId, GENERATION, "data", txWritePtr, (short) 0));
try (MessageTable messageTable = getMessageTable(topic);
PayloadTable payloadTable = getPayloadTable(topic)) {
messageTable.store(entries.iterator());
payloadTable.store(pentries.iterator());
}
// Fetch the entries and make sure we are able to read it
try (MessageTable messageTable = getMessageTable(topic);
CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
checkMessageEntry(iterator, txWritePtr);
}
try (PayloadTable payloadTable = getPayloadTable(topic);
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
checkPayloadEntry(iterator, txWritePtr);
}
// Now run full compaction
forceFlushAndCompact(Table.MESSAGE);
forceFlushAndCompact(Table.PAYLOAD);
// Fetch the entries and make sure we are able to read it
try (MessageTable messageTable = getMessageTable(topic);
CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
checkMessageEntry(iterator, txWritePtr);
}
try (PayloadTable payloadTable = getPayloadTable(topic);
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
checkPayloadEntry(iterator, txWritePtr);
}
// delete the topic and recreate it with an incremented generation
metadataTable.deleteTopic(topicId);
Map<String, String> newProperties = new HashMap<>(topic.getProperties());
newProperties.put(TopicMetadata.GENERATION_KEY, Integer.toString(topic.getGeneration() + 1));
topic = new TopicMetadata(topicId, newProperties);
metadataTable.createTopic(topic);
// Sleep so that the metadata cache in coprocessor expires
TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
forceFlushAndCompact(Table.MESSAGE);
forceFlushAndCompact(Table.PAYLOAD);
topic = metadataTable.getMetadata(topicId);
try (MessageTable messageTable = getMessageTable(topic);
CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
Assert.assertFalse(iterator.hasNext());
}
try (PayloadTable payloadTable = getPayloadTable(topic);
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
Assert.assertFalse(iterator.hasNext());
}
}
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class DataCleanupTest method testMessageTTLCleanup.
@Test
public void testMessageTTLCleanup() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("t1");
TopicMetadata topic = new TopicMetadata(topicId, "ttl", "3", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
try (MetadataTable metadataTable = getMetadataTable();
MessageTable messageTable = getMessageTable(topic);
PayloadTable payloadTable = getPayloadTable(topic)) {
Assert.assertNotNull(payloadTable);
metadataTable.createTopic(topic);
List<MessageTable.Entry> entries = new ArrayList<>();
entries.add(new TestMessageEntry(topicId, GENERATION, "data", 100, (short) 0));
messageTable.store(entries.iterator());
// Fetch the entries and make sure we are able to read it
CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
Assert.assertTrue(iterator.hasNext());
MessageTable.Entry entry = iterator.next();
Assert.assertFalse(iterator.hasNext());
Assert.assertEquals(100, entry.getTransactionWritePointer());
Assert.assertEquals("data", Bytes.toString(entry.getPayload()));
iterator.close();
forceFlushAndCompact(Table.MESSAGE);
// Entry should still be there since the ttl has not expired
iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
entry = iterator.next();
Assert.assertFalse(iterator.hasNext());
Assert.assertEquals(100, entry.getTransactionWritePointer());
Assert.assertEquals("data", Bytes.toString(entry.getPayload()));
iterator.close();
TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
forceFlushAndCompact(Table.MESSAGE);
iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
Assert.assertFalse(iterator.hasNext());
iterator.close();
}
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class HBaseMetadataTable method scanTopics.
/**
* Scans the HBase table to get a list of {@link TopicId}.
*/
private List<TopicId> scanTopics(ScanBuilder scanBuilder) throws IOException {
Scan scan = scanBuilder.setFilter(new FirstKeyOnlyFilter()).setCaching(scanCacheRows).build();
try {
List<TopicId> topicIds = new ArrayList<>();
try (ResultScanner resultScanner = table.getScanner(scan)) {
for (Result result : resultScanner) {
TopicId topicId = MessagingUtils.toTopicId(result.getRow());
byte[] value = result.getValue(columnFamily, COL);
Map<String, String> properties = GSON.fromJson(Bytes.toString(value), MAP_TYPE);
TopicMetadata metadata = new TopicMetadata(topicId, properties);
if (metadata.exists()) {
topicIds.add(topicId);
}
}
}
return topicIds;
} catch (IOException e) {
throw exceptionHandler.handle(e);
}
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class LevelDBMetadataTable method createTopic.
@Override
public void createTopic(TopicMetadata topicMetadata) throws TopicAlreadyExistsException, IOException {
try {
TopicId topicId = topicMetadata.getTopicId();
byte[] key = MessagingUtils.toMetadataRowKey(topicId);
TreeMap<String, String> properties = new TreeMap<>(topicMetadata.getProperties());
properties.put(TopicMetadata.GENERATION_KEY, MessagingUtils.Constants.DEFAULT_GENERATION);
synchronized (this) {
byte[] tableValue = levelDB.get(key);
if (tableValue != null) {
Map<String, String> oldProperties = GSON.fromJson(Bytes.toString(tableValue), MAP_TYPE);
TopicMetadata metadata = new TopicMetadata(topicId, oldProperties);
if (metadata.exists()) {
throw new TopicAlreadyExistsException(topicId.getNamespace(), topicId.getTopic());
}
int newGenerationId = (metadata.getGeneration() * -1) + 1;
properties.put(TopicMetadata.GENERATION_KEY, Integer.toString(newGenerationId));
}
byte[] value = Bytes.toBytes(GSON.toJson(properties, MAP_TYPE));
levelDB.put(key, value, WRITE_OPTIONS);
}
} catch (DBException e) {
throw new IOException(e);
}
}
Aggregations