use of co.cask.cdap.messaging.store.ImmutableMessageTableEntry in project cdap by caskdata.
the class CachingMessageTableTest method testCachePruning.
@Test
public void testCachePruning() throws Exception {
long txGracePeriod = 6;
CConfiguration cConf = CConfiguration.create();
cConf.setLong(CachingMessageTable.PRUNE_GRACE_PERIOD, txGracePeriod);
// Creates a CachingMessageTable with a controlled time provider
final AtomicLong currentTimeMillis = new AtomicLong(0);
MessageTable messageTable = new CachingMessageTable(cConf, super.getMessageTable(), cacheProvider, new TimeProvider() {
@Override
public long currentTimeMillis() {
return currentTimeMillis.get();
}
});
// Insert 10 entries, with different publish time
TopicMetadata metadata = new TopicMetadata(NamespaceId.DEFAULT.topic("test"), TopicMetadata.GENERATION_KEY, 1, TopicMetadata.TTL_KEY, 86400);
for (int i = 0; i < 10; i++) {
// Key is (topic, generation, publish time, sequence id)
byte[] key = Bytes.concat(MessagingUtils.toDataKeyPrefix(metadata.getTopicId(), metadata.getGeneration()), Bytes.toBytes((long) i), Bytes.toBytes((short) 0));
// Store a message with a write pointer
messageTable.store(Collections.singleton(new ImmutableMessageTableEntry(key, Bytes.toBytes("Payload " + i), Bytes.toBytes((long) i))).iterator());
}
// Update the current time to 11
currentTimeMillis.set(11);
// Fetch from the table without transaction, should get all entries.
try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, 0, 100, null)) {
List<MessageTable.Entry> entries = Lists.newArrayList(iter);
Assert.assertEquals(10, entries.size());
// All entries must be from the cache
for (MessageTable.Entry entry : entries) {
Assert.assertTrue(entry instanceof CachingMessageTable.CacheMessageTableEntry);
}
}
// Fetch with a transaction, with start time older than tx grace period / 2
Transaction tx = new Transaction(10, 11, new long[0], new long[0], 11);
long startTime = currentTimeMillis.get() - txGracePeriod / 2 - 1;
try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, startTime, 100, tx)) {
List<MessageTable.Entry> entries = Lists.newArrayList(iter);
// Should get three entries (7, 8, 9)
Assert.assertEquals(3, entries.size());
// The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
// from cache (current time = 11, grace period = 3)
Iterator<MessageTable.Entry> iterator = entries.iterator();
Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
}
// Fetch with a transaction, with start messageId publish time older than tx grace period / 2
byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(startTime, (short) 0, 0L, (short) 0, rawId, 0);
MessageId messageId = new MessageId(rawId);
try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, messageId, true, 100, tx)) {
List<MessageTable.Entry> entries = Lists.newArrayList(iter);
// Should get three entries (7, 8, 9)
Assert.assertEquals(3, entries.size());
// The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
// from cache (current time = 11, grace period = 3)
Iterator<MessageTable.Entry> iterator = entries.iterator();
Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
}
}
use of co.cask.cdap.messaging.store.ImmutableMessageTableEntry in project cdap by caskdata.
the class LevelDBMessageTable method pruneMessages.
/**
* Delete messages of a {@link TopicId} that has exceeded the TTL or if it belongs to an older generation
*
* @param topicMetadata {@link TopicMetadata}
* @param currentTime current timestamp
* @throws IOException error occurred while trying to delete a row in LevelDB
*/
public void pruneMessages(TopicMetadata topicMetadata, long currentTime) throws IOException {
WriteBatch writeBatch = levelDB.createWriteBatch();
long ttlInMs = TimeUnit.SECONDS.toMillis(topicMetadata.getTTL());
byte[] startRow = MessagingUtils.toDataKeyPrefix(topicMetadata.getTopicId(), Integer.parseInt(MessagingUtils.Constants.DEFAULT_GENERATION));
byte[] stopRow = Bytes.stopKeyForPrefix(startRow);
try (CloseableIterator<Map.Entry<byte[], byte[]>> rowIterator = new DBScanIterator(levelDB, startRow, stopRow)) {
while (rowIterator.hasNext()) {
Map.Entry<byte[], byte[]> entry = rowIterator.next();
MessageTable.Entry messageTableEntry = new ImmutableMessageTableEntry(entry.getKey(), null, null);
int dataGeneration = messageTableEntry.getGeneration();
int currGeneration = topicMetadata.getGeneration();
if (MessagingUtils.isOlderGeneration(dataGeneration, currGeneration)) {
writeBatch.delete(entry.getKey());
continue;
}
if ((dataGeneration == Math.abs(currGeneration)) && ((currentTime - messageTableEntry.getPublishTimestamp()) > ttlInMs)) {
writeBatch.delete(entry.getKey());
} else {
// since the entries are sorted by time.
break;
}
}
}
try {
levelDB.write(writeBatch, WRITE_OPTIONS);
} catch (DBException ex) {
throw new IOException(ex);
}
}
Aggregations