use of io.cdap.cdap.messaging.store.MessageTable in project cdap by caskdata.
the class CoreMessagingService method rollback.
@Override
public void rollback(TopicId topicId, RollbackDetail rollbackDetail) throws TopicNotFoundException, IOException {
TopicMetadata metadata = getTopic(topicId);
Exception failure = null;
try (MessageTable messageTable = createMessageTable(metadata)) {
messageTable.rollback(metadata, rollbackDetail);
} catch (Exception e) {
failure = e;
}
// Throw if there is any failure in rollback.
if (failure != null) {
Throwables.propagateIfPossible(failure, TopicNotFoundException.class, IOException.class);
throw Throwables.propagate(failure);
}
}
use of io.cdap.cdap.messaging.store.MessageTable in project cdap by caskdata.
the class LevelDBMessageTableTest method testMultiPartitionReadWrite.
@Test
public void testMultiPartitionReadWrite() throws Exception {
TopicId topicId = new TopicId("default", "multipart");
int generation = 1;
TopicMetadata topicMetadata = new TopicMetadata(topicId, Collections.singletonMap(TopicMetadata.GENERATION_KEY, String.valueOf(generation)));
try (MessageTable table = tableFactory.createMessageTable(topicMetadata)) {
List<MessageTable.Entry> writes = new ArrayList<>();
Map<Long, Byte> expected = new HashMap<>();
for (int i = 0; i < 10 * PARTITION_SECONDS; i++) {
long publishTime = i * 1000;
expected.put(publishTime, (byte) i);
writes.add(new TestMessageEntry(topicId, generation, publishTime, 0, null, new byte[] { (byte) i }));
}
table.store(writes.iterator());
Map<Long, Byte> actual = new HashMap<>();
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
try (CloseableIterator<MessageTable.Entry> iter = table.fetch(topicMetadata, new MessageId(messageId), true, 100, null)) {
while (iter.hasNext()) {
MessageTable.Entry entry = iter.next();
actual.put(entry.getPublishTimestamp(), entry.getPayload()[0]);
}
}
Assert.assertEquals(expected, actual);
}
}
use of io.cdap.cdap.messaging.store.MessageTable in project cdap by caskdata.
the class CachingMessageTableTest method testCachePruning.
@Test
public void testCachePruning() throws Exception {
long txGracePeriod = 6;
CConfiguration cConf = CConfiguration.create();
cConf.setLong(CachingMessageTable.PRUNE_GRACE_PERIOD, txGracePeriod);
// Insert 10 entries, with different publish time
TopicMetadata metadata = new TopicMetadata(NamespaceId.DEFAULT.topic("test"), TopicMetadata.GENERATION_KEY, 1, TopicMetadata.TTL_KEY, 86400);
// Creates a CachingMessageTable with a controlled time provider
final AtomicLong currentTimeMillis = new AtomicLong(0);
MessageTable messageTable = new CachingMessageTable(cConf, super.getMessageTable(metadata), cacheProvider, new TimeProvider() {
@Override
public long currentTimeMillis() {
return currentTimeMillis.get();
}
});
for (int i = 0; i < 10; i++) {
// Key is (topic, generation, publish time, sequence id)
byte[] key = Bytes.concat(MessagingUtils.toDataKeyPrefix(metadata.getTopicId(), metadata.getGeneration()), Bytes.toBytes((long) i), Bytes.toBytes((short) 0));
// Store a message with a write pointer
messageTable.store(Collections.singleton(new ImmutableMessageTableEntry(key, Bytes.toBytes("Payload " + i), Bytes.toBytes((long) i))).iterator());
}
// Update the current time to 11
currentTimeMillis.set(11);
// Fetch from the table without transaction, should get all entries.
try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, 0, 100, null)) {
List<MessageTable.Entry> entries = Lists.newArrayList(iter);
Assert.assertEquals(10, entries.size());
// All entries must be from the cache
for (MessageTable.Entry entry : entries) {
Assert.assertTrue(entry instanceof CachingMessageTable.CacheMessageTableEntry);
}
}
// Fetch with a transaction, with start time older than tx grace period / 2
Transaction tx = new Transaction(10, 11, new long[0], new long[0], 11);
long startTime = currentTimeMillis.get() - txGracePeriod / 2 - 1;
try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, startTime, 100, tx)) {
List<MessageTable.Entry> entries = Lists.newArrayList(iter);
// Should get three entries (7, 8, 9)
Assert.assertEquals(3, entries.size());
// The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
// from cache (current time = 11, grace period = 3)
Iterator<MessageTable.Entry> iterator = entries.iterator();
Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
}
// Fetch with a transaction, with start messageId publish time older than tx grace period / 2
byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(startTime, (short) 0, 0L, (short) 0, rawId, 0);
MessageId messageId = new MessageId(rawId);
try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, messageId, true, 100, tx)) {
List<MessageTable.Entry> entries = Lists.newArrayList(iter);
// Should get three entries (7, 8, 9)
Assert.assertEquals(3, entries.size());
// The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
// from cache (current time = 11, grace period = 3)
Iterator<MessageTable.Entry> iterator = entries.iterator();
Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
}
}
use of io.cdap.cdap.messaging.store.MessageTable in project cdap by caskdata.
the class LevelDBMessageTableTest method testUpgrade.
@Test
public void testUpgrade() throws Exception {
File baseDir = tmpFolder.newFolder();
String tableName = "tms.messages";
CConfiguration cConf = CConfiguration.create();
cConf.set(Constants.MessagingSystem.LOCAL_DATA_DIR, baseDir.getAbsolutePath());
cConf.set(Constants.MessagingSystem.MESSAGE_TABLE_NAME, tableName);
cConf.set(Constants.MessagingSystem.LOCAL_DATA_PARTITION_SECONDS, Integer.toString(1));
LevelDBTableFactory tableFactory = new LevelDBTableFactory(cConf);
TopicId topicId = new TopicId("default", "preview");
int generation = 1;
TopicMetadata topicMetadata = new TopicMetadata(topicId, Collections.singletonMap(TopicMetadata.GENERATION_KEY, String.valueOf(generation)));
// write a message to a table, then rename the underlying directory to the old format
long publishTime = 1000;
try (MessageTable table = tableFactory.createMessageTable(topicMetadata)) {
List<MessageTable.Entry> writes = new ArrayList<>();
writes.add(new TestMessageEntry(topicId, generation, publishTime, 0, null, new byte[] { 0 }));
table.store(writes.iterator());
}
tableFactory.close();
File topicDir = LevelDBTableFactory.getMessageTablePath(baseDir, topicId, generation, tableName);
File partitionDir = LevelDBPartitionManager.getPartitionDir(topicDir, 0, publishTime + 1000);
File oldDir = new File(baseDir, topicDir.getName().substring(LevelDBTableFactory.MESSAGE_TABLE_VERSION.length() + 1));
Files.move(partitionDir.toPath(), oldDir.toPath());
Files.delete(topicDir.toPath());
// now run the upgrade and make sure the table is readable.
tableFactory = new LevelDBTableFactory(cConf);
tableFactory.init();
try (MessageTable table = tableFactory.createMessageTable(topicMetadata)) {
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
try (CloseableIterator<MessageTable.Entry> iter = table.fetch(topicMetadata, new MessageId(messageId), true, 100, null)) {
Assert.assertTrue(iter.hasNext());
MessageTable.Entry entry = iter.next();
Assert.assertEquals(publishTime, entry.getPublishTimestamp());
Assert.assertFalse(iter.hasNext());
}
}
}
use of io.cdap.cdap.messaging.store.MessageTable in project cdap by caskdata.
the class HBaseTableCoprocessorTestRun method testInvalidTx.
@Test
public void testInvalidTx() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("invalidTx");
TopicMetadata topic = new TopicMetadata(topicId, TopicMetadata.TTL_KEY, "1000000", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
try (MetadataTable metadataTable = getMetadataTable();
MessageTable messageTable = getMessageTable(topic)) {
metadataTable.createTopic(topic);
List<MessageTable.Entry> entries = new ArrayList<>();
long invalidTxWritePtr = invalidList.toRawList().get(0);
entries.add(new TestMessageEntry(topicId, GENERATION, "data", invalidTxWritePtr, (short) 0));
messageTable.store(entries.iterator());
// Fetch the entries and make sure we are able to read it
try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
checkEntry(iterator, invalidTxWritePtr);
}
// Fetch the entries with tx and make sure we are able to read it
Transaction tx = new Transaction(V[8], V[8], new long[0], new long[0], -1);
try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, tx)) {
checkEntry(iterator, invalidTxWritePtr);
}
// Now run full compaction
forceFlushAndCompact(Table.MESSAGE);
// Try to fetch the entry non-transactionally and the entry should still be there
try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
checkEntry(iterator, invalidTxWritePtr);
}
// Fetch the entries transactionally and we should see no entries returned
try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, tx)) {
Assert.assertFalse(iterator.hasNext());
}
metadataTable.deleteTopic(topicId);
// Sleep so that the metadata cache expires
TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
forceFlushAndCompact(Table.MESSAGE);
// Test deletion of messages from a deleted topic
try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
Assert.assertFalse(iterator.hasNext());
}
}
}
Aggregations