Examples with MessageTable co.cask.cdap.messaging.store.MessageTable used on opensource projects

Example 1 with MessageTable

use of co.cask.cdap.messaging.store.MessageTable in project cdap by caskdata.

the class CachingMessageTableTest method testCachePruning.

@Test
public void testCachePruning() throws Exception {
    long txGracePeriod = 6;
    CConfiguration cConf = CConfiguration.create();
    cConf.setLong(CachingMessageTable.PRUNE_GRACE_PERIOD, txGracePeriod);
    // Creates a CachingMessageTable with a controlled time provider
    final AtomicLong currentTimeMillis = new AtomicLong(0);
    MessageTable messageTable = new CachingMessageTable(cConf, super.getMessageTable(), cacheProvider, new TimeProvider() {

        @Override
        public long currentTimeMillis() {
            return currentTimeMillis.get();
        }
    });
    // Insert 10 entries, with different publish time
    TopicMetadata metadata = new TopicMetadata(NamespaceId.DEFAULT.topic("test"), TopicMetadata.GENERATION_KEY, 1, TopicMetadata.TTL_KEY, 86400);
    for (int i = 0; i < 10; i++) {
        // Key is (topic, generation, publish time, sequence id)
        byte[] key = Bytes.concat(MessagingUtils.toDataKeyPrefix(metadata.getTopicId(), metadata.getGeneration()), Bytes.toBytes((long) i), Bytes.toBytes((short) 0));
        // Store a message with a write pointer
        messageTable.store(Collections.singleton(new ImmutableMessageTableEntry(key, Bytes.toBytes("Payload " + i), Bytes.toBytes((long) i))).iterator());
    }
    // Update the current time to 11
    currentTimeMillis.set(11);
    // Fetch from the table without transaction, should get all entries.
    try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, 0, 100, null)) {
        List<MessageTable.Entry> entries = Lists.newArrayList(iter);
        Assert.assertEquals(10, entries.size());
        // All entries must be from the cache
        for (MessageTable.Entry entry : entries) {
            Assert.assertTrue(entry instanceof CachingMessageTable.CacheMessageTableEntry);
        }
    }
    // Fetch with a transaction, with start time older than tx grace period / 2
    Transaction tx = new Transaction(10, 11, new long[0], new long[0], 11);
    long startTime = currentTimeMillis.get() - txGracePeriod / 2 - 1;
    try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, startTime, 100, tx)) {
        List<MessageTable.Entry> entries = Lists.newArrayList(iter);
        // Should get three entries (7, 8, 9)
        Assert.assertEquals(3, entries.size());
        // The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
        // from cache (current time = 11, grace period = 3)
        Iterator<MessageTable.Entry> iterator = entries.iterator();
        Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
    }
    // Fetch with a transaction, with start messageId publish time older than tx grace period / 2
    byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
    MessageId.putRawId(startTime, (short) 0, 0L, (short) 0, rawId, 0);
    MessageId messageId = new MessageId(rawId);
    try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, messageId, true, 100, tx)) {
        List<MessageTable.Entry> entries = Lists.newArrayList(iter);
        // Should get three entries (7, 8, 9)
        Assert.assertEquals(3, entries.size());
        // The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
        // from cache (current time = 11, grace period = 3)
        Iterator<MessageTable.Entry> iterator = entries.iterator();
        Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
    }
}

Example 2 with MessageTable

use of co.cask.cdap.messaging.store.MessageTable in project cdap by caskdata.

the class CoreMessagingService method rollback.

@Override
public void rollback(TopicId topicId, RollbackDetail rollbackDetail) throws TopicNotFoundException, IOException {
    TopicMetadata metadata = getTopic(topicId);
    Exception failure = null;
    try (MessageTable messageTable = createMessageTable(metadata)) {
        messageTable.rollback(metadata, rollbackDetail);
    } catch (Exception e) {
        failure = e;
    }
    // Throw if there is any failure in rollback.
    if (failure != null) {
        Throwables.propagateIfPossible(failure, TopicNotFoundException.class, IOException.class);
        throw Throwables.propagate(failure);
    }
}

Example 3 with MessageTable

use of co.cask.cdap.messaging.store.MessageTable in project cdap by caskdata.

the class HBaseTableCoprocessorTestRun method testInvalidTx.

@Test
public void testInvalidTx() throws Exception {
    try (MetadataTable metadataTable = getMetadataTable();
        MessageTable messageTable = getMessageTable()) {
        TopicId topicId = NamespaceId.DEFAULT.topic("invalidTx");
        TopicMetadata topic = new TopicMetadata(topicId, TopicMetadata.TTL_KEY, "1000000", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
        metadataTable.createTopic(topic);
        List<MessageTable.Entry> entries = new ArrayList<>();
        long invalidTxWritePtr = invalidList.toRawList().get(0);
        entries.add(new TestMessageEntry(topicId, GENERATION, "data", invalidTxWritePtr, (short) 0));
        messageTable.store(entries.iterator());
        // Fetch the entries and make sure we are able to read it
        try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            checkEntry(iterator, invalidTxWritePtr);
        }
        // Fetch the entries with tx and make sure we are able to read it
        Transaction tx = new Transaction(V[8], V[8], new long[0], new long[0], -1);
        try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, tx)) {
            checkEntry(iterator, invalidTxWritePtr);
        }
        // Now run full compaction
        forceFlushAndCompact(Table.MESSAGE);
        // Try to fetch the entry non-transactionally and the entry should still be there
        try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            checkEntry(iterator, invalidTxWritePtr);
        }
        // Fetch the entries transactionally and we should see no entries returned
        try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, tx)) {
            Assert.assertFalse(iterator.hasNext());
        }
        metadataTable.deleteTopic(topicId);
        // Sleep so that the metadata cache expires
        TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
        forceFlushAndCompact(Table.MESSAGE);
        // Test deletion of messages from a deleted topic
        try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            Assert.assertFalse(iterator.hasNext());
        }
    }
}