Examples with MessageId io.cdap.cdap.messaging.data.MessageId used on opensource projects

Example 11 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class CachingMessageTableTest method testCachePruning.

@Test
public void testCachePruning() throws Exception {
    long txGracePeriod = 6;
    CConfiguration cConf = CConfiguration.create();
    cConf.setLong(CachingMessageTable.PRUNE_GRACE_PERIOD, txGracePeriod);
    // Insert 10 entries, with different publish time
    TopicMetadata metadata = new TopicMetadata(NamespaceId.DEFAULT.topic("test"), TopicMetadata.GENERATION_KEY, 1, TopicMetadata.TTL_KEY, 86400);
    // Creates a CachingMessageTable with a controlled time provider
    final AtomicLong currentTimeMillis = new AtomicLong(0);
    MessageTable messageTable = new CachingMessageTable(cConf, super.getMessageTable(metadata), cacheProvider, new TimeProvider() {

        @Override
        public long currentTimeMillis() {
            return currentTimeMillis.get();
        }
    });
    for (int i = 0; i < 10; i++) {
        // Key is (topic, generation, publish time, sequence id)
        byte[] key = Bytes.concat(MessagingUtils.toDataKeyPrefix(metadata.getTopicId(), metadata.getGeneration()), Bytes.toBytes((long) i), Bytes.toBytes((short) 0));
        // Store a message with a write pointer
        messageTable.store(Collections.singleton(new ImmutableMessageTableEntry(key, Bytes.toBytes("Payload " + i), Bytes.toBytes((long) i))).iterator());
    }
    // Update the current time to 11
    currentTimeMillis.set(11);
    // Fetch from the table without transaction, should get all entries.
    try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, 0, 100, null)) {
        List<MessageTable.Entry> entries = Lists.newArrayList(iter);
        Assert.assertEquals(10, entries.size());
        // All entries must be from the cache
        for (MessageTable.Entry entry : entries) {
            Assert.assertTrue(entry instanceof CachingMessageTable.CacheMessageTableEntry);
        }
    }
    // Fetch with a transaction, with start time older than tx grace period / 2
    Transaction tx = new Transaction(10, 11, new long[0], new long[0], 11);
    long startTime = currentTimeMillis.get() - txGracePeriod / 2 - 1;
    try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, startTime, 100, tx)) {
        List<MessageTable.Entry> entries = Lists.newArrayList(iter);
        // Should get three entries (7, 8, 9)
        Assert.assertEquals(3, entries.size());
        // The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
        // from cache (current time = 11, grace period = 3)
        Iterator<MessageTable.Entry> iterator = entries.iterator();
        Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
    }
    // Fetch with a transaction, with start messageId publish time older than tx grace period / 2
    byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
    MessageId.putRawId(startTime, (short) 0, 0L, (short) 0, rawId, 0);
    MessageId messageId = new MessageId(rawId);
    try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, messageId, true, 100, tx)) {
        List<MessageTable.Entry> entries = Lists.newArrayList(iter);
        // Should get three entries (7, 8, 9)
        Assert.assertEquals(3, entries.size());
        // The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
        // from cache (current time = 11, grace period = 3)
        Iterator<MessageTable.Entry> iterator = entries.iterator();
        Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
    }
}

Example 12 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class PayloadTableTest method testConsumption.

@Test
public void testConsumption() throws Exception {
    try (MetadataTable metadataTable = getMetadataTable();
        PayloadTable table1 = getPayloadTable(M1);
        PayloadTable table2 = getPayloadTable(M2)) {
        metadataTable.createTopic(M1);
        metadataTable.createTopic(M2);
        List<PayloadTable.Entry> entryList = new ArrayList<>();
        populateList(entryList);
        table1.store(entryList.iterator());
        table2.store(entryList.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        // Fetch data with 100 write pointer
        try (CloseableIterator<PayloadTable.Entry> iterator = table1.fetch(M1, 100, new MessageId(messageId), true, Integer.MAX_VALUE)) {
            checkData(iterator, 123, ImmutableSet.of(100L), 50);
        }
        // Fetch only 10 items with 101 write pointer
        try (CloseableIterator<PayloadTable.Entry> iterator = table1.fetch(M1, 101, new MessageId(messageId), true, 1)) {
            checkData(iterator, 123, ImmutableSet.of(101L), 1);
        }
        // Fetch items with 102 write pointer
        try (CloseableIterator<PayloadTable.Entry> iterator = table1.fetch(M1, 102, new MessageId(messageId), true, Integer.MAX_VALUE)) {
            checkData(iterator, 123, ImmutableSet.of(102L), 50);
        }
        // Fetch from t2 with 101 write pointer
        try (CloseableIterator<PayloadTable.Entry> iterator = table2.fetch(M2, 101, new MessageId(messageId), true, Integer.MAX_VALUE)) {
            checkData(iterator, 123, ImmutableSet.of(101L), 50);
        }
    }
}

Example 13 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testMaxSequence.

@Test
public void testMaxSequence() throws IOException {
    // This test the case when a single StoreRequest has more than SEQUENCE_ID_LIMIT (65536) payload.
    // Expected entries beyond the max seqId will be rolled to the next timestamp with seqId reset to start from 0
    // Generate SEQUENCE_ID_LIMIT + 1 payloads
    int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT + 1;
    List<String> payloads = new ArrayList<>(msgCount);
    for (int i = 0; i < msgCount; i++) {
        payloads.add(Integer.toString(i));
    }
    TopicId topicId = new NamespaceId("ns1").topic("t1");
    TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
    // Write the payloads
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter);
    writer.persist(new TestStoreRequest(topicId, payloads), metadata);
    List<RawMessage> messages = testWriter.getMessages().get(topicId);
    Assert.assertEquals(msgCount, messages.size());
    // The first SEQUENCE_ID_LIMIT messages should be with the same timestamp, with seqId from 0 to SEQUENCE_ID_LIMIT
    for (int i = 0; i < StoreRequestWriter.SEQUENCE_ID_LIMIT; i++) {
        MessageId id = new MessageId(messages.get(i).getId());
        Assert.assertEquals(0L, id.getPublishTimestamp());
        Assert.assertEquals((short) i, id.getSequenceId());
    }
    // The (SEQUENCE_ID_LIMIT + 1)th message should have a different timestamp and seqId = 0
    MessageId id = new MessageId(messages.get(msgCount - 1).getId());
    Assert.assertEquals(1L, id.getPublishTimestamp());
    Assert.assertEquals(0, id.getPayloadSequenceId());
}

Example 14 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class LevelDBMessageTableTest method testUpgrade.

@Test
public void testUpgrade() throws Exception {
    File baseDir = tmpFolder.newFolder();
    String tableName = "tms.messages";
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.MessagingSystem.LOCAL_DATA_DIR, baseDir.getAbsolutePath());
    cConf.set(Constants.MessagingSystem.MESSAGE_TABLE_NAME, tableName);
    cConf.set(Constants.MessagingSystem.LOCAL_DATA_PARTITION_SECONDS, Integer.toString(1));
    LevelDBTableFactory tableFactory = new LevelDBTableFactory(cConf);
    TopicId topicId = new TopicId("default", "preview");
    int generation = 1;
    TopicMetadata topicMetadata = new TopicMetadata(topicId, Collections.singletonMap(TopicMetadata.GENERATION_KEY, String.valueOf(generation)));
    // write a message to a table, then rename the underlying directory to the old format
    long publishTime = 1000;
    try (MessageTable table = tableFactory.createMessageTable(topicMetadata)) {
        List<MessageTable.Entry> writes = new ArrayList<>();
        writes.add(new TestMessageEntry(topicId, generation, publishTime, 0, null, new byte[] { 0 }));
        table.store(writes.iterator());
    }
    tableFactory.close();
    File topicDir = LevelDBTableFactory.getMessageTablePath(baseDir, topicId, generation, tableName);
    File partitionDir = LevelDBPartitionManager.getPartitionDir(topicDir, 0, publishTime + 1000);
    File oldDir = new File(baseDir, topicDir.getName().substring(LevelDBTableFactory.MESSAGE_TABLE_VERSION.length() + 1));
    Files.move(partitionDir.toPath(), oldDir.toPath());
    Files.delete(topicDir.toPath());
    // now run the upgrade and make sure the table is readable.
    tableFactory = new LevelDBTableFactory(cConf);
    tableFactory.init();
    try (MessageTable table = tableFactory.createMessageTable(topicMetadata)) {
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        try (CloseableIterator<MessageTable.Entry> iter = table.fetch(topicMetadata, new MessageId(messageId), true, 100, null)) {
            Assert.assertTrue(iter.hasNext());
            MessageTable.Entry entry = iter.next();
            Assert.assertEquals(publishTime, entry.getPublishTimestamp());
            Assert.assertFalse(iter.hasNext());
        }
    }
}

Example 15 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class CoreSchedulerServiceTest method getLastMessageId.

@Nullable
private MessageId getLastMessageId(final TopicId topic) {
    return TransactionRunners.run(transactionRunner, context -> {
        JobQueueTable jobQueue = JobQueueTable.getJobQueue(context, cConf);
        String id = jobQueue.retrieveSubscriberState(topic.getTopic());
        if (id == null) {
            return null;
        }
        byte[] bytes = Bytes.fromHexString(id);
        return new MessageId(bytes);
    });
}