Search in sources :

Example 6 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class DataCleanupTest method testOldGenCleanup.

@Test
public void testOldGenCleanup() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("oldGenCleanup");
    TopicMetadata topic = new TopicMetadata(topicId, TopicMetadata.TTL_KEY, "100000", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
    try (MetadataTable metadataTable = getMetadataTable()) {
        int txWritePtr = 100;
        metadataTable.createTopic(topic);
        List<MessageTable.Entry> entries = new ArrayList<>();
        List<PayloadTable.Entry> pentries = new ArrayList<>();
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        entries.add(new TestMessageEntry(topicId, GENERATION, "data", txWritePtr, (short) 0));
        pentries.add(new TestPayloadEntry(topicId, GENERATION, "data", txWritePtr, (short) 0));
        try (MessageTable messageTable = getMessageTable(topic);
            PayloadTable payloadTable = getPayloadTable(topic)) {
            messageTable.store(entries.iterator());
            payloadTable.store(pentries.iterator());
        }
        // Fetch the entries and make sure we are able to read it
        try (MessageTable messageTable = getMessageTable(topic);
            CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            checkMessageEntry(iterator, txWritePtr);
        }
        try (PayloadTable payloadTable = getPayloadTable(topic);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
            checkPayloadEntry(iterator, txWritePtr);
        }
        // Now run full compaction
        forceFlushAndCompact(Table.MESSAGE);
        forceFlushAndCompact(Table.PAYLOAD);
        // Fetch the entries and make sure we are able to read it
        try (MessageTable messageTable = getMessageTable(topic);
            CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            checkMessageEntry(iterator, txWritePtr);
        }
        try (PayloadTable payloadTable = getPayloadTable(topic);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
            checkPayloadEntry(iterator, txWritePtr);
        }
        // delete the topic and recreate it with an incremented generation
        metadataTable.deleteTopic(topicId);
        Map<String, String> newProperties = new HashMap<>(topic.getProperties());
        newProperties.put(TopicMetadata.GENERATION_KEY, Integer.toString(topic.getGeneration() + 1));
        topic = new TopicMetadata(topicId, newProperties);
        metadataTable.createTopic(topic);
        // Sleep so that the metadata cache in coprocessor expires
        TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
        forceFlushAndCompact(Table.MESSAGE);
        forceFlushAndCompact(Table.PAYLOAD);
        topic = metadataTable.getMetadata(topicId);
        try (MessageTable messageTable = getMessageTable(topic);
            CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            Assert.assertFalse(iterator.hasNext());
        }
        try (PayloadTable payloadTable = getPayloadTable(topic);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
            Assert.assertFalse(iterator.hasNext());
        }
    }
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) TopicId(io.cdap.cdap.proto.id.TopicId) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Example 7 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class LevelDBMessageTableTest method testMultiPartitionReadWrite.

@Test
public void testMultiPartitionReadWrite() throws Exception {
    TopicId topicId = new TopicId("default", "multipart");
    int generation = 1;
    TopicMetadata topicMetadata = new TopicMetadata(topicId, Collections.singletonMap(TopicMetadata.GENERATION_KEY, String.valueOf(generation)));
    try (MessageTable table = tableFactory.createMessageTable(topicMetadata)) {
        List<MessageTable.Entry> writes = new ArrayList<>();
        Map<Long, Byte> expected = new HashMap<>();
        for (int i = 0; i < 10 * PARTITION_SECONDS; i++) {
            long publishTime = i * 1000;
            expected.put(publishTime, (byte) i);
            writes.add(new TestMessageEntry(topicId, generation, publishTime, 0, null, new byte[] { (byte) i }));
        }
        table.store(writes.iterator());
        Map<Long, Byte> actual = new HashMap<>();
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        try (CloseableIterator<MessageTable.Entry> iter = table.fetch(topicMetadata, new MessageId(messageId), true, 100, null)) {
            while (iter.hasNext()) {
                MessageTable.Entry entry = iter.next();
                actual.put(entry.getPublishTimestamp(), entry.getPayload()[0]);
            }
        }
        Assert.assertEquals(expected, actual);
    }
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) TestMessageEntry(io.cdap.cdap.messaging.store.TestMessageEntry) MessageTable(io.cdap.cdap.messaging.store.MessageTable) TopicId(io.cdap.cdap.proto.id.TopicId) TestMessageEntry(io.cdap.cdap.messaging.store.TestMessageEntry) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test) MessageTableTest(io.cdap.cdap.messaging.store.MessageTableTest)

Example 8 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class DefaultMetricStore method getMetricsProcessorStats.

/**
 * Read the metrics processing stats from meta table and return the map of topic information to stats
 * @return Map of topic to metrics processing stats
 * @throws Exception
 */
@Override
public Map<String, MetricsProcessorStatus> getMetricsProcessorStats() throws Exception {
    MetricsConsumerMetaTable metaTable = metaTableSupplier.get();
    Map<String, MetricsProcessorStatus> processMap = new HashMap<>();
    for (TopicId topicId : metricsTopics) {
        TopicProcessMeta topicProcessMeta = metaTable.getTopicProcessMeta(new TopicIdMetaKey(topicId));
        if (topicProcessMeta != null) {
            MessageId messageId = new MessageId(topicProcessMeta.getMessageId());
            MetricsMessageId metricsMessageId = new MetricsMessageId(messageId.getPublishTimestamp(), messageId.getSequenceId(), messageId.getPayloadWriteTimestamp(), messageId.getPayloadSequenceId());
            processMap.put(topicId.getTopic(), new MetricsProcessorStatus(metricsMessageId, topicProcessMeta.getOldestMetricsTimestamp(), topicProcessMeta.getLatestMetricsTimestamp(), topicProcessMeta.getMessagesProcessed(), topicProcessMeta.getLastProcessedTimestamp()));
        }
    }
    return processMap;
}
Also used : MetricsConsumerMetaTable(io.cdap.cdap.metrics.process.MetricsConsumerMetaTable) TopicProcessMeta(io.cdap.cdap.metrics.process.TopicProcessMeta) TopicIdMetaKey(io.cdap.cdap.metrics.process.TopicIdMetaKey) HashMap(java.util.HashMap) TopicId(io.cdap.cdap.proto.id.TopicId) MetricsProcessorStatus(io.cdap.cdap.api.metrics.MetricsProcessorStatus) MetricsMessageId(io.cdap.cdap.api.metrics.MetricsMessageId) MessageId(io.cdap.cdap.messaging.data.MessageId) MetricsMessageId(io.cdap.cdap.api.metrics.MetricsMessageId)

Example 9 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class PayloadTableTest method testConcurrentWrites.

@Test
public void testConcurrentWrites() throws Exception {
    // Create two threads, each of them writes to a different topic with two events in one store call.
    // The iterators in the two threads would alternate to produce payload. This is for testing CDAP-12013
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch storeCompletion = new CountDownLatch(2);
    for (int i = 0; i < 2; i++) {
        final TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
        TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
        try (MetadataTable metadataTable = getMetadataTable()) {
            metadataTable.createTopic(metadata);
        }
        final int threadId = i;
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                try (PayloadTable payloadTable = getPayloadTable(metadata)) {
                    payloadTable.store(new AbstractIterator<PayloadTable.Entry>() {

                        short messageCount = 0;

                        @Override
                        protected PayloadTable.Entry computeNext() {
                            if (messageCount >= 2) {
                                return endOfData();
                            }
                            try {
                                barrier.await();
                            } catch (Exception e) {
                                throw Throwables.propagate(e);
                            }
                            return new TestPayloadEntry(topicId, GENERATION, threadId, 0, messageCount, Bytes.toBytes("message " + threadId + " " + messageCount++));
                        }
                    });
                    storeCompletion.countDown();
                } catch (Exception e) {
                    LOG.error("Failed to store to MessageTable", e);
                }
                return null;
            }
        });
    }
    executor.shutdown();
    Assert.assertTrue(storeCompletion.await(5, TimeUnit.SECONDS));
    // Read from each topic. Each topic should have two messages
    for (int i = 0; i < 2; i++) {
        TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
        TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
        byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0, (short) 0, rawId, 0);
        MessageId messageId = new MessageId(rawId);
        try (PayloadTable payloadTable = getPayloadTable(metadata);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(metadata, i, messageId, true, 10)) {
            List<PayloadTable.Entry> entries = Lists.newArrayList(iterator);
            Assert.assertEquals(2, entries.size());
            int count = 0;
            for (PayloadTable.Entry entry : entries) {
                Assert.assertEquals("message " + i + " " + count++, Bytes.toString(entry.getPayload()));
            }
        }
    }
}
Also used : CountDownLatch(java.util.concurrent.CountDownLatch) CyclicBarrier(java.util.concurrent.CyclicBarrier) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) TopicId(io.cdap.cdap.proto.id.TopicId) AbstractIterator(com.google.common.collect.AbstractIterator) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Example 10 with MessageId

use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.

the class PayloadTableTest method testSingleMessage.

@Test
public void testSingleMessage() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("singlePayload");
    TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
    String payload = "data";
    long txWritePtr = 123L;
    try (MetadataTable metadataTable = getMetadataTable();
        PayloadTable table = getPayloadTable(metadata)) {
        metadataTable.createTopic(metadata);
        List<PayloadTable.Entry> entryList = new ArrayList<>();
        entryList.add(new TestPayloadEntry(topicId, GENERATION, txWritePtr, 1L, (short) 1, Bytes.toBytes(payload)));
        table.store(entryList.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(metadata, txWritePtr, new MessageId(messageId), false, Integer.MAX_VALUE)) {
            // Fetch not including the first message, expect empty
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(metadata, txWritePtr, new MessageId(messageId), true, Integer.MAX_VALUE)) {
            // Fetch including the first message
            Assert.assertTrue(iterator.hasNext());
            PayloadTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertEquals(txWritePtr, entry.getTransactionWritePointer());
            Assert.assertFalse(iterator.hasNext());
        }
    }
}
Also used : ArrayList(java.util.ArrayList) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) TopicId(io.cdap.cdap.proto.id.TopicId) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Aggregations

MessageId (io.cdap.cdap.messaging.data.MessageId)30 Test (org.junit.Test)26 TopicMetadata (io.cdap.cdap.messaging.TopicMetadata)24 TopicId (io.cdap.cdap.proto.id.TopicId)24 ArrayList (java.util.ArrayList)22 TimeProvider (io.cdap.cdap.common.utils.TimeProvider)8 RawMessage (io.cdap.cdap.messaging.data.RawMessage)8 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)8 MessageTable (io.cdap.cdap.messaging.store.MessageTable)6 HashMap (java.util.HashMap)6 Transaction (org.apache.tephra.Transaction)6 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)4 RollbackDetail (io.cdap.cdap.messaging.RollbackDetail)4 MessageTableTest (io.cdap.cdap.messaging.store.MessageTableTest)4 TestMessageEntry (io.cdap.cdap.messaging.store.TestMessageEntry)4 CountDownLatch (java.util.concurrent.CountDownLatch)4 AbstractIterator (com.google.common.collect.AbstractIterator)2 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)2 TopicNotFoundException (io.cdap.cdap.api.messaging.TopicNotFoundException)2 MetricsCollector (io.cdap.cdap.api.metrics.MetricsCollector)2