use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.
the class DataCleanupTest method testOldGenCleanup.
@Test
public void testOldGenCleanup() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("oldGenCleanup");
TopicMetadata topic = new TopicMetadata(topicId, TopicMetadata.TTL_KEY, "100000", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
try (MetadataTable metadataTable = getMetadataTable()) {
int txWritePtr = 100;
metadataTable.createTopic(topic);
List<MessageTable.Entry> entries = new ArrayList<>();
List<PayloadTable.Entry> pentries = new ArrayList<>();
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
entries.add(new TestMessageEntry(topicId, GENERATION, "data", txWritePtr, (short) 0));
pentries.add(new TestPayloadEntry(topicId, GENERATION, "data", txWritePtr, (short) 0));
try (MessageTable messageTable = getMessageTable(topic);
PayloadTable payloadTable = getPayloadTable(topic)) {
messageTable.store(entries.iterator());
payloadTable.store(pentries.iterator());
}
// Fetch the entries and make sure we are able to read it
try (MessageTable messageTable = getMessageTable(topic);
CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
checkMessageEntry(iterator, txWritePtr);
}
try (PayloadTable payloadTable = getPayloadTable(topic);
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
checkPayloadEntry(iterator, txWritePtr);
}
// Now run full compaction
forceFlushAndCompact(Table.MESSAGE);
forceFlushAndCompact(Table.PAYLOAD);
// Fetch the entries and make sure we are able to read it
try (MessageTable messageTable = getMessageTable(topic);
CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
checkMessageEntry(iterator, txWritePtr);
}
try (PayloadTable payloadTable = getPayloadTable(topic);
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
checkPayloadEntry(iterator, txWritePtr);
}
// delete the topic and recreate it with an incremented generation
metadataTable.deleteTopic(topicId);
Map<String, String> newProperties = new HashMap<>(topic.getProperties());
newProperties.put(TopicMetadata.GENERATION_KEY, Integer.toString(topic.getGeneration() + 1));
topic = new TopicMetadata(topicId, newProperties);
metadataTable.createTopic(topic);
// Sleep so that the metadata cache in coprocessor expires
TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
forceFlushAndCompact(Table.MESSAGE);
forceFlushAndCompact(Table.PAYLOAD);
topic = metadataTable.getMetadata(topicId);
try (MessageTable messageTable = getMessageTable(topic);
CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
Assert.assertFalse(iterator.hasNext());
}
try (PayloadTable payloadTable = getPayloadTable(topic);
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
Assert.assertFalse(iterator.hasNext());
}
}
}
use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.
the class LevelDBMessageTableTest method testMultiPartitionReadWrite.
@Test
public void testMultiPartitionReadWrite() throws Exception {
TopicId topicId = new TopicId("default", "multipart");
int generation = 1;
TopicMetadata topicMetadata = new TopicMetadata(topicId, Collections.singletonMap(TopicMetadata.GENERATION_KEY, String.valueOf(generation)));
try (MessageTable table = tableFactory.createMessageTable(topicMetadata)) {
List<MessageTable.Entry> writes = new ArrayList<>();
Map<Long, Byte> expected = new HashMap<>();
for (int i = 0; i < 10 * PARTITION_SECONDS; i++) {
long publishTime = i * 1000;
expected.put(publishTime, (byte) i);
writes.add(new TestMessageEntry(topicId, generation, publishTime, 0, null, new byte[] { (byte) i }));
}
table.store(writes.iterator());
Map<Long, Byte> actual = new HashMap<>();
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
try (CloseableIterator<MessageTable.Entry> iter = table.fetch(topicMetadata, new MessageId(messageId), true, 100, null)) {
while (iter.hasNext()) {
MessageTable.Entry entry = iter.next();
actual.put(entry.getPublishTimestamp(), entry.getPayload()[0]);
}
}
Assert.assertEquals(expected, actual);
}
}
use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.
the class DefaultMetricStore method getMetricsProcessorStats.
/**
* Read the metrics processing stats from meta table and return the map of topic information to stats
* @return Map of topic to metrics processing stats
* @throws Exception
*/
@Override
public Map<String, MetricsProcessorStatus> getMetricsProcessorStats() throws Exception {
MetricsConsumerMetaTable metaTable = metaTableSupplier.get();
Map<String, MetricsProcessorStatus> processMap = new HashMap<>();
for (TopicId topicId : metricsTopics) {
TopicProcessMeta topicProcessMeta = metaTable.getTopicProcessMeta(new TopicIdMetaKey(topicId));
if (topicProcessMeta != null) {
MessageId messageId = new MessageId(topicProcessMeta.getMessageId());
MetricsMessageId metricsMessageId = new MetricsMessageId(messageId.getPublishTimestamp(), messageId.getSequenceId(), messageId.getPayloadWriteTimestamp(), messageId.getPayloadSequenceId());
processMap.put(topicId.getTopic(), new MetricsProcessorStatus(metricsMessageId, topicProcessMeta.getOldestMetricsTimestamp(), topicProcessMeta.getLatestMetricsTimestamp(), topicProcessMeta.getMessagesProcessed(), topicProcessMeta.getLastProcessedTimestamp()));
}
}
return processMap;
}
use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.
the class PayloadTableTest method testConcurrentWrites.
@Test
public void testConcurrentWrites() throws Exception {
// Create two threads, each of them writes to a different topic with two events in one store call.
// The iterators in the two threads would alternate to produce payload. This is for testing CDAP-12013
ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
final CyclicBarrier barrier = new CyclicBarrier(2);
final CountDownLatch storeCompletion = new CountDownLatch(2);
for (int i = 0; i < 2; i++) {
final TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
try (MetadataTable metadataTable = getMetadataTable()) {
metadataTable.createTopic(metadata);
}
final int threadId = i;
executor.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
try (PayloadTable payloadTable = getPayloadTable(metadata)) {
payloadTable.store(new AbstractIterator<PayloadTable.Entry>() {
short messageCount = 0;
@Override
protected PayloadTable.Entry computeNext() {
if (messageCount >= 2) {
return endOfData();
}
try {
barrier.await();
} catch (Exception e) {
throw Throwables.propagate(e);
}
return new TestPayloadEntry(topicId, GENERATION, threadId, 0, messageCount, Bytes.toBytes("message " + threadId + " " + messageCount++));
}
});
storeCompletion.countDown();
} catch (Exception e) {
LOG.error("Failed to store to MessageTable", e);
}
return null;
}
});
}
executor.shutdown();
Assert.assertTrue(storeCompletion.await(5, TimeUnit.SECONDS));
// Read from each topic. Each topic should have two messages
for (int i = 0; i < 2; i++) {
TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0, (short) 0, rawId, 0);
MessageId messageId = new MessageId(rawId);
try (PayloadTable payloadTable = getPayloadTable(metadata);
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(metadata, i, messageId, true, 10)) {
List<PayloadTable.Entry> entries = Lists.newArrayList(iterator);
Assert.assertEquals(2, entries.size());
int count = 0;
for (PayloadTable.Entry entry : entries) {
Assert.assertEquals("message " + i + " " + count++, Bytes.toString(entry.getPayload()));
}
}
}
}
use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.
the class PayloadTableTest method testSingleMessage.
@Test
public void testSingleMessage() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("singlePayload");
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
String payload = "data";
long txWritePtr = 123L;
try (MetadataTable metadataTable = getMetadataTable();
PayloadTable table = getPayloadTable(metadata)) {
metadataTable.createTopic(metadata);
List<PayloadTable.Entry> entryList = new ArrayList<>();
entryList.add(new TestPayloadEntry(topicId, GENERATION, txWritePtr, 1L, (short) 1, Bytes.toBytes(payload)));
table.store(entryList.iterator());
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(metadata, txWritePtr, new MessageId(messageId), false, Integer.MAX_VALUE)) {
// Fetch not including the first message, expect empty
Assert.assertFalse(iterator.hasNext());
}
try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(metadata, txWritePtr, new MessageId(messageId), true, Integer.MAX_VALUE)) {
// Fetch including the first message
Assert.assertTrue(iterator.hasNext());
PayloadTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertEquals(txWritePtr, entry.getTransactionWritePointer());
Assert.assertFalse(iterator.hasNext());
}
}
}
Aggregations