use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.
the class ConcurrentMessageWriterTest method testMultiMaxSequence.
@Test
public void testMultiMaxSequence() throws IOException, InterruptedException {
TopicId topicId = new NamespaceId("ns1").topic("t1");
final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
// This test the case when multiple StoreRequests combined exceeding the 65536 payload.
// See testMaxSequence() for more details when it matters
// Generate 3 StoreRequests, each with 43690 messages
int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT / 3 * 2;
int requestCount = 3;
List<StoreRequest> requests = new ArrayList<>();
for (int i = 0; i < requestCount; i++) {
List<String> payloads = new ArrayList<>(msgCount);
for (int j = 0; j < msgCount; j++) {
payloads.add(Integer.toString(j));
}
requests.add(new TestStoreRequest(topicId, payloads));
}
TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
// We use a custom metrics collector here to make all the persist calls reached the same latch,
// since we know that the ConcurrentMessageWriter will emit a metrics "persist.requested" after enqueued but
// before flushing.
// This will make all requests batched together
final CountDownLatch latch = new CountDownLatch(requestCount);
final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter, new MetricsCollector() {
@Override
public void increment(String metricName, long value) {
if ("persist.requested".equals(metricName)) {
latch.countDown();
Uninterruptibles.awaitUninterruptibly(latch);
}
}
@Override
public void gauge(String metricName, long value) {
LOG.info("MetricsContext.gauge: {} = {}", metricName, value);
}
});
ExecutorService executor = Executors.newFixedThreadPool(3);
for (final StoreRequest request : requests) {
executor.submit(new Runnable() {
@Override
public void run() {
try {
writer.persist(request, metadata);
} catch (IOException e) {
LOG.error("Failed to persist", e);
}
}
});
}
executor.shutdown();
Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
// Validates all messages are being written
List<RawMessage> messages = testWriter.getMessages().get(topicId);
Assert.assertEquals(requestCount * msgCount, messages.size());
// We expect the payload is in repeated sequence of [0..msgCount-1]
int expectedPayload = 0;
// The sequenceId should be (i % SEQUENCE_ID_LIMIT)
for (int i = 0; i < messages.size(); i++) {
RawMessage message = messages.get(i);
MessageId messageId = new MessageId(message.getId());
Assert.assertEquals(i / StoreRequestWriter.SEQUENCE_ID_LIMIT, messageId.getPublishTimestamp());
Assert.assertEquals((short) (i % StoreRequestWriter.SEQUENCE_ID_LIMIT), messageId.getSequenceId());
Assert.assertEquals(expectedPayload, Integer.parseInt(Bytes.toString(message.getPayload())));
expectedPayload = (expectedPayload + 1) % msgCount;
}
}
use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.
the class ConcurrentMessageWriterTest method testBasic.
@Test
public void testBasic() throws IOException {
TopicId topicId1 = new NamespaceId("ns1").topic("t1");
TopicId topicId2 = new NamespaceId("ns2").topic("t2");
TopicMetadata metadata1 = new TopicMetadata(topicId1, new HashMap<String, String>(), 1);
TopicMetadata metadata2 = new TopicMetadata(topicId2, new HashMap<String, String>(), 1);
TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter);
writer.persist(new TestStoreRequest(topicId1, Arrays.asList("1", "2", "3")), metadata1);
// There should be 3 messages being written
List<RawMessage> messages = testWriter.getMessages().get(topicId1);
Assert.assertEquals(3, messages.size());
// All messages should be written with timestamp 0
List<String> payloads = new ArrayList<>();
for (RawMessage message : messages) {
Assert.assertEquals(0L, new MessageId(message.getId()).getPublishTimestamp());
payloads.add(Bytes.toString(message.getPayload()));
}
Assert.assertEquals(Arrays.asList("1", "2", "3"), payloads);
// Write to another topic
writer.persist(new TestStoreRequest(topicId2, Arrays.asList("a", "b", "c")), metadata2);
// There should be 3 messages being written to topic2
messages = testWriter.getMessages().get(topicId2);
Assert.assertEquals(3, messages.size());
// All messages should be written with timestamp 1
payloads.clear();
for (RawMessage message : messages) {
Assert.assertEquals(1L, new MessageId(message.getId()).getPublishTimestamp());
payloads.add(Bytes.toString(message.getPayload()));
}
Assert.assertEquals(Arrays.asList("a", "b", "c"), payloads);
}
use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.
the class MessageTableTest method testSingleMessage.
@Test
public void testSingleMessage() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("singleMessage");
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
String payload = "data";
long txWritePtr = 123L;
try (MessageTable table = getMessageTable(metadata);
MetadataTable metadataTable = getMetadataTable()) {
metadataTable.createTopic(metadata);
List<MessageTable.Entry> entryList = new ArrayList<>();
entryList.add(new TestMessageEntry(topicId, GENERATION, 0L, 0, txWritePtr, Bytes.toBytes(payload)));
table.store(entryList.iterator());
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), false, 50, null)) {
// Fetch not including the first message, expect empty
Assert.assertFalse(iterator.hasNext());
}
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
// Fetch including the first message, should get the message
Assert.assertTrue(iterator.hasNext());
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0, 50, null)) {
// Fetch by time, should get the entry
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
RollbackDetail rollbackDetail = new TestRollbackDetail(123L, 0, (short) 0, 0L, (short) 0);
table.rollback(metadata, rollbackDetail);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
// Fetching the message non-tx should provide a result even after deletion
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
Transaction tx = new Transaction(200, 200, new long[0], new long[0], -1);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, tx)) {
// Fetching messages transactionally should not return any entry
Assert.assertFalse(iterator.hasNext());
}
}
}
use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.
the class MessagingHttpServiceTest method testBasicPubSub.
@Test
public void testBasicPubSub() throws Exception {
TopicId topicId = new NamespaceId("ns1").topic("testBasicPubSub");
// Publish to a non-existing topic should get not found exception
try {
client.publish(StoreRequestBuilder.of(topicId).addPayload("a").build());
Assert.fail("Expected TopicNotFoundException");
} catch (TopicNotFoundException e) {
// Expected
}
// Consume from a non-existing topic should get not found exception
try {
client.prepareFetch(topicId).fetch();
Assert.fail("Expected TopicNotFoundException");
} catch (TopicNotFoundException e) {
// Expected
}
client.createTopic(new TopicMetadata(topicId));
// Publish a non-transactional message with empty payload should result in failure
try {
client.publish(StoreRequestBuilder.of(topicId).build());
Assert.fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
// Expected
}
// Publish a non-tx message, no RollbackDetail is returned
Assert.assertNull(client.publish(StoreRequestBuilder.of(topicId).addPayload("m0").addPayload("m1").build()));
// Publish a transactional message, a RollbackDetail should be returned
RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topicId).addPayload("m2").setTransaction(1L).build());
Assert.assertNotNull(rollbackDetail);
// Rollback the published message
client.rollback(topicId, rollbackDetail);
// Fetch messages non-transactionally (should be able to read all the messages since rolled back messages
// are still visible until ttl kicks in)
List<RawMessage> messages = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(3, messages.size());
for (int i = 0; i < 3; i++) {
Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
}
// Consume transactionally. It should get only m0 and m1 since m2 has been rolled back
List<RawMessage> txMessages = new ArrayList<>();
Transaction transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(0).setTransaction(transaction).fetch()) {
Iterators.addAll(txMessages, iterator);
}
Assert.assertEquals(2, txMessages.size());
for (int i = 0; i < 2; i++) {
Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
}
// Fetch again from a given message offset exclusively.
// Expects one message to be fetched
byte[] startMessageId = messages.get(1).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
// It should have only one message (m2)
Assert.assertTrue(iterator.hasNext());
RawMessage msg = iterator.next();
Assert.assertEquals("m2", Bytes.toString(msg.getPayload()));
}
// Fetch again from the last message offset exclusively
// Expects no message to be fetched
startMessageId = messages.get(2).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
Assert.assertFalse(iterator.hasNext());
}
// Fetch with start time. It should get both m0 and m1 since they are published in the same request, hence
// having the same publish time
startMessageId = messages.get(1).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(new MessageId(startMessageId).getPublishTimestamp()).setLimit(2).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(2, messages.size());
for (int i = 0; i < 2; i++) {
Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
}
// Publish 2 messages, one transactionally, one without transaction
client.publish(StoreRequestBuilder.of(topicId).addPayload("m3").setTransaction(2L).build());
client.publish(StoreRequestBuilder.of(topicId).addPayload("m4").build());
// Consume without transactional, it should see m2, m3 and m4
startMessageId = messages.get(1).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(3, messages.size());
for (int i = 0; i < 3; i++) {
Assert.assertEquals("m" + (i + 2), Bytes.toString(messages.get(i).getPayload()));
}
// Consume using a transaction that doesn't have tx = 2L visible. It should get no message as it should block on m3
transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
Assert.assertFalse(iterator.hasNext());
}
// Consume using a transaction that has tx = 2L in the invalid list. It should skip m3 and got m4
transaction = new Transaction(3L, 3L, new long[] { 2L }, new long[0], 0L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(1, messages.size());
Assert.assertEquals("m4", Bytes.toString(messages.get(0).getPayload()));
// Consume using a transaction that has tx = 2L committed. It should get m3 and m4
transaction = new Transaction(3L, 3L, new long[0], new long[0], 0L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(2, messages.size());
for (int i = 0; i < 2; i++) {
Assert.assertEquals("m" + (i + 3), Bytes.toString(messages.get(i).getPayload()));
}
client.deleteTopic(topicId);
}
use of io.cdap.cdap.messaging.data.MessageId in project cdap by caskdata.
the class DataCleanupTest method testPayloadTTLCleanup.
@Test
public void testPayloadTTLCleanup() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("t2");
TopicMetadata topic = new TopicMetadata(topicId, "ttl", "3", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
try (MetadataTable metadataTable = getMetadataTable();
PayloadTable payloadTable = getPayloadTable(topic);
MessageTable messageTable = getMessageTable(topic)) {
Assert.assertNotNull(messageTable);
metadataTable.createTopic(topic);
List<PayloadTable.Entry> entries = new ArrayList<>();
entries.add(new TestPayloadEntry(topicId, GENERATION, "payloaddata", 101, (short) 0));
payloadTable.store(entries.iterator());
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
Assert.assertTrue(iterator.hasNext());
PayloadTable.Entry entry = iterator.next();
Assert.assertFalse(iterator.hasNext());
Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
Assert.assertEquals(101L, entry.getTransactionWritePointer());
iterator.close();
forceFlushAndCompact(Table.PAYLOAD);
// Entry should still be there since ttl has not expired
iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
Assert.assertTrue(iterator.hasNext());
entry = iterator.next();
Assert.assertFalse(iterator.hasNext());
Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
Assert.assertEquals(101L, entry.getTransactionWritePointer());
iterator.close();
TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
forceFlushAndCompact(Table.PAYLOAD);
iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
Assert.assertFalse(iterator.hasNext());
iterator.close();
metadataTable.deleteTopic(topicId);
}
}
Aggregations