use of io.cdap.cdap.messaging.data.RawMessage in project cdap by caskdata.
the class AuditPublishTest method fetchAuditMessages.
private List<AuditMessage> fetchAuditMessages() throws TopicNotFoundException, IOException {
List<AuditMessage> result = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = messagingService.prepareFetch(auditTopic).fetch()) {
while (iterator.hasNext()) {
RawMessage message = iterator.next();
result.add(GSON.fromJson(new String(message.getPayload(), StandardCharsets.UTF_8), AuditMessage.class));
}
}
return result;
}
use of io.cdap.cdap.messaging.data.RawMessage in project cdap by caskdata.
the class ConcurrentMessageWriterTest method testMultiMaxSequence.
@Test
public void testMultiMaxSequence() throws IOException, InterruptedException {
TopicId topicId = new NamespaceId("ns1").topic("t1");
final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
// This test the case when multiple StoreRequests combined exceeding the 65536 payload.
// See testMaxSequence() for more details when it matters
// Generate 3 StoreRequests, each with 43690 messages
int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT / 3 * 2;
int requestCount = 3;
List<StoreRequest> requests = new ArrayList<>();
for (int i = 0; i < requestCount; i++) {
List<String> payloads = new ArrayList<>(msgCount);
for (int j = 0; j < msgCount; j++) {
payloads.add(Integer.toString(j));
}
requests.add(new TestStoreRequest(topicId, payloads));
}
TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
// We use a custom metrics collector here to make all the persist calls reached the same latch,
// since we know that the ConcurrentMessageWriter will emit a metrics "persist.requested" after enqueued but
// before flushing.
// This will make all requests batched together
final CountDownLatch latch = new CountDownLatch(requestCount);
final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter, new MetricsCollector() {
@Override
public void increment(String metricName, long value) {
if ("persist.requested".equals(metricName)) {
latch.countDown();
Uninterruptibles.awaitUninterruptibly(latch);
}
}
@Override
public void gauge(String metricName, long value) {
LOG.info("MetricsContext.gauge: {} = {}", metricName, value);
}
});
ExecutorService executor = Executors.newFixedThreadPool(3);
for (final StoreRequest request : requests) {
executor.submit(new Runnable() {
@Override
public void run() {
try {
writer.persist(request, metadata);
} catch (IOException e) {
LOG.error("Failed to persist", e);
}
}
});
}
executor.shutdown();
Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
// Validates all messages are being written
List<RawMessage> messages = testWriter.getMessages().get(topicId);
Assert.assertEquals(requestCount * msgCount, messages.size());
// We expect the payload is in repeated sequence of [0..msgCount-1]
int expectedPayload = 0;
// The sequenceId should be (i % SEQUENCE_ID_LIMIT)
for (int i = 0; i < messages.size(); i++) {
RawMessage message = messages.get(i);
MessageId messageId = new MessageId(message.getId());
Assert.assertEquals(i / StoreRequestWriter.SEQUENCE_ID_LIMIT, messageId.getPublishTimestamp());
Assert.assertEquals((short) (i % StoreRequestWriter.SEQUENCE_ID_LIMIT), messageId.getSequenceId());
Assert.assertEquals(expectedPayload, Integer.parseInt(Bytes.toString(message.getPayload())));
expectedPayload = (expectedPayload + 1) % msgCount;
}
}
use of io.cdap.cdap.messaging.data.RawMessage in project cdap by caskdata.
the class ConcurrentMessageWriterTest method testBasic.
@Test
public void testBasic() throws IOException {
TopicId topicId1 = new NamespaceId("ns1").topic("t1");
TopicId topicId2 = new NamespaceId("ns2").topic("t2");
TopicMetadata metadata1 = new TopicMetadata(topicId1, new HashMap<String, String>(), 1);
TopicMetadata metadata2 = new TopicMetadata(topicId2, new HashMap<String, String>(), 1);
TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter);
writer.persist(new TestStoreRequest(topicId1, Arrays.asList("1", "2", "3")), metadata1);
// There should be 3 messages being written
List<RawMessage> messages = testWriter.getMessages().get(topicId1);
Assert.assertEquals(3, messages.size());
// All messages should be written with timestamp 0
List<String> payloads = new ArrayList<>();
for (RawMessage message : messages) {
Assert.assertEquals(0L, new MessageId(message.getId()).getPublishTimestamp());
payloads.add(Bytes.toString(message.getPayload()));
}
Assert.assertEquals(Arrays.asList("1", "2", "3"), payloads);
// Write to another topic
writer.persist(new TestStoreRequest(topicId2, Arrays.asList("a", "b", "c")), metadata2);
// There should be 3 messages being written to topic2
messages = testWriter.getMessages().get(topicId2);
Assert.assertEquals(3, messages.size());
// All messages should be written with timestamp 1
payloads.clear();
for (RawMessage message : messages) {
Assert.assertEquals(1L, new MessageId(message.getId()).getPublishTimestamp());
payloads.add(Bytes.toString(message.getPayload()));
}
Assert.assertEquals(Arrays.asList("a", "b", "c"), payloads);
}
use of io.cdap.cdap.messaging.data.RawMessage in project cdap by caskdata.
the class MessagingHttpServiceTest method testLargePublish.
@Test
public void testLargePublish() throws IOException, TopicAlreadyExistsException, TopicNotFoundException, UnauthorizedException {
// A 5MB message, which is larger than the 1MB buffer.
String message = Strings.repeat("01234", 1024 * 1024);
TopicId topicId = new NamespaceId("ns1").topic("testLargePublish");
client.createTopic(new TopicMetadata(topicId));
StoreRequest request = StoreRequestBuilder.of(topicId).addPayload(message).build();
client.publish(request);
// Read it back
List<RawMessage> messages = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(10).fetch()) {
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(1, messages.size());
Assert.assertEquals(Collections.singletonList(message), messages.stream().map(RawMessage::getPayload).map(Bytes::toString).collect(Collectors.toList()));
}
use of io.cdap.cdap.messaging.data.RawMessage in project cdap by caskdata.
the class MessagingHttpServiceTest method testChunkConsume.
@Test
public void testChunkConsume() throws Exception {
// This test is to verify the message fetching body producer works correctly
TopicId topicId = new NamespaceId("ns1").topic("testChunkConsume");
client.createTopic(new TopicMetadata(topicId));
// Publish 10 messages, each payload is half the size of the chunk size
int payloadSize = cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_CONSUME_CHUNK_SIZE) / 2;
for (int i = 0; i < 10; i++) {
String payload = Strings.repeat(Integer.toString(i), payloadSize);
client.publish(StoreRequestBuilder.of(topicId).addPayload(payload).build());
}
// Fetch messages. All of them should be fetched correctly
List<RawMessage> messages = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(10, messages.size());
for (int i = 0; i < 10; i++) {
RawMessage message = messages.get(i);
Assert.assertEquals(payloadSize, message.getPayload().length);
String payload = Strings.repeat(Integer.toString(i), payloadSize);
Assert.assertEquals(payload, Bytes.toString(message.getPayload()));
}
client.deleteTopic(topicId);
}
Aggregations