use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.
the class LeaderElectionMessagingServiceTest method testTransition.
@Test
public void testTransition() throws Throwable {
final TopicId topicId = NamespaceId.SYSTEM.topic("topic");
Injector injector1 = createInjector(0);
Injector injector2 = createInjector(1);
// Start a messaging service, which would becomes leader
ZKClientService zkClient1 = injector1.getInstance(ZKClientService.class);
zkClient1.startAndWait();
final MessagingService firstService = injector1.getInstance(MessagingService.class);
if (firstService instanceof Service) {
((Service) firstService).startAndWait();
}
// Publish a message with the leader
firstService.publish(StoreRequestBuilder.of(topicId).addPayloads("Testing1").build());
// Start another messaging service, this one would be follower
ZKClientService zkClient2 = injector2.getInstance(ZKClientService.class);
zkClient2.startAndWait();
final MessagingService secondService = injector2.getInstance(MessagingService.class);
if (secondService instanceof Service) {
((Service) secondService).startAndWait();
}
// Try to call the follower, should get service unavailable.
try {
secondService.listTopics(NamespaceId.SYSTEM);
Assert.fail("Expected service unavailable");
} catch (ServiceUnavailableException e) {
// Expected
}
// Make the ZK session timeout for the leader service. The second one should pickup.
KillZKSession.kill(zkClient1.getZooKeeperSupplier().get(), zkClient1.getConnectString(), 10000);
// Publish one more message and then fetch from the current leader
List<String> messages = Retries.callWithRetries(new Retries.Callable<List<String>, Throwable>() {
@Override
public List<String> call() throws Throwable {
secondService.publish(StoreRequestBuilder.of(topicId).addPayloads("Testing2").build());
List<String> messages = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = secondService.prepareFetch(topicId).fetch()) {
while (iterator.hasNext()) {
messages.add(new String(iterator.next().getPayload(), "UTF-8"));
}
}
return messages;
}
}, RetryStrategies.timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS)));
Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages);
// Shutdown the current leader. The session timeout one should becomes leader again.
if (secondService instanceof Service) {
((Service) secondService).stopAndWait();
}
// Try to fetch message from the current leader again.
// Should see two messages (because the cache is cleared and fetch is from the backing store).
messages = Retries.callWithRetries(new Retries.Callable<List<String>, Throwable>() {
@Override
public List<String> call() throws Throwable {
List<String> messages = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = firstService.prepareFetch(topicId).fetch()) {
while (iterator.hasNext()) {
messages.add(new String(iterator.next().getPayload(), "UTF-8"));
}
}
return messages;
}
}, RetryStrategies.timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS)));
Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages);
zkClient1.stopAndWait();
zkClient2.stopAndWait();
}
use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.
the class ConcurrentMessageWriterTest method testMultiMaxSequence.
@Test
public void testMultiMaxSequence() throws IOException, InterruptedException {
TopicId topicId = new NamespaceId("ns1").topic("t1");
final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
// This test the case when multiple StoreRequests combined exceeding the 65536 payload.
// See testMaxSequence() for more details when it matters
// Generate 3 StoreRequests, each with 43690 messages
int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT / 3 * 2;
int requestCount = 3;
List<StoreRequest> requests = new ArrayList<>();
for (int i = 0; i < requestCount; i++) {
List<String> payloads = new ArrayList<>(msgCount);
for (int j = 0; j < msgCount; j++) {
payloads.add(Integer.toString(j));
}
requests.add(new TestStoreRequest(topicId, payloads));
}
TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
// We use a custom metrics collector here to make all the persist calls reached the same latch,
// since we know that the ConcurrentMessageWriter will emit a metrics "persist.requested" after enqueued but
// before flushing.
// This will make all requests batched together
final CountDownLatch latch = new CountDownLatch(requestCount);
final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter, new MetricsCollector() {
@Override
public void increment(String metricName, long value) {
if ("persist.requested".equals(metricName)) {
latch.countDown();
Uninterruptibles.awaitUninterruptibly(latch);
}
}
@Override
public void gauge(String metricName, long value) {
LOG.info("MetricsContext.gauge: {} = {}", metricName, value);
}
});
ExecutorService executor = Executors.newFixedThreadPool(3);
for (final StoreRequest request : requests) {
executor.submit(new Runnable() {
@Override
public void run() {
try {
writer.persist(request, metadata);
} catch (IOException e) {
LOG.error("Failed to persist", e);
}
}
});
}
executor.shutdown();
Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
// Validates all messages are being written
List<RawMessage> messages = testWriter.getMessages().get(topicId);
Assert.assertEquals(requestCount * msgCount, messages.size());
// We expect the payload is in repeated sequence of [0..msgCount-1]
int expectedPayload = 0;
// The sequenceId should be (i % SEQUENCE_ID_LIMIT)
for (int i = 0; i < messages.size(); i++) {
RawMessage message = messages.get(i);
MessageId messageId = new MessageId(message.getId());
Assert.assertEquals(i / StoreRequestWriter.SEQUENCE_ID_LIMIT, messageId.getPublishTimestamp());
Assert.assertEquals((short) (i % StoreRequestWriter.SEQUENCE_ID_LIMIT), messageId.getSequenceId());
Assert.assertEquals(expectedPayload, Integer.parseInt(Bytes.toString(message.getPayload())));
expectedPayload = (expectedPayload + 1) % msgCount;
}
}
use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.
the class MessageTableTest method testConcurrentWrites.
@Test
public void testConcurrentWrites() throws Exception {
// Create two threads, each of them writes to a different topic with two events in one store call.
// The iterators in the two threads would alternate to produce payload. This is for testing CDAP-12013
ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
final CyclicBarrier barrier = new CyclicBarrier(2);
final CountDownLatch storeCompletion = new CountDownLatch(2);
for (int i = 0; i < 2; i++) {
final TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
try (MetadataTable metadataTable = getMetadataTable()) {
metadataTable.createTopic(metadata);
}
final int threadId = i;
executor.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
try (MessageTable messageTable = getMessageTable()) {
messageTable.store(new AbstractIterator<MessageTable.Entry>() {
int messageCount = 0;
@Override
protected MessageTable.Entry computeNext() {
if (messageCount >= 2) {
return endOfData();
}
try {
barrier.await();
} catch (Exception e) {
throw Throwables.propagate(e);
}
return new TestMessageEntry(topicId, GENERATION, System.currentTimeMillis(), messageCount, null, Bytes.toBytes("message " + threadId + " " + messageCount++));
}
});
storeCompletion.countDown();
} catch (Exception e) {
LOG.error("Failed to store to MessageTable", e);
}
return null;
}
});
}
executor.shutdown();
Assert.assertTrue(storeCompletion.await(30, TimeUnit.SECONDS));
// Read from each topic. Each topic should have two messages
for (int i = 0; i < 2; i++) {
TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
try (MessageTable messageTable = getMessageTable();
CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(metadata, 0, 10, null)) {
List<MessageTable.Entry> entries = Lists.newArrayList(iterator);
Assert.assertEquals(2, entries.size());
int count = 0;
for (MessageTable.Entry entry : entries) {
Assert.assertEquals("message " + i + " " + count++, Bytes.toString(entry.getPayload()));
}
}
}
}
use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.
the class MessageTableTest method testSingleMessage.
@Test
public void testSingleMessage() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("singleMessage");
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
String payload = "data";
long txWritePtr = 123L;
try (MessageTable table = getMessageTable();
MetadataTable metadataTable = getMetadataTable()) {
metadataTable.createTopic(metadata);
List<MessageTable.Entry> entryList = new ArrayList<>();
entryList.add(new TestMessageEntry(topicId, GENERATION, 0L, 0, txWritePtr, Bytes.toBytes(payload)));
table.store(entryList.iterator());
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), false, 50, null)) {
// Fetch not including the first message, expect empty
Assert.assertFalse(iterator.hasNext());
}
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
// Fetch including the first message, should get the message
Assert.assertTrue(iterator.hasNext());
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0, 50, null)) {
// Fetch by time, should get the entry
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
RollbackDetail rollbackDetail = new TestRollbackDetail(123L, 0, (short) 0, 0L, (short) 0);
table.rollback(metadata, rollbackDetail);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
// Fetching the message non-tx should provide a result even after deletion
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
Transaction tx = new Transaction(200, 200, new long[0], new long[0], -1);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, tx)) {
// Fetching messages transactionally should not return any entry
Assert.assertFalse(iterator.hasNext());
}
}
}
use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.
the class PayloadTableTest method testSingleMessage.
@Test
public void testSingleMessage() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("singlePayload");
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
String payload = "data";
long txWritePtr = 123L;
try (MetadataTable metadataTable = getMetadataTable();
PayloadTable table = getPayloadTable()) {
metadataTable.createTopic(metadata);
List<PayloadTable.Entry> entryList = new ArrayList<>();
entryList.add(new TestPayloadEntry(topicId, GENERATION, txWritePtr, 1L, (short) 1, Bytes.toBytes(payload)));
table.store(entryList.iterator());
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(metadata, txWritePtr, new MessageId(messageId), false, Integer.MAX_VALUE)) {
// Fetch not including the first message, expect empty
Assert.assertFalse(iterator.hasNext());
}
try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(metadata, txWritePtr, new MessageId(messageId), true, Integer.MAX_VALUE)) {
// Fetch including the first message
Assert.assertTrue(iterator.hasNext());
PayloadTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertEquals(txWritePtr, entry.getTransactionWritePointer());
Assert.assertFalse(iterator.hasNext());
}
}
}
Aggregations