use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.
the class PayloadTableTest method testConcurrentWrites.
@Test
public void testConcurrentWrites() throws Exception {
// Create two threads, each of them writes to a different topic with two events in one store call.
// The iterators in the two threads would alternate to produce payload. This is for testing CDAP-12013
ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
final CyclicBarrier barrier = new CyclicBarrier(2);
final CountDownLatch storeCompletion = new CountDownLatch(2);
for (int i = 0; i < 2; i++) {
final TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
try (MetadataTable metadataTable = getMetadataTable()) {
metadataTable.createTopic(metadata);
}
final int threadId = i;
executor.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
try (PayloadTable payloadTable = getPayloadTable()) {
payloadTable.store(new AbstractIterator<PayloadTable.Entry>() {
short messageCount = 0;
@Override
protected PayloadTable.Entry computeNext() {
if (messageCount >= 2) {
return endOfData();
}
try {
barrier.await();
} catch (Exception e) {
throw Throwables.propagate(e);
}
return new TestPayloadEntry(topicId, GENERATION, threadId, 0, messageCount, Bytes.toBytes("message " + threadId + " " + messageCount++));
}
});
storeCompletion.countDown();
} catch (Exception e) {
LOG.error("Failed to store to MessageTable", e);
}
return null;
}
});
}
executor.shutdown();
Assert.assertTrue(storeCompletion.await(5, TimeUnit.SECONDS));
// Read from each topic. Each topic should have two messages
for (int i = 0; i < 2; i++) {
TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0, (short) 0, rawId, 0);
MessageId messageId = new MessageId(rawId);
try (PayloadTable payloadTable = getPayloadTable();
CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(metadata, i, messageId, true, 10)) {
List<PayloadTable.Entry> entries = Lists.newArrayList(iterator);
Assert.assertEquals(2, entries.size());
int count = 0;
for (PayloadTable.Entry entry : entries) {
Assert.assertEquals("message " + i + " " + count++, Bytes.toString(entry.getPayload()));
}
}
}
}
use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.
the class LevelDBMetadataTable method createTopic.
@Override
public void createTopic(TopicMetadata topicMetadata) throws TopicAlreadyExistsException, IOException {
try {
TopicId topicId = topicMetadata.getTopicId();
byte[] key = MessagingUtils.toMetadataRowKey(topicId);
TreeMap<String, String> properties = new TreeMap<>(topicMetadata.getProperties());
properties.put(TopicMetadata.GENERATION_KEY, MessagingUtils.Constants.DEFAULT_GENERATION);
synchronized (this) {
byte[] tableValue = levelDB.get(key);
if (tableValue != null) {
Map<String, String> oldProperties = GSON.fromJson(Bytes.toString(tableValue), MAP_TYPE);
TopicMetadata metadata = new TopicMetadata(topicId, oldProperties);
if (metadata.exists()) {
throw new TopicAlreadyExistsException(topicId.getNamespace(), topicId.getTopic());
}
int newGenerationId = (metadata.getGeneration() * -1) + 1;
properties.put(TopicMetadata.GENERATION_KEY, Integer.toString(newGenerationId));
}
byte[] value = Bytes.toBytes(GSON.toJson(properties, MAP_TYPE));
levelDB.put(key, value, WRITE_OPTIONS);
}
} catch (DBException e) {
throw new IOException(e);
}
}
use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.
the class LevelDBMetadataTable method updateTopic.
@Override
public void updateTopic(TopicMetadata topicMetadata) throws TopicNotFoundException, IOException {
try {
TopicId topicId = topicMetadata.getTopicId();
byte[] key = MessagingUtils.toMetadataRowKey(topicId);
synchronized (this) {
byte[] tableValue = levelDB.get(key);
if (tableValue == null) {
throw new TopicNotFoundException(topicId.getNamespace(), topicId.getTopic());
}
Map<String, String> oldProperties = GSON.fromJson(Bytes.toString(tableValue), MAP_TYPE);
TopicMetadata oldMetadata = new TopicMetadata(topicId, oldProperties);
if (!oldMetadata.exists()) {
throw new TopicNotFoundException(topicId.getNamespace(), topicId.getTopic());
}
TreeMap<String, String> newProperties = new TreeMap<>(topicMetadata.getProperties());
newProperties.put(TopicMetadata.GENERATION_KEY, Integer.toString(oldMetadata.getGeneration()));
levelDB.put(key, Bytes.toBytes(GSON.toJson(newProperties, MAP_TYPE)), WRITE_OPTIONS);
}
} catch (DBException e) {
throw new IOException(e);
}
}
use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.
the class MessagingHttpServiceTest method testBasicPubSub.
@Test
public void testBasicPubSub() throws Exception {
TopicId topicId = new NamespaceId("ns1").topic("testBasicPubSub");
// Publish to a non-existing topic should get not found exception
try {
client.publish(StoreRequestBuilder.of(topicId).addPayloads("a").build());
Assert.fail("Expected TopicNotFoundException");
} catch (TopicNotFoundException e) {
// Expected
}
// Consume from a non-existing topic should get not found exception
try {
client.prepareFetch(topicId).fetch();
Assert.fail("Expected TopicNotFoundException");
} catch (TopicNotFoundException e) {
// Expected
}
client.createTopic(new TopicMetadata(topicId));
// Publish a non-transactional message with empty payload should result in failure
try {
client.publish(StoreRequestBuilder.of(topicId).build());
Assert.fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
// Expected
}
// Publish a non-tx message, no RollbackDetail is returned
Assert.assertNull(client.publish(StoreRequestBuilder.of(topicId).addPayloads("m0", "m1").build()));
// Publish a transactional message, a RollbackDetail should be returned
RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topicId).addPayloads("m2").setTransaction(1L).build());
Assert.assertNotNull(rollbackDetail);
// Rollback the published message
client.rollback(topicId, rollbackDetail);
// Fetch messages non-transactionally (should be able to read all the messages since rolled back messages
// are still visible until ttl kicks in)
List<RawMessage> messages = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(3, messages.size());
for (int i = 0; i < 3; i++) {
Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
}
// Consume transactionally. It should get only m0 and m1 since m2 has been rolled back
List<RawMessage> txMessages = new ArrayList<>();
Transaction transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(0).setTransaction(transaction).fetch()) {
Iterators.addAll(txMessages, iterator);
}
Assert.assertEquals(2, txMessages.size());
for (int i = 0; i < 2; i++) {
Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
}
// Fetch again from a given message offset exclusively.
// Expects one message to be fetched
byte[] startMessageId = messages.get(1).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
// It should have only one message (m2)
Assert.assertTrue(iterator.hasNext());
RawMessage msg = iterator.next();
Assert.assertEquals("m2", Bytes.toString(msg.getPayload()));
}
// Fetch again from the last message offset exclusively
// Expects no message to be fetched
startMessageId = messages.get(2).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
Assert.assertFalse(iterator.hasNext());
}
// Fetch with start time. It should get both m0 and m1 since they are published in the same request, hence
// having the same publish time
startMessageId = messages.get(1).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(new MessageId(startMessageId).getPublishTimestamp()).setLimit(2).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(2, messages.size());
for (int i = 0; i < 2; i++) {
Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
}
// Publish 2 messages, one transactionally, one without transaction
client.publish(StoreRequestBuilder.of(topicId).addPayloads("m3").setTransaction(2L).build());
client.publish(StoreRequestBuilder.of(topicId).addPayloads("m4").build());
// Consume without transactional, it should see m2, m3 and m4
startMessageId = messages.get(1).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(3, messages.size());
for (int i = 0; i < 3; i++) {
Assert.assertEquals("m" + (i + 2), Bytes.toString(messages.get(i).getPayload()));
}
// Consume using a transaction that doesn't have tx = 2L visible. It should get no message as it should block on m3
transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
Assert.assertFalse(iterator.hasNext());
}
// Consume using a transaction that has tx = 2L in the invalid list. It should skip m3 and got m4
transaction = new Transaction(3L, 3L, new long[] { 2L }, new long[0], 0L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(1, messages.size());
Assert.assertEquals("m4", Bytes.toString(messages.get(0).getPayload()));
// Consume using a transaction that has tx = 2L committed. It should get m3 and m4
transaction = new Transaction(3L, 3L, new long[0], new long[0], 0L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(2, messages.size());
for (int i = 0; i < 2; i++) {
Assert.assertEquals("m" + (i + 3), Bytes.toString(messages.get(i).getPayload()));
}
client.deleteTopic(topicId);
}
use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.
the class MessagingHttpServiceTest method testDeletes.
@Test
public void testDeletes() throws Exception {
TopicId topicId = new NamespaceId("ns1").topic("del");
TopicMetadata metadata = new TopicMetadata(topicId, "ttl", "100");
for (int j = 0; j < 10; j++) {
client.createTopic(metadata);
String m1 = String.format("m%d", j);
String m2 = String.format("m%d", j + 1);
Assert.assertNull(client.publish(StoreRequestBuilder.of(topicId).addPayloads(m1, m2).build()));
// Fetch messages non-transactionally
List<RawMessage> messages = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(2, messages.size());
Set<String> receivedMessages = new HashSet<>();
for (RawMessage message : messages) {
receivedMessages.add(Bytes.toString(message.getPayload()));
}
Assert.assertTrue(receivedMessages.contains(m1));
Assert.assertTrue(receivedMessages.contains(m2));
client.deleteTopic(topicId);
}
}
Aggregations