Examples with MessageId co.cask.cdap.messaging.data.MessageId used on opensource projects

Example 1 with MessageId

use of co.cask.cdap.messaging.data.MessageId in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testMultiMaxSequence.

@Test
public void testMultiMaxSequence() throws IOException, InterruptedException {
    TopicId topicId = new NamespaceId("ns1").topic("t1");
    final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
    // This test the case when multiple StoreRequests combined exceeding the 65536 payload.
    // See testMaxSequence() for more details when it matters
    // Generate 3 StoreRequests, each with 43690 messages
    int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT / 3 * 2;
    int requestCount = 3;
    List<StoreRequest> requests = new ArrayList<>();
    for (int i = 0; i < requestCount; i++) {
        List<String> payloads = new ArrayList<>(msgCount);
        for (int j = 0; j < msgCount; j++) {
            payloads.add(Integer.toString(j));
        }
        requests.add(new TestStoreRequest(topicId, payloads));
    }
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    // We use a custom metrics collector here to make all the persist calls reached the same latch,
    // since we know that the ConcurrentMessageWriter will emit a metrics "persist.requested" after enqueued but
    // before flushing.
    // This will make all requests batched together
    final CountDownLatch latch = new CountDownLatch(requestCount);
    final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter, new MetricsCollector() {

        @Override
        public void increment(String metricName, long value) {
            if ("persist.requested".equals(metricName)) {
                latch.countDown();
                Uninterruptibles.awaitUninterruptibly(latch);
            }
        }

        @Override
        public void gauge(String metricName, long value) {
            LOG.info("MetricsContext.gauge: {} = {}", metricName, value);
        }
    });
    ExecutorService executor = Executors.newFixedThreadPool(3);
    for (final StoreRequest request : requests) {
        executor.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    writer.persist(request, metadata);
                } catch (IOException e) {
                    LOG.error("Failed to persist", e);
                }
            }
        });
    }
    executor.shutdown();
    Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
    // Validates all messages are being written
    List<RawMessage> messages = testWriter.getMessages().get(topicId);
    Assert.assertEquals(requestCount * msgCount, messages.size());
    // We expect the payload is in repeated sequence of [0..msgCount-1]
    int expectedPayload = 0;
    // The sequenceId should be (i % SEQUENCE_ID_LIMIT)
    for (int i = 0; i < messages.size(); i++) {
        RawMessage message = messages.get(i);
        MessageId messageId = new MessageId(message.getId());
        Assert.assertEquals(i / StoreRequestWriter.SEQUENCE_ID_LIMIT, messageId.getPublishTimestamp());
        Assert.assertEquals((short) (i % StoreRequestWriter.SEQUENCE_ID_LIMIT), messageId.getSequenceId());
        Assert.assertEquals(expectedPayload, Integer.parseInt(Bytes.toString(message.getPayload())));
        expectedPayload = (expectedPayload + 1) % msgCount;
    }
}

Example 2 with MessageId

use of co.cask.cdap.messaging.data.MessageId in project cdap by caskdata.

the class MessageTableTest method testSingleMessage.

@Test
public void testSingleMessage() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("singleMessage");
    TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
    String payload = "data";
    long txWritePtr = 123L;
    try (MessageTable table = getMessageTable();
        MetadataTable metadataTable = getMetadataTable()) {
        metadataTable.createTopic(metadata);
        List<MessageTable.Entry> entryList = new ArrayList<>();
        entryList.add(new TestMessageEntry(topicId, GENERATION, 0L, 0, txWritePtr, Bytes.toBytes(payload)));
        table.store(entryList.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), false, 50, null)) {
            // Fetch not including the first message, expect empty
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
            // Fetch including the first message, should get the message
            Assert.assertTrue(iterator.hasNext());
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0, 50, null)) {
            // Fetch by time, should get the entry
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        RollbackDetail rollbackDetail = new TestRollbackDetail(123L, 0, (short) 0, 0L, (short) 0);
        table.rollback(metadata, rollbackDetail);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
            // Fetching the message non-tx should provide a result even after deletion
            MessageTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertFalse(iterator.hasNext());
        }
        Transaction tx = new Transaction(200, 200, new long[0], new long[0], -1);
        try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, tx)) {
            // Fetching messages transactionally should not return any entry
            Assert.assertFalse(iterator.hasNext());
        }
    }
}

Example 3 with MessageId

use of co.cask.cdap.messaging.data.MessageId in project cdap by caskdata.

the class PayloadTableTest method testSingleMessage.

@Test
public void testSingleMessage() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("singlePayload");
    TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
    String payload = "data";
    long txWritePtr = 123L;
    try (MetadataTable metadataTable = getMetadataTable();
        PayloadTable table = getPayloadTable()) {
        metadataTable.createTopic(metadata);
        List<PayloadTable.Entry> entryList = new ArrayList<>();
        entryList.add(new TestPayloadEntry(topicId, GENERATION, txWritePtr, 1L, (short) 1, Bytes.toBytes(payload)));
        table.store(entryList.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(metadata, txWritePtr, new MessageId(messageId), false, Integer.MAX_VALUE)) {
            // Fetch not including the first message, expect empty
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(metadata, txWritePtr, new MessageId(messageId), true, Integer.MAX_VALUE)) {
            // Fetch including the first message
            Assert.assertTrue(iterator.hasNext());
            PayloadTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertEquals(txWritePtr, entry.getTransactionWritePointer());
            Assert.assertFalse(iterator.hasNext());
        }
    }
}

Example 4 with MessageId

use of co.cask.cdap.messaging.data.MessageId in project cdap by caskdata.

the class PayloadTableTest method testConsumption.

@Test
public void testConsumption() throws Exception {
    try (MetadataTable metadataTable = getMetadataTable();
        PayloadTable table = getPayloadTable()) {
        metadataTable.createTopic(M1);
        metadataTable.createTopic(M2);
        List<PayloadTable.Entry> entryList = new ArrayList<>();
        populateList(entryList);
        table.store(entryList.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        // Fetch data with 100 write pointer
        try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(M1, 100, new MessageId(messageId), true, Integer.MAX_VALUE)) {
            checkData(iterator, 123, ImmutableSet.of(100L), 50);
        }
        // Fetch only 10 items with 101 write pointer
        try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(M1, 101, new MessageId(messageId), true, 1)) {
            checkData(iterator, 123, ImmutableSet.of(101L), 1);
        }
        // Fetch items with 102 write pointer
        try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(M1, 102, new MessageId(messageId), true, Integer.MAX_VALUE)) {
            checkData(iterator, 123, ImmutableSet.of(102L), 50);
        }
        // Fetch from t2 with 101 write pointer
        try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(M2, 101, new MessageId(messageId), true, Integer.MAX_VALUE)) {
            checkData(iterator, 123, ImmutableSet.of(101L), 50);
        }
    }
}

Example 5 with MessageId

use of co.cask.cdap.messaging.data.MessageId in project cdap by caskdata.

the class PayloadTableTest method testConcurrentWrites.

@Test
public void testConcurrentWrites() throws Exception {
    // Create two threads, each of them writes to a different topic with two events in one store call.
    // The iterators in the two threads would alternate to produce payload. This is for testing CDAP-12013
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch storeCompletion = new CountDownLatch(2);
    for (int i = 0; i < 2; i++) {
        final TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
        TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
        try (MetadataTable metadataTable = getMetadataTable()) {
            metadataTable.createTopic(metadata);
        }
        final int threadId = i;
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                try (PayloadTable payloadTable = getPayloadTable()) {
                    payloadTable.store(new AbstractIterator<PayloadTable.Entry>() {

                        short messageCount = 0;

                        @Override
                        protected PayloadTable.Entry computeNext() {
                            if (messageCount >= 2) {
                                return endOfData();
                            }
                            try {
                                barrier.await();
                            } catch (Exception e) {
                                throw Throwables.propagate(e);
                            }
                            return new TestPayloadEntry(topicId, GENERATION, threadId, 0, messageCount, Bytes.toBytes("message " + threadId + " " + messageCount++));
                        }
                    });
                    storeCompletion.countDown();
                } catch (Exception e) {
                    LOG.error("Failed to store to MessageTable", e);
                }
                return null;
            }
        });
    }
    executor.shutdown();
    Assert.assertTrue(storeCompletion.await(5, TimeUnit.SECONDS));
    // Read from each topic. Each topic should have two messages
    for (int i = 0; i < 2; i++) {
        TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
        TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
        byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0, (short) 0, rawId, 0);
        MessageId messageId = new MessageId(rawId);
        try (PayloadTable payloadTable = getPayloadTable();
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(metadata, i, messageId, true, 10)) {
            List<PayloadTable.Entry> entries = Lists.newArrayList(iterator);
            Assert.assertEquals(2, entries.size());
            int count = 0;
            for (PayloadTable.Entry entry : entries) {
                Assert.assertEquals("message " + i + " " + count++, Bytes.toString(entry.getPayload()));
            }
        }
    }
}