Examples with TopicMetadata io.cdap.cdap.messaging.TopicMetadata used on opensource projects

Example 11 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class MessagingHttpServiceTest method testReuseRequest.

@Test
public void testReuseRequest() throws IOException, TopicAlreadyExistsException, TopicNotFoundException, UnauthorizedException {
    // This test a StoreRequest object can be reused.
    // This test is to verify storing transaction messages to the payload table
    TopicId topicId = new NamespaceId("ns1").topic("testReuseRequest");
    client.createTopic(new TopicMetadata(topicId));
    StoreRequest request = StoreRequestBuilder.of(topicId).addPayload("m1").addPayload("m2").build();
    // Publish the request twice
    client.publish(request);
    client.publish(request);
    // Expects four messages
    List<RawMessage> messages = new ArrayList<>();
    try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(10).fetch()) {
        Iterators.addAll(messages, iterator);
    }
    Assert.assertEquals(4, messages.size());
    List<String> expected = Arrays.asList("m1", "m2", "m1", "m2");
    Assert.assertEquals(expected, messages.stream().map(RawMessage::getPayload).map(Bytes::toString).collect(Collectors.toList()));
}

Example 12 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testMultiMaxSequence.

@Test
public void testMultiMaxSequence() throws IOException, InterruptedException {
    TopicId topicId = new NamespaceId("ns1").topic("t1");
    final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
    // This test the case when multiple StoreRequests combined exceeding the 65536 payload.
    // See testMaxSequence() for more details when it matters
    // Generate 3 StoreRequests, each with 43690 messages
    int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT / 3 * 2;
    int requestCount = 3;
    List<StoreRequest> requests = new ArrayList<>();
    for (int i = 0; i < requestCount; i++) {
        List<String> payloads = new ArrayList<>(msgCount);
        for (int j = 0; j < msgCount; j++) {
            payloads.add(Integer.toString(j));
        }
        requests.add(new TestStoreRequest(topicId, payloads));
    }
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    // We use a custom metrics collector here to make all the persist calls reached the same latch,
    // since we know that the ConcurrentMessageWriter will emit a metrics "persist.requested" after enqueued but
    // before flushing.
    // This will make all requests batched together
    final CountDownLatch latch = new CountDownLatch(requestCount);
    final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter, new MetricsCollector() {

        @Override
        public void increment(String metricName, long value) {
            if ("persist.requested".equals(metricName)) {
                latch.countDown();
                Uninterruptibles.awaitUninterruptibly(latch);
            }
        }

        @Override
        public void gauge(String metricName, long value) {
            LOG.info("MetricsContext.gauge: {} = {}", metricName, value);
        }
    });
    ExecutorService executor = Executors.newFixedThreadPool(3);
    for (final StoreRequest request : requests) {
        executor.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    writer.persist(request, metadata);
                } catch (IOException e) {
                    LOG.error("Failed to persist", e);
                }
            }
        });
    }
    executor.shutdown();
    Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
    // Validates all messages are being written
    List<RawMessage> messages = testWriter.getMessages().get(topicId);
    Assert.assertEquals(requestCount * msgCount, messages.size());
    // We expect the payload is in repeated sequence of [0..msgCount-1]
    int expectedPayload = 0;
    // The sequenceId should be (i % SEQUENCE_ID_LIMIT)
    for (int i = 0; i < messages.size(); i++) {
        RawMessage message = messages.get(i);
        MessageId messageId = new MessageId(message.getId());
        Assert.assertEquals(i / StoreRequestWriter.SEQUENCE_ID_LIMIT, messageId.getPublishTimestamp());
        Assert.assertEquals((short) (i % StoreRequestWriter.SEQUENCE_ID_LIMIT), messageId.getSequenceId());
        Assert.assertEquals(expectedPayload, Integer.parseInt(Bytes.toString(message.getPayload())));
        expectedPayload = (expectedPayload + 1) % msgCount;
    }
}

Example 13 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testBasic.

@Test
public void testBasic() throws IOException {
    TopicId topicId1 = new NamespaceId("ns1").topic("t1");
    TopicId topicId2 = new NamespaceId("ns2").topic("t2");
    TopicMetadata metadata1 = new TopicMetadata(topicId1, new HashMap<String, String>(), 1);
    TopicMetadata metadata2 = new TopicMetadata(topicId2, new HashMap<String, String>(), 1);
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter);
    writer.persist(new TestStoreRequest(topicId1, Arrays.asList("1", "2", "3")), metadata1);
    // There should be 3 messages being written
    List<RawMessage> messages = testWriter.getMessages().get(topicId1);
    Assert.assertEquals(3, messages.size());
    // All messages should be written with timestamp 0
    List<String> payloads = new ArrayList<>();
    for (RawMessage message : messages) {
        Assert.assertEquals(0L, new MessageId(message.getId()).getPublishTimestamp());
        payloads.add(Bytes.toString(message.getPayload()));
    }
    Assert.assertEquals(Arrays.asList("1", "2", "3"), payloads);
    // Write to another topic
    writer.persist(new TestStoreRequest(topicId2, Arrays.asList("a", "b", "c")), metadata2);
    // There should be 3 messages being written to topic2
    messages = testWriter.getMessages().get(topicId2);
    Assert.assertEquals(3, messages.size());
    // All messages should be written with timestamp 1
    payloads.clear();
    for (RawMessage message : messages) {
        Assert.assertEquals(1L, new MessageId(message.getId()).getPublishTimestamp());
        payloads.add(Bytes.toString(message.getPayload()));
    }
    Assert.assertEquals(Arrays.asList("a", "b", "c"), payloads);
}

Example 14 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class DataCleanupTest method testOldGenCleanup.

@Test
public void testOldGenCleanup() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("oldGenCleanup");
    TopicMetadata topic = new TopicMetadata(topicId, TopicMetadata.TTL_KEY, "100000", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
    try (MetadataTable metadataTable = getMetadataTable()) {
        int txWritePtr = 100;
        metadataTable.createTopic(topic);
        List<MessageTable.Entry> entries = new ArrayList<>();
        List<PayloadTable.Entry> pentries = new ArrayList<>();
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        entries.add(new TestMessageEntry(topicId, GENERATION, "data", txWritePtr, (short) 0));
        pentries.add(new TestPayloadEntry(topicId, GENERATION, "data", txWritePtr, (short) 0));
        try (MessageTable messageTable = getMessageTable(topic);
            PayloadTable payloadTable = getPayloadTable(topic)) {
            messageTable.store(entries.iterator());
            payloadTable.store(pentries.iterator());
        }
        // Fetch the entries and make sure we are able to read it
        try (MessageTable messageTable = getMessageTable(topic);
            CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            checkMessageEntry(iterator, txWritePtr);
        }
        try (PayloadTable payloadTable = getPayloadTable(topic);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
            checkPayloadEntry(iterator, txWritePtr);
        }
        // Now run full compaction
        forceFlushAndCompact(Table.MESSAGE);
        forceFlushAndCompact(Table.PAYLOAD);
        // Fetch the entries and make sure we are able to read it
        try (MessageTable messageTable = getMessageTable(topic);
            CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            checkMessageEntry(iterator, txWritePtr);
        }
        try (PayloadTable payloadTable = getPayloadTable(topic);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
            checkPayloadEntry(iterator, txWritePtr);
        }
        // delete the topic and recreate it with an incremented generation
        metadataTable.deleteTopic(topicId);
        Map<String, String> newProperties = new HashMap<>(topic.getProperties());
        newProperties.put(TopicMetadata.GENERATION_KEY, Integer.toString(topic.getGeneration() + 1));
        topic = new TopicMetadata(topicId, newProperties);
        metadataTable.createTopic(topic);
        // Sleep so that the metadata cache in coprocessor expires
        TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
        forceFlushAndCompact(Table.MESSAGE);
        forceFlushAndCompact(Table.PAYLOAD);
        topic = metadataTable.getMetadata(topicId);
        try (MessageTable messageTable = getMessageTable(topic);
            CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            Assert.assertFalse(iterator.hasNext());
        }
        try (PayloadTable payloadTable = getPayloadTable(topic);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
            Assert.assertFalse(iterator.hasNext());
        }
    }
}

Example 15 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class DataCleanupTest method testMessageTTLCleanup.

@Test
public void testMessageTTLCleanup() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("t1");
    TopicMetadata topic = new TopicMetadata(topicId, "ttl", "3", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
    try (MetadataTable metadataTable = getMetadataTable();
        MessageTable messageTable = getMessageTable(topic);
        PayloadTable payloadTable = getPayloadTable(topic)) {
        Assert.assertNotNull(payloadTable);
        metadataTable.createTopic(topic);
        List<MessageTable.Entry> entries = new ArrayList<>();
        entries.add(new TestMessageEntry(topicId, GENERATION, "data", 100, (short) 0));
        messageTable.store(entries.iterator());
        // Fetch the entries and make sure we are able to read it
        CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
        Assert.assertTrue(iterator.hasNext());
        MessageTable.Entry entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals(100, entry.getTransactionWritePointer());
        Assert.assertEquals("data", Bytes.toString(entry.getPayload()));
        iterator.close();
        forceFlushAndCompact(Table.MESSAGE);
        // Entry should still be there since the ttl has not expired
        iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
        entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals(100, entry.getTransactionWritePointer());
        Assert.assertEquals("data", Bytes.toString(entry.getPayload()));
        iterator.close();
        TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
        forceFlushAndCompact(Table.MESSAGE);
        iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
        Assert.assertFalse(iterator.hasNext());
        iterator.close();
    }
}