Examples with TopicMetadata io.cdap.cdap.messaging.TopicMetadata used on opensource projects

Example 21 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class PayloadTableTest method testSingleMessage.

@Test
public void testSingleMessage() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("singlePayload");
    TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
    String payload = "data";
    long txWritePtr = 123L;
    try (MetadataTable metadataTable = getMetadataTable();
        PayloadTable table = getPayloadTable(metadata)) {
        metadataTable.createTopic(metadata);
        List<PayloadTable.Entry> entryList = new ArrayList<>();
        entryList.add(new TestPayloadEntry(topicId, GENERATION, txWritePtr, 1L, (short) 1, Bytes.toBytes(payload)));
        table.store(entryList.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(metadata, txWritePtr, new MessageId(messageId), false, Integer.MAX_VALUE)) {
            // Fetch not including the first message, expect empty
            Assert.assertFalse(iterator.hasNext());
        }
        try (CloseableIterator<PayloadTable.Entry> iterator = table.fetch(metadata, txWritePtr, new MessageId(messageId), true, Integer.MAX_VALUE)) {
            // Fetch including the first message
            Assert.assertTrue(iterator.hasNext());
            PayloadTable.Entry entry = iterator.next();
            Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
            Assert.assertEquals(txWritePtr, entry.getTransactionWritePointer());
            Assert.assertFalse(iterator.hasNext());
        }
    }
}

Example 22 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class PayloadTableTest method testConcurrentWrites.

@Test
public void testConcurrentWrites() throws Exception {
    // Create two threads, each of them writes to a different topic with two events in one store call.
    // The iterators in the two threads would alternate to produce payload. This is for testing CDAP-12013
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch storeCompletion = new CountDownLatch(2);
    for (int i = 0; i < 2; i++) {
        final TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
        TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
        try (MetadataTable metadataTable = getMetadataTable()) {
            metadataTable.createTopic(metadata);
        }
        final int threadId = i;
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                try (PayloadTable payloadTable = getPayloadTable(metadata)) {
                    payloadTable.store(new AbstractIterator<PayloadTable.Entry>() {

                        short messageCount;

                        @Override
                        protected PayloadTable.Entry computeNext() {
                            if (messageCount >= 2) {
                                return endOfData();
                            }
                            try {
                                barrier.await();
                            } catch (Exception e) {
                                throw Throwables.propagate(e);
                            }
                            return new TestPayloadEntry(topicId, GENERATION, threadId, 0, messageCount, Bytes.toBytes("message " + threadId + " " + messageCount++));
                        }
                    });
                    storeCompletion.countDown();
                } catch (Exception e) {
                    LOG.error("Failed to store to MessageTable", e);
                }
                return null;
            }
        });
    }
    executor.shutdown();
    Assert.assertTrue(storeCompletion.await(5, TimeUnit.SECONDS));
    // Read from each topic. Each topic should have two messages
    for (int i = 0; i < 2; i++) {
        TopicId topicId = NamespaceId.DEFAULT.topic("testConcurrentWrites" + i);
        TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
        byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0, (short) 0, rawId, 0);
        MessageId messageId = new MessageId(rawId);
        try (PayloadTable payloadTable = getPayloadTable(metadata);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(metadata, i, messageId, true, 10)) {
            List<PayloadTable.Entry> entries = Lists.newArrayList(iterator);
            Assert.assertEquals(2, entries.size());
            int count = 0;
            for (PayloadTable.Entry entry : entries) {
                Assert.assertEquals("message " + i + " " + count++, Bytes.toString(entry.getPayload()));
            }
        }
    }
}

Example 23 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class CachingMessageTableTest method testCachePruning.

@Test
public void testCachePruning() throws Exception {
    long txGracePeriod = 6;
    CConfiguration cConf = CConfiguration.create();
    cConf.setLong(CachingMessageTable.PRUNE_GRACE_PERIOD, txGracePeriod);
    // Insert 10 entries, with different publish time
    TopicMetadata metadata = new TopicMetadata(NamespaceId.DEFAULT.topic("test"), TopicMetadata.GENERATION_KEY, 1, TopicMetadata.TTL_KEY, 86400);
    // Creates a CachingMessageTable with a controlled time provider
    final AtomicLong currentTimeMillis = new AtomicLong(0);
    MessageTable messageTable = new CachingMessageTable(cConf, super.getMessageTable(metadata), cacheProvider, new TimeProvider() {

        @Override
        public long currentTimeMillis() {
            return currentTimeMillis.get();
        }
    });
    for (int i = 0; i < 10; i++) {
        // Key is (topic, generation, publish time, sequence id)
        byte[] key = Bytes.concat(MessagingUtils.toDataKeyPrefix(metadata.getTopicId(), metadata.getGeneration()), Bytes.toBytes((long) i), Bytes.toBytes((short) 0));
        // Store a message with a write pointer
        messageTable.store(Collections.singleton(new ImmutableMessageTableEntry(key, Bytes.toBytes("Payload " + i), Bytes.toBytes((long) i))).iterator());
    }
    // Update the current time to 11
    currentTimeMillis.set(11);
    // Fetch from the table without transaction, should get all entries.
    try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, 0, 100, null)) {
        List<MessageTable.Entry> entries = Lists.newArrayList(iter);
        Assert.assertEquals(10, entries.size());
        // All entries must be from the cache
        for (MessageTable.Entry entry : entries) {
            Assert.assertTrue(entry instanceof CachingMessageTable.CacheMessageTableEntry);
        }
    }
    // Fetch with a transaction, with start time older than tx grace period / 2
    Transaction tx = new Transaction(10, 11, new long[0], new long[0], 11);
    long startTime = currentTimeMillis.get() - txGracePeriod / 2 - 1;
    try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, startTime, 100, tx)) {
        List<MessageTable.Entry> entries = Lists.newArrayList(iter);
        // Should get three entries (7, 8, 9)
        Assert.assertEquals(3, entries.size());
        // The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
        // from cache (current time = 11, grace period = 3)
        Iterator<MessageTable.Entry> iterator = entries.iterator();
        Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
    }
    // Fetch with a transaction, with start messageId publish time older than tx grace period / 2
    byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
    MessageId.putRawId(startTime, (short) 0, 0L, (short) 0, rawId, 0);
    MessageId messageId = new MessageId(rawId);
    try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, messageId, true, 100, tx)) {
        List<MessageTable.Entry> entries = Lists.newArrayList(iter);
        // Should get three entries (7, 8, 9)
        Assert.assertEquals(3, entries.size());
        // The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
        // from cache (current time = 11, grace period = 3)
        Iterator<MessageTable.Entry> iterator = entries.iterator();
        Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
        Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
    }
}

Example 24 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class LevelDBMessageTableTest method testMultiPartitionReadWrite.

@Test
public void testMultiPartitionReadWrite() throws Exception {
    TopicId topicId = new TopicId("default", "multipart");
    int generation = 1;
    TopicMetadata topicMetadata = new TopicMetadata(topicId, Collections.singletonMap(TopicMetadata.GENERATION_KEY, String.valueOf(generation)));
    try (MessageTable table = tableFactory.createMessageTable(topicMetadata)) {
        List<MessageTable.Entry> writes = new ArrayList<>();
        Map<Long, Byte> expected = new HashMap<>();
        for (int i = 0; i < 10 * PARTITION_SECONDS; i++) {
            long publishTime = i * 1000;
            expected.put(publishTime, (byte) i);
            writes.add(new TestMessageEntry(topicId, generation, publishTime, 0, null, new byte[] { (byte) i }));
        }
        table.store(writes.iterator());
        Map<Long, Byte> actual = new HashMap<>();
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        try (CloseableIterator<MessageTable.Entry> iter = table.fetch(topicMetadata, new MessageId(messageId), true, 100, null)) {
            while (iter.hasNext()) {
                MessageTable.Entry entry = iter.next();
                actual.put(entry.getPublishTimestamp(), entry.getPayload()[0]);
            }
        }
        Assert.assertEquals(expected, actual);
    }
}

Example 25 with TopicMetadata

use of io.cdap.cdap.messaging.TopicMetadata in project cdap by caskdata.

the class HBaseMetadataTable method scanTopics.

/**
 * Scans the HBase table to get a list of {@link TopicId}.
 */
private List<TopicId> scanTopics(ScanBuilder scanBuilder) throws IOException {
    Scan scan = scanBuilder.setFilter(new FirstKeyOnlyFilter()).setCaching(scanCacheRows).build();
    try {
        List<TopicId> topicIds = new ArrayList<>();
        try (ResultScanner resultScanner = table.getScanner(scan)) {
            for (Result result : resultScanner) {
                TopicId topicId = MessagingUtils.toTopicId(result.getRow());
                byte[] value = result.getValue(columnFamily, COL);
                Map<String, String> properties = GSON.fromJson(Bytes.toString(value), MAP_TYPE);
                TopicMetadata metadata = new TopicMetadata(topicId, properties);
                if (metadata.exists()) {
                    topicIds.add(topicId);
                }
            }
        }
        return topicIds;
    } catch (IOException e) {
        throw exceptionHandler.handle(e);
    }
}