Search in sources :

Example 21 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class DataCleanupTest method testPayloadTTLCleanup.

@Test
public void testPayloadTTLCleanup() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("t2");
    TopicMetadata topic = new TopicMetadata(topicId, "ttl", "3", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
    try (MetadataTable metadataTable = getMetadataTable();
        PayloadTable payloadTable = getPayloadTable(topic);
        MessageTable messageTable = getMessageTable(topic)) {
        Assert.assertNotNull(messageTable);
        metadataTable.createTopic(topic);
        List<PayloadTable.Entry> entries = new ArrayList<>();
        entries.add(new TestPayloadEntry(topicId, GENERATION, "payloaddata", 101, (short) 0));
        payloadTable.store(entries.iterator());
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
        Assert.assertTrue(iterator.hasNext());
        PayloadTable.Entry entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
        Assert.assertEquals(101L, entry.getTransactionWritePointer());
        iterator.close();
        forceFlushAndCompact(Table.PAYLOAD);
        // Entry should still be there since ttl has not expired
        iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
        Assert.assertTrue(iterator.hasNext());
        entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals("payloaddata", Bytes.toString(entry.getPayload()));
        Assert.assertEquals(101L, entry.getTransactionWritePointer());
        iterator.close();
        TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
        forceFlushAndCompact(Table.PAYLOAD);
        iterator = payloadTable.fetch(topic, 101L, new MessageId(messageId), true, Integer.MAX_VALUE);
        Assert.assertFalse(iterator.hasNext());
        iterator.close();
        metadataTable.deleteTopic(topicId);
    }
}
Also used : ArrayList(java.util.ArrayList) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) TopicId(io.cdap.cdap.proto.id.TopicId) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Example 22 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class DataCleanupTest method testOldGenCleanup.

@Test
public void testOldGenCleanup() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("oldGenCleanup");
    TopicMetadata topic = new TopicMetadata(topicId, TopicMetadata.TTL_KEY, "100000", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
    try (MetadataTable metadataTable = getMetadataTable()) {
        int txWritePtr = 100;
        metadataTable.createTopic(topic);
        List<MessageTable.Entry> entries = new ArrayList<>();
        List<PayloadTable.Entry> pentries = new ArrayList<>();
        byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
        MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
        entries.add(new TestMessageEntry(topicId, GENERATION, "data", txWritePtr, (short) 0));
        pentries.add(new TestPayloadEntry(topicId, GENERATION, "data", txWritePtr, (short) 0));
        try (MessageTable messageTable = getMessageTable(topic);
            PayloadTable payloadTable = getPayloadTable(topic)) {
            messageTable.store(entries.iterator());
            payloadTable.store(pentries.iterator());
        }
        // Fetch the entries and make sure we are able to read it
        try (MessageTable messageTable = getMessageTable(topic);
            CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            checkMessageEntry(iterator, txWritePtr);
        }
        try (PayloadTable payloadTable = getPayloadTable(topic);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
            checkPayloadEntry(iterator, txWritePtr);
        }
        // Now run full compaction
        forceFlushAndCompact(Table.MESSAGE);
        forceFlushAndCompact(Table.PAYLOAD);
        // Fetch the entries and make sure we are able to read it
        try (MessageTable messageTable = getMessageTable(topic);
            CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            checkMessageEntry(iterator, txWritePtr);
        }
        try (PayloadTable payloadTable = getPayloadTable(topic);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
            checkPayloadEntry(iterator, txWritePtr);
        }
        // delete the topic and recreate it with an incremented generation
        metadataTable.deleteTopic(topicId);
        Map<String, String> newProperties = new HashMap<>(topic.getProperties());
        newProperties.put(TopicMetadata.GENERATION_KEY, Integer.toString(topic.getGeneration() + 1));
        topic = new TopicMetadata(topicId, newProperties);
        metadataTable.createTopic(topic);
        // Sleep so that the metadata cache in coprocessor expires
        TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
        forceFlushAndCompact(Table.MESSAGE);
        forceFlushAndCompact(Table.PAYLOAD);
        topic = metadataTable.getMetadata(topicId);
        try (MessageTable messageTable = getMessageTable(topic);
            CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
            Assert.assertFalse(iterator.hasNext());
        }
        try (PayloadTable payloadTable = getPayloadTable(topic);
            CloseableIterator<PayloadTable.Entry> iterator = payloadTable.fetch(topic, txWritePtr, new MessageId(messageId), true, 100)) {
            Assert.assertFalse(iterator.hasNext());
        }
    }
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) TopicId(io.cdap.cdap.proto.id.TopicId) MessageId(io.cdap.cdap.messaging.data.MessageId) Test(org.junit.Test)

Example 23 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class DataCleanupTest method testMessageTTLCleanup.

@Test
public void testMessageTTLCleanup() throws Exception {
    TopicId topicId = NamespaceId.DEFAULT.topic("t1");
    TopicMetadata topic = new TopicMetadata(topicId, "ttl", "3", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
    try (MetadataTable metadataTable = getMetadataTable();
        MessageTable messageTable = getMessageTable(topic);
        PayloadTable payloadTable = getPayloadTable(topic)) {
        Assert.assertNotNull(payloadTable);
        metadataTable.createTopic(topic);
        List<MessageTable.Entry> entries = new ArrayList<>();
        entries.add(new TestMessageEntry(topicId, GENERATION, "data", 100, (short) 0));
        messageTable.store(entries.iterator());
        // Fetch the entries and make sure we are able to read it
        CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
        Assert.assertTrue(iterator.hasNext());
        MessageTable.Entry entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals(100, entry.getTransactionWritePointer());
        Assert.assertEquals("data", Bytes.toString(entry.getPayload()));
        iterator.close();
        forceFlushAndCompact(Table.MESSAGE);
        // Entry should still be there since the ttl has not expired
        iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
        entry = iterator.next();
        Assert.assertFalse(iterator.hasNext());
        Assert.assertEquals(100, entry.getTransactionWritePointer());
        Assert.assertEquals("data", Bytes.toString(entry.getPayload()));
        iterator.close();
        TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
        forceFlushAndCompact(Table.MESSAGE);
        iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null);
        Assert.assertFalse(iterator.hasNext());
        iterator.close();
    }
}
Also used : ArrayList(java.util.ArrayList) TopicId(io.cdap.cdap.proto.id.TopicId) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) Test(org.junit.Test)

Example 24 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class HBaseMetadataTable method scanTopics.

/**
 * Scans the HBase table to get a list of {@link TopicId}.
 */
private List<TopicId> scanTopics(ScanBuilder scanBuilder) throws IOException {
    Scan scan = scanBuilder.setFilter(new FirstKeyOnlyFilter()).setCaching(scanCacheRows).build();
    try {
        List<TopicId> topicIds = new ArrayList<>();
        try (ResultScanner resultScanner = table.getScanner(scan)) {
            for (Result result : resultScanner) {
                TopicId topicId = MessagingUtils.toTopicId(result.getRow());
                byte[] value = result.getValue(columnFamily, COL);
                Map<String, String> properties = GSON.fromJson(Bytes.toString(value), MAP_TYPE);
                TopicMetadata metadata = new TopicMetadata(topicId, properties);
                if (metadata.exists()) {
                    topicIds.add(topicId);
                }
            }
        }
        return topicIds;
    } catch (IOException e) {
        throw exceptionHandler.handle(e);
    }
}
Also used : ResultScanner(org.apache.hadoop.hbase.client.ResultScanner) FirstKeyOnlyFilter(org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter) ArrayList(java.util.ArrayList) Scan(org.apache.hadoop.hbase.client.Scan) TopicId(io.cdap.cdap.proto.id.TopicId) IOException(java.io.IOException) Result(org.apache.hadoop.hbase.client.Result) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata)

Example 25 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class LevelDBMetadataTable method createTopic.

@Override
public void createTopic(TopicMetadata topicMetadata) throws TopicAlreadyExistsException, IOException {
    try {
        TopicId topicId = topicMetadata.getTopicId();
        byte[] key = MessagingUtils.toMetadataRowKey(topicId);
        TreeMap<String, String> properties = new TreeMap<>(topicMetadata.getProperties());
        properties.put(TopicMetadata.GENERATION_KEY, MessagingUtils.Constants.DEFAULT_GENERATION);
        synchronized (this) {
            byte[] tableValue = levelDB.get(key);
            if (tableValue != null) {
                Map<String, String> oldProperties = GSON.fromJson(Bytes.toString(tableValue), MAP_TYPE);
                TopicMetadata metadata = new TopicMetadata(topicId, oldProperties);
                if (metadata.exists()) {
                    throw new TopicAlreadyExistsException(topicId.getNamespace(), topicId.getTopic());
                }
                int newGenerationId = (metadata.getGeneration() * -1) + 1;
                properties.put(TopicMetadata.GENERATION_KEY, Integer.toString(newGenerationId));
            }
            byte[] value = Bytes.toBytes(GSON.toJson(properties, MAP_TYPE));
            levelDB.put(key, value, WRITE_OPTIONS);
        }
    } catch (DBException e) {
        throw new IOException(e);
    }
}
Also used : DBException(org.iq80.leveldb.DBException) TopicId(io.cdap.cdap.proto.id.TopicId) IOException(java.io.IOException) TreeMap(java.util.TreeMap) TopicAlreadyExistsException(io.cdap.cdap.api.messaging.TopicAlreadyExistsException) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata)

Aggregations

TopicId (io.cdap.cdap.proto.id.TopicId)152 Test (org.junit.Test)84 TopicMetadata (io.cdap.cdap.messaging.TopicMetadata)80 ArrayList (java.util.ArrayList)60 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)44 RawMessage (io.cdap.cdap.messaging.data.RawMessage)32 TopicNotFoundException (io.cdap.cdap.api.messaging.TopicNotFoundException)28 IOException (java.io.IOException)26 MessageId (io.cdap.cdap.messaging.data.MessageId)24 Path (javax.ws.rs.Path)24 HashMap (java.util.HashMap)16 TopicAlreadyExistsException (io.cdap.cdap.api.messaging.TopicAlreadyExistsException)14 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)14 Message (io.cdap.cdap.api.messaging.Message)12 BadRequestException (io.cdap.cdap.common.BadRequestException)12 Injector (com.google.inject.Injector)10 MessagingService (io.cdap.cdap.messaging.MessagingService)10 StoreRequest (io.cdap.cdap.messaging.StoreRequest)10 POST (javax.ws.rs.POST)10 NoOpMetricsCollectionService (io.cdap.cdap.common.metrics.NoOpMetricsCollectionService)8