Search in sources :

Example 1 with RawMessageTableEntry

use of io.cdap.cdap.messaging.store.RawMessageTableEntry in project cdap by caskdata.

the class LevelDBMessageTable method persist.

@Override
protected void persist(Iterator<RawMessageTableEntry> entries) throws IOException {
    // entries are sorted by publish time. accumulate all entries for a partition into a batch and
    // write the batch when the next entry is outside of the current partition
    LevelDBPartition partition = null;
    WriteBatch writeBatch = null;
    while (entries.hasNext()) {
        RawMessageTableEntry entry = entries.next();
        byte[] rowKey = entry.getKey().getRowKey();
        long publishTime = entry.getKey().getPublishTimestamp();
        // check if this entry belongs in a different partition. If so, write the current batch.
        if (partition == null || publishTime < partition.getStartTime() || publishTime >= partition.getEndTime()) {
            if (partition != null) {
                try {
                    partition.getLevelDB().write(writeBatch, WRITE_OPTIONS);
                } finally {
                    writeBatch.close();
                }
            }
            partition = partitionManager.getOrCreatePartition(publishTime);
            writeBatch = partition.getLevelDB().createWriteBatch();
        }
        // LevelDB doesn't make copies, and since we reuse RawMessageTableEntry object, we need to create copies.
        writeBatch.put(Arrays.copyOf(rowKey, rowKey.length), encodeValue(entry.getTxPtr(), entry.getPayload()));
    }
    if (partition != null) {
        try {
            partition.getLevelDB().write(writeBatch, WRITE_OPTIONS);
        } finally {
            writeBatch.close();
        }
    }
}
Also used : RawMessageTableEntry(io.cdap.cdap.messaging.store.RawMessageTableEntry) WriteBatch(org.iq80.leveldb.WriteBatch)

Example 2 with RawMessageTableEntry

use of io.cdap.cdap.messaging.store.RawMessageTableEntry in project cdap by caskdata.

the class HBaseMessageTable method persist.

@Override
protected void persist(Iterator<RawMessageTableEntry> entries) throws IOException {
    List<Put> batchPuts = new ArrayList<>();
    while (entries.hasNext()) {
        RawMessageTableEntry entry = entries.next();
        PutBuilder putBuilder = tableUtil.buildPut(rowKeyDistributor.getDistributedKey(entry.getKey().getRowKey()));
        if (entry.getTxPtr() != null) {
            putBuilder.add(columnFamily, TX_COL, entry.getTxPtr());
        }
        if (entry.getPayload() != null) {
            putBuilder.add(columnFamily, PAYLOAD_COL, entry.getPayload());
        }
        batchPuts.add(putBuilder.build());
    }
    try {
        if (!batchPuts.isEmpty()) {
            mutator.mutate(batchPuts);
            mutator.flush();
        }
    } catch (IOException e) {
        throw exceptionHandler.handle(e);
    }
}
Also used : PutBuilder(io.cdap.cdap.data2.util.hbase.PutBuilder) ArrayList(java.util.ArrayList) RawMessageTableEntry(io.cdap.cdap.messaging.store.RawMessageTableEntry) IOException(java.io.IOException) Put(org.apache.hadoop.hbase.client.Put)

Example 3 with RawMessageTableEntry

use of io.cdap.cdap.messaging.store.RawMessageTableEntry in project cdap by caskdata.

the class HBaseMessageTable method scan.

@Override
protected CloseableIterator<RawMessageTableEntry> scan(ScanRequest scanRequest) throws IOException {
    Scan scan = tableUtil.buildScan().setStartRow(scanRequest.getStartRow()).setStopRow(scanRequest.getStopRow()).setCaching(scanCacheRows).build();
    TopicMetadata topicMetadata = scanRequest.getTopicMetadata();
    byte[] topic = MessagingUtils.toDataKeyPrefix(topicMetadata.getTopicId(), topicMetadata.getGeneration());
    MessageTableKey messageTableKey = MessageTableKey.fromTopic(topic);
    try {
        final ResultScanner scanner = DistributedScanner.create(table, scan, rowKeyDistributor, scanExecutor);
        final RawMessageTableEntry tableEntry = new RawMessageTableEntry();
        return new AbstractCloseableIterator<RawMessageTableEntry>() {

            private boolean closed = false;

            @Override
            protected RawMessageTableEntry computeNext() {
                if (closed) {
                    return endOfData();
                }
                Result result;
                try {
                    result = scanner.next();
                } catch (IOException e) {
                    throw exceptionHandler.handleAndWrap(e);
                }
                if (result == null) {
                    return endOfData();
                }
                byte[] originalKey = rowKeyDistributor.getOriginalKey(result.getRow());
                messageTableKey.setFromRowKey(originalKey);
                return tableEntry.set(messageTableKey, result.getValue(columnFamily, TX_COL), result.getValue(columnFamily, PAYLOAD_COL));
            }

            @Override
            public void close() {
                try {
                    scanner.close();
                } finally {
                    endOfData();
                    closed = true;
                }
            }
        };
    } catch (IOException e) {
        throw exceptionHandler.handle(e);
    }
}
Also used : ResultScanner(org.apache.hadoop.hbase.client.ResultScanner) AbstractCloseableIterator(io.cdap.cdap.api.dataset.lib.AbstractCloseableIterator) RawMessageTableEntry(io.cdap.cdap.messaging.store.RawMessageTableEntry) Scan(org.apache.hadoop.hbase.client.Scan) IOException(java.io.IOException) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) MessageTableKey(io.cdap.cdap.messaging.store.MessageTableKey) Result(org.apache.hadoop.hbase.client.Result)

Example 4 with RawMessageTableEntry

use of io.cdap.cdap.messaging.store.RawMessageTableEntry in project cdap by caskdata.

the class LevelDBMessageTable method scan.

@Override
protected CloseableIterator<RawMessageTableEntry> scan(ScanRequest scanRequest) throws IOException {
    Collection<LevelDBPartition> partitions = partitionManager.getPartitions(scanRequest.getStartTime());
    if (partitions.isEmpty()) {
        return CloseableIterator.empty();
    }
    RawMessageTableEntry tableEntry = new RawMessageTableEntry();
    TopicMetadata topicMetadata = scanRequest.getTopicMetadata();
    byte[] topic = MessagingUtils.toDataKeyPrefix(topicMetadata.getTopicId(), topicMetadata.getGeneration());
    MessageTableKey messageTableKey = MessageTableKey.fromTopic(topic);
    BiFunction<byte[], byte[], RawMessageTableEntry> decodeFunction = (key, value) -> {
        Map<String, byte[]> columns = decodeValue(value);
        messageTableKey.setFromRowKey(key);
        return tableEntry.set(messageTableKey, columns.get(TX_COL), columns.get(PAYLOAD_COL));
    };
    return new PartitionedDBScanIterator<>(partitions.iterator(), scanRequest.getStartRow(), scanRequest.getStopRow(), decodeFunction);
}
Also used : WriteBatch(org.iq80.leveldb.WriteBatch) MessagingUtils(io.cdap.cdap.messaging.MessagingUtils) DB(org.iq80.leveldb.DB) AbstractMessageTable(io.cdap.cdap.messaging.store.AbstractMessageTable) WriteOptions(org.iq80.leveldb.WriteOptions) Arrays(java.util.Arrays) ScanRequest(io.cdap.cdap.messaging.store.ScanRequest) MessageTable(io.cdap.cdap.messaging.store.MessageTable) Iterator(java.util.Iterator) DBException(org.iq80.leveldb.DBException) Collection(java.util.Collection) BiFunction(java.util.function.BiFunction) Bytes(io.cdap.cdap.api.common.Bytes) IOException(java.io.IOException) HashMap(java.util.HashMap) CloseableIterator(io.cdap.cdap.api.dataset.lib.CloseableIterator) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) RawMessageTableEntry(io.cdap.cdap.messaging.store.RawMessageTableEntry) Map(java.util.Map) Preconditions(com.google.common.base.Preconditions) MessageTableKey(io.cdap.cdap.messaging.store.MessageTableKey) RollbackRequest(io.cdap.cdap.messaging.store.RollbackRequest) Nullable(javax.annotation.Nullable) RawMessageTableEntry(io.cdap.cdap.messaging.store.RawMessageTableEntry) HashMap(java.util.HashMap) Map(java.util.Map) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) MessageTableKey(io.cdap.cdap.messaging.store.MessageTableKey)

Aggregations

RawMessageTableEntry (io.cdap.cdap.messaging.store.RawMessageTableEntry)4 IOException (java.io.IOException)3 TopicMetadata (io.cdap.cdap.messaging.TopicMetadata)2 MessageTableKey (io.cdap.cdap.messaging.store.MessageTableKey)2 WriteBatch (org.iq80.leveldb.WriteBatch)2 Preconditions (com.google.common.base.Preconditions)1 Bytes (io.cdap.cdap.api.common.Bytes)1 AbstractCloseableIterator (io.cdap.cdap.api.dataset.lib.AbstractCloseableIterator)1 CloseableIterator (io.cdap.cdap.api.dataset.lib.CloseableIterator)1 PutBuilder (io.cdap.cdap.data2.util.hbase.PutBuilder)1 MessagingUtils (io.cdap.cdap.messaging.MessagingUtils)1 AbstractMessageTable (io.cdap.cdap.messaging.store.AbstractMessageTable)1 MessageTable (io.cdap.cdap.messaging.store.MessageTable)1 RollbackRequest (io.cdap.cdap.messaging.store.RollbackRequest)1 ScanRequest (io.cdap.cdap.messaging.store.ScanRequest)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collection (java.util.Collection)1 HashMap (java.util.HashMap)1 Iterator (java.util.Iterator)1