Examples with Transaction org.apache.rocketmq.hbase.sink.Transaction used on opensource projects

Example 1 with Transaction

use of org.apache.rocketmq.hbase.sink.Transaction in project rocketmq-externals by apache.

the class ReplicatorTest method insertData.

/**
 * Adds data to the previously created HBase table.
 *
 * @throws IOException
 */
private Transaction insertData(int numberOfRecords) throws IOException {
    final Transaction transaction = new Transaction(numberOfRecords);
    try (Table hTable = ConnectionFactory.createConnection(utility.getConfiguration()).getTable(TABLE_NAME)) {
        for (int i = 0; i < numberOfRecords; i++) {
            final byte[] rowKey = toBytes(String.format(ROWKEY, i));
            final byte[] family = toBytes(COLUMN_FAMILY);
            final Put put = new Put(rowKey);
            put.addColumn(family, toBytes(QUALIFIER), toBytes(VALUE));
            hTable.put(put);
            List<Cell> cells = put.getFamilyCellMap().get(family);
            transaction.addRow(TABLE_NAME_STR, rowKey, cells);
        }
    }
    return transaction;
}

Example 2 with Transaction

use of org.apache.rocketmq.hbase.sink.Transaction in project rocketmq-externals by apache.

the class ReplicatorTest method testCustomReplicationEndpoint.

/**
 * This method tests the replicator by writing data from hbase to rocketmq and reading it back.
 *
 * @throws IOException
 * @throws ReplicationException
 * @throws InterruptedException
 * @throws MQClientException
 * @throws RemotingException
 * @throws MQBrokerException
 */
@Test
public void testCustomReplicationEndpoint() throws IOException, ReplicationException, InterruptedException, MQClientException, RemotingException, MQBrokerException {
    final DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(CONSUMER_GROUP_NAME);
    try {
        createTestTable();
        final Map<TableName, List<String>> tableCfs = new HashMap<>();
        List<String> cfs = new ArrayList<>();
        cfs.add(COLUMN_FAMILY);
        tableCfs.put(TABLE_NAME, cfs);
        addPeer(utility.getConfiguration(), PEER_NAME, tableCfs);
        // wait for new peer to be added
        Thread.sleep(500);
        final int numberOfRecords = 10;
        final Transaction inTransaction = insertData(numberOfRecords);
        // wait for data to be replicated
        Thread.sleep(500);
        consumer.setNamesrvAddr(NAMESERVER);
        consumer.setMessageModel(MessageModel.valueOf("BROADCASTING"));
        consumer.registerMessageQueueListener(ROCKETMQ_TOPIC, null);
        consumer.start();
        int receiveNum = 0;
        String receiveMsg = null;
        Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(ROCKETMQ_TOPIC);
        for (MessageQueue queue : queues) {
            long offset = getMessageQueueOffset(consumer, queue);
            PullResult pullResult = consumer.pull(queue, null, offset, batchSize);
            if (pullResult.getPullStatus() == PullStatus.FOUND) {
                for (MessageExt message : pullResult.getMsgFoundList()) {
                    byte[] body = message.getBody();
                    receiveMsg = new String(body, "UTF-8");
                    // String[] receiveMsgKv = receiveMsg.split(",");
                    // msgs.remove(receiveMsgKv[1]);
                    logger.info("receive message : {}", receiveMsg);
                    receiveNum++;
                }
                long nextBeginOffset = pullResult.getNextBeginOffset();
                consumer.updateConsumeOffset(queue, offset);
            }
        }
        logger.info("receive message num={}", receiveNum);
        // wait for processQueueTable init
        Thread.sleep(1000);
        assertEquals(inTransaction.toJson(), receiveMsg);
    } finally {
        removePeer();
        consumer.shutdown();
    }
}