use of co.cask.cdap.data2.transaction.queue.hbase.HBaseConsumerStateStore in project cdap by caskdata.
the class HBaseQueueTest method testQueueUpgrade.
// This test upgrade from old queue (salted base) to new queue (sharded base)
@Test(timeout = 30000L)
public void testQueueUpgrade() throws Exception {
final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "upgrade");
HBaseQueueAdmin hbaseQueueAdmin = (HBaseQueueAdmin) queueAdmin;
HBaseQueueClientFactory hBaseQueueClientFactory = (HBaseQueueClientFactory) queueClientFactory;
// Create the old queue table explicitly
HBaseQueueAdmin oldQueueAdmin = new HBaseQueueAdmin(hConf, cConf, injector.getInstance(LocationFactory.class), injector.getInstance(HBaseTableUtil.class), injector.getInstance(DatasetFramework.class), injector.getInstance(TransactionExecutorFactory.class), QueueConstants.QueueType.QUEUE, injector.getInstance(NamespaceQueryAdmin.class), injector.getInstance(Impersonator.class));
oldQueueAdmin.create(queueName);
int buckets = cConf.getInt(QueueConstants.ConfigKeys.QUEUE_TABLE_PRESPLITS);
try (final HBaseQueueProducer oldProducer = hBaseQueueClientFactory.createProducer(oldQueueAdmin, queueName, QueueConstants.QueueType.QUEUE, QueueMetrics.NOOP_QUEUE_METRICS, new SaltedHBaseQueueStrategy(tableUtil, buckets), new ArrayList<ConsumerGroupConfig>())) {
// Enqueue 10 items to old queue table
Transactions.createTransactionExecutor(executorFactory, oldProducer).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
for (int i = 0; i < 10; i++) {
oldProducer.enqueue(new QueueEntry("key", i, Bytes.toBytes("Message " + i)));
}
}
});
}
// Configure the consumer
final ConsumerConfig consumerConfig = new ConsumerConfig(0L, 0, 1, DequeueStrategy.HASH, "key");
try (QueueConfigurer configurer = queueAdmin.getQueueConfigurer(queueName)) {
Transactions.createTransactionExecutor(executorFactory, configurer).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
configurer.configureGroups(ImmutableList.of(consumerConfig));
}
});
}
// explicit set the consumer state to be the lowest start row
try (HBaseConsumerStateStore stateStore = hbaseQueueAdmin.getConsumerStateStore(queueName)) {
Transactions.createTransactionExecutor(executorFactory, stateStore).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
stateStore.updateState(consumerConfig.getGroupId(), consumerConfig.getInstanceId(), QueueEntryRow.getQueueEntryRowKey(queueName, 0L, 0));
}
});
}
// Enqueue 10 more items to new queue table
createEnqueueRunnable(queueName, 10, 1, null).run();
// Verify both old and new table have 10 rows each
Assert.assertEquals(10, countRows(hbaseQueueAdmin.getDataTableId(queueName, QueueConstants.QueueType.QUEUE)));
Assert.assertEquals(10, countRows(hbaseQueueAdmin.getDataTableId(queueName, QueueConstants.QueueType.SHARDED_QUEUE)));
// Create a consumer. It should see all 20 items
final List<String> messages = Lists.newArrayList();
try (final QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
while (messages.size() != 20) {
Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
DequeueResult<byte[]> result = consumer.dequeue(20);
for (byte[] data : result) {
messages.add(Bytes.toString(data));
}
}
});
}
}
verifyQueueIsEmpty(queueName, ImmutableList.of(consumerConfig));
}
use of co.cask.cdap.data2.transaction.queue.hbase.HBaseConsumerStateStore in project cdap by caskdata.
the class HBaseQueueDebugger method scanQueue.
/**
* Only works for {@link co.cask.cdap.data2.transaction.queue.hbase.ShardedHBaseQueueStrategy}.
*/
public QueueStatistics scanQueue(final QueueName queueName, @Nullable Long consumerGroupId) throws Exception {
HBaseConsumerStateStore stateStore;
try {
stateStore = queueAdmin.getConsumerStateStore(queueName);
} catch (IllegalStateException e) {
throw new NotFoundException(queueName);
}
TransactionExecutor txExecutor = Transactions.createTransactionExecutor(txExecutorFactory, stateStore);
Multimap<Long, QueueBarrier> barriers = txExecutor.execute(new TransactionExecutor.Function<HBaseConsumerStateStore, Multimap<Long, QueueBarrier>>() {
@Override
public Multimap<Long, QueueBarrier> apply(HBaseConsumerStateStore input) throws Exception {
return input.getAllBarriers();
}
}, stateStore);
printProgress("Got %d barriers\n", barriers.size());
QueueStatistics stats = new QueueStatistics();
if (consumerGroupId != null) {
barriers = Multimaps.filterKeys(barriers, Predicates.equalTo(consumerGroupId));
}
for (Map.Entry<Long, Collection<QueueBarrier>> entry : barriers.asMap().entrySet()) {
long groupId = entry.getKey();
Collection<QueueBarrier> groupBarriers = entry.getValue();
printProgress("Scanning barriers for group %d\n", groupId);
int currentSection = 1;
PeekingIterator<QueueBarrier> barrierIterator = Iterators.peekingIterator(groupBarriers.iterator());
while (barrierIterator.hasNext()) {
QueueBarrier start = barrierIterator.next();
QueueBarrier end = barrierIterator.hasNext() ? barrierIterator.peek() : null;
printProgress("Scanning section %d/%d...\n", currentSection, groupBarriers.size());
scanQueue(txExecutor, stateStore, queueName, start, end, stats);
printProgress("Current results: %s\n", stats.getReport(showTxTimestampOnly()));
currentSection++;
}
printProgress("Scanning complete");
}
System.out.printf("Results for queue %s: %s\n", queueName.toString(), stats.getReport(showTxTimestampOnly()));
return stats;
}
Aggregations