use of co.cask.cdap.data2.transaction.queue.hbase.QueueBarrier in project cdap by caskdata.
the class HBaseQueueTest method configTest.
@Test
public void configTest() throws Exception {
final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "configure");
queueAdmin.create(queueName);
final List<ConsumerGroupConfig> groupConfigs = ImmutableList.of(new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(2L, 2, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(3L, 3, DequeueStrategy.FIFO, null));
try (HBaseConsumerStateStore stateStore = ((HBaseQueueAdmin) queueAdmin).getConsumerStateStore(queueName)) {
TransactionExecutor txExecutor = Transactions.createTransactionExecutor(executorFactory, stateStore);
// Intentionally set a row state for group 2, instance 0. It's for testing upgrade of config.
txExecutor.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
stateStore.updateState(2L, 0, QueueEntryRow.getQueueEntryRowKey(queueName, 10L, 0));
}
});
// Set the group info
configureGroups(queueName, groupConfigs);
txExecutor.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
for (ConsumerGroupConfig groupConfig : groupConfigs) {
long groupId = groupConfig.getGroupId();
List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(groupId);
Assert.assertEquals(1, queueBarriers.size());
for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
HBaseConsumerState state = stateStore.getState(groupId, instanceId);
if (groupId == 2L && instanceId == 0) {
// For group 2L instance 0, the start row shouldn't be changed.
// End row should be the same as the first barrier
Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), QueueEntryRow.getQueueEntryRowKey(queueName, 10L, 0)));
Assert.assertEquals(0, Bytes.compareTo(state.getNextBarrier(), queueBarriers.get(0).getStartRow()));
} else {
// For other group, they should have the start row the same as the first barrier info
Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(0).getStartRow()));
}
}
}
}
});
txExecutor.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// Check consumers are all processed up to the barrier boundary
for (long groupId = 1L; groupId <= 3L; groupId++) {
List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(groupId);
boolean allConsumed = stateStore.isAllConsumed(groupId, queueBarriers.get(0).getStartRow());
// For group 2, instance 0 is not consumed up to the boundary yet
Assert.assertTrue((groupId == 2L) != allConsumed);
if (groupId == 2L) {
// Mark group 2, instance 0 as completed the barrier.
stateStore.completed(groupId, 0);
}
}
}
});
txExecutor.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// After group 2, instance 0 completed the current barrier, all consumers in group 2 should be able to
// proceed
List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(2L);
byte[] startRow = stateStore.getState(2L, 0).getStartRow();
Assert.assertEquals(0, Bytes.compareTo(startRow, queueBarriers.get(0).getStartRow()));
Assert.assertTrue(stateStore.isAllConsumed(2L, startRow));
}
});
// Add instance to group 2
txExecutor.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
stateStore.configureInstances(2L, 3);
}
});
txExecutor.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(2L);
Assert.assertEquals(2, queueBarriers.size());
// For existing instances, the start row shouldn't changed.
for (int instanceId = 0; instanceId < 2; instanceId++) {
HBaseConsumerState state = stateStore.getState(2L, instanceId);
Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(0).getStartRow()));
Assert.assertEquals(0, Bytes.compareTo(state.getNextBarrier(), queueBarriers.get(1).getStartRow()));
// Complete the existing instance
stateStore.completed(2L, instanceId);
}
// For new instances, the start row should be the same as the new barrier
HBaseConsumerState state = stateStore.getState(2L, 2);
Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(1).getStartRow()));
Assert.assertNull(state.getNextBarrier());
// All instances should be consumed up to the beginning of the last barrier info
Assert.assertTrue(stateStore.isAllConsumed(2L, queueBarriers.get(1).getStartRow()));
}
});
// Reduce instances of group 2 through group reconfiguration, remove group 1 and 3, add group 4.
configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(2L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(4L, 1, DequeueStrategy.FIFO, null)));
txExecutor.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// States and barrier info for removed groups should be gone
try {
// There should be no barrier info for group 1
List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(1L);
Assert.assertTrue(queueBarriers.isEmpty());
stateStore.getState(1L, 0);
Assert.fail("Not expected to get state for group 1");
} catch (Exception e) {
// Expected
}
try {
// There should be no barrier info for group 3
List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(3L);
Assert.assertTrue(queueBarriers.isEmpty());
stateStore.getState(3L, 0);
Assert.fail("Not expected to get state for group 3");
} catch (Exception e) {
// Expected
}
// For group 2, there should be two barrier infos,
// since all consumers passed the first barrier (groupSize = 2). Only the size = 3 and size = 1 left
List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(2L);
Assert.assertEquals(2, queueBarriers.size());
// Make all consumers (3 of them before reconfigure) in group 2 consumes everything
for (int instanceId = 0; instanceId < 3; instanceId++) {
stateStore.completed(2L, instanceId);
}
// For the remaining consumer, it should start consuming from the latest barrier
HBaseConsumerState state = stateStore.getState(2L, 0);
Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(1).getStartRow()));
Assert.assertNull(state.getNextBarrier());
// For removed instances, they should throw exception when retrieving their states
for (int i = 1; i < 3; i++) {
try {
stateStore.getState(2L, i);
Assert.fail("Not expected to get state for group 2, instance " + i);
} catch (Exception e) {
// Expected
}
}
}
});
} finally {
queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
}
}
use of co.cask.cdap.data2.transaction.queue.hbase.QueueBarrier in project cdap by caskdata.
the class HBaseConsumerStateStore method decodeBarrierInfo.
@Nullable
private QueueBarrier decodeBarrierInfo(byte[] rowKey, @Nullable byte[] groupInfo) {
if (groupInfo == null) {
return null;
}
ConsumerGroupConfig groupConfig = GSON.fromJson(new String(groupInfo, Charsets.UTF_8), ConsumerGroupConfig.class);
byte[] startRow = Arrays.copyOfRange(rowKey, queueName.toBytes().length, rowKey.length);
return new QueueBarrier(groupConfig, startRow);
}
use of co.cask.cdap.data2.transaction.queue.hbase.QueueBarrier in project cdap by caskdata.
the class HBaseConsumerStateStore method completed.
/**
* Called by consumer to signal process completion up to the current barrier that the consumer is in.
*/
void completed(long groupId, int instanceId) {
// Get the current consumer state to get the end barrier info
ConsumerState consumerState = getConsumerState(groupId, instanceId);
QueueBarrier nextBarrier = consumerState.getNextBarrier();
if (nextBarrier == null) {
// End row shouldn't be null if this method is called
throw new IllegalArgumentException(String.format("No end barrier information for consumer. Queue: %s, GroupId: %d, InstanceId: %d", queueName, groupId, instanceId));
}
byte[] stateColumn = getConsumerStateColumn(groupId, instanceId);
// If the instance exists in the next barrier, set the start row to the barrier start
if (instanceId < nextBarrier.getGroupConfig().getGroupSize()) {
table.put(queueName.toBytes(), stateColumn, nextBarrier.getStartRow());
return;
}
// find the next start barrier that this instance needs to consume from
try (Scanner scanner = table.scan(Bytes.add(queueName.toBytes(), nextBarrier.getStartRow()), barrierScanEndRow)) {
Row row;
boolean found = false;
while (!found && (row = scanner.next()) != null) {
QueueBarrier queueBarrier = decodeBarrierInfo(row, groupId);
if (queueBarrier == null || instanceId >= queueBarrier.getGroupConfig().getGroupSize()) {
continue;
}
table.put(queueName.toBytes(), stateColumn, queueBarrier.getStartRow());
found = true;
}
if (!found) {
// Remove the state since this consumer instance is not longer active
table.delete(queueName.toBytes(), stateColumn);
}
}
}
use of co.cask.cdap.data2.transaction.queue.hbase.QueueBarrier in project cdap by caskdata.
the class HBaseQueueDebugger method scanQueue.
/**
* Only works for {@link co.cask.cdap.data2.transaction.queue.hbase.ShardedHBaseQueueStrategy}.
*/
public QueueStatistics scanQueue(final QueueName queueName, @Nullable Long consumerGroupId) throws Exception {
HBaseConsumerStateStore stateStore;
try {
stateStore = queueAdmin.getConsumerStateStore(queueName);
} catch (IllegalStateException e) {
throw new NotFoundException(queueName);
}
TransactionExecutor txExecutor = Transactions.createTransactionExecutor(txExecutorFactory, stateStore);
Multimap<Long, QueueBarrier> barriers = txExecutor.execute(new TransactionExecutor.Function<HBaseConsumerStateStore, Multimap<Long, QueueBarrier>>() {
@Override
public Multimap<Long, QueueBarrier> apply(HBaseConsumerStateStore input) throws Exception {
return input.getAllBarriers();
}
}, stateStore);
printProgress("Got %d barriers\n", barriers.size());
QueueStatistics stats = new QueueStatistics();
if (consumerGroupId != null) {
barriers = Multimaps.filterKeys(barriers, Predicates.equalTo(consumerGroupId));
}
for (Map.Entry<Long, Collection<QueueBarrier>> entry : barriers.asMap().entrySet()) {
long groupId = entry.getKey();
Collection<QueueBarrier> groupBarriers = entry.getValue();
printProgress("Scanning barriers for group %d\n", groupId);
int currentSection = 1;
PeekingIterator<QueueBarrier> barrierIterator = Iterators.peekingIterator(groupBarriers.iterator());
while (barrierIterator.hasNext()) {
QueueBarrier start = barrierIterator.next();
QueueBarrier end = barrierIterator.hasNext() ? barrierIterator.peek() : null;
printProgress("Scanning section %d/%d...\n", currentSection, groupBarriers.size());
scanQueue(txExecutor, stateStore, queueName, start, end, stats);
printProgress("Current results: %s\n", stats.getReport(showTxTimestampOnly()));
currentSection++;
}
printProgress("Scanning complete");
}
System.out.printf("Results for queue %s: %s\n", queueName.toString(), stats.getReport(showTxTimestampOnly()));
return stats;
}
Aggregations