use of co.cask.cdap.data2.transaction.queue.hbase.ShardedHBaseQueueStrategy in project cdap by caskdata.
the class HBaseQueueClientFactory method createConsumer.
@Override
public QueueConsumer createConsumer(final QueueName queueName, final ConsumerConfig consumerConfig, int numGroups) throws IOException {
final HBaseQueueAdmin admin = ensureTableExists(queueName);
try {
final long groupId = consumerConfig.getGroupId();
// A callback for create a list of HBaseQueueConsumer
// based on the current queue consumer state of the given group
Callable<List<HBaseQueueConsumer>> consumerCreator = new Callable<List<HBaseQueueConsumer>>() {
@Override
public List<HBaseQueueConsumer> call() throws Exception {
List<HBaseConsumerState> states;
try (HBaseConsumerStateStore stateStore = admin.getConsumerStateStore(queueName)) {
TransactionExecutor txExecutor = Transactions.createTransactionExecutor(txExecutorFactory, stateStore);
// Find all consumer states for consumers that need to be created based on current state
states = txExecutor.execute(new Callable<List<HBaseConsumerState>>() {
@Override
public List<HBaseConsumerState> call() throws Exception {
List<HBaseConsumerState> consumerStates = Lists.newArrayList();
HBaseConsumerState state = stateStore.getState(groupId, consumerConfig.getInstanceId());
if (state.getPreviousBarrier() == null) {
// Old HBase consumer (Salted based, not sharded)
consumerStates.add(state);
return consumerStates;
}
// Find the smallest start barrier that has something to consume for this instance.
// It should always exists since we assume the queue is configured before this method is called
List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(groupId);
if (queueBarriers.isEmpty()) {
throw new IllegalStateException(String.format("No consumer information available. Queue: %s, GroupId: %d, InstanceId: %d", queueName, groupId, consumerConfig.getInstanceId()));
}
QueueBarrier startBarrier = Iterables.find(Lists.reverse(queueBarriers), new Predicate<QueueBarrier>() {
@Override
public boolean apply(QueueBarrier barrier) {
return barrier.getGroupConfig().getGroupSize() > consumerConfig.getInstanceId() && stateStore.isAllConsumed(consumerConfig, barrier.getStartRow());
}
}, queueBarriers.get(0));
int groupSize = startBarrier.getGroupConfig().getGroupSize();
for (int i = consumerConfig.getInstanceId(); i < groupSize; i += consumerConfig.getGroupSize()) {
consumerStates.add(stateStore.getState(groupId, i));
}
return consumerStates;
}
});
}
List<HBaseQueueConsumer> consumers = Lists.newArrayList();
for (HBaseConsumerState state : states) {
QueueType queueType = (state.getPreviousBarrier() == null) ? QueueType.QUEUE : QueueType.SHARDED_QUEUE;
HTable hTable = createHTable(admin.getDataTableId(queueName, queueType));
int distributorBuckets = getDistributorBuckets(hTable.getTableDescriptor());
HBaseQueueStrategy strategy = (state.getPreviousBarrier() == null) ? new SaltedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets) : new ShardedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets);
consumers.add(queueUtil.getQueueConsumer(cConf, hTable, queueName, state, admin.getConsumerStateStore(queueName), strategy));
}
return consumers;
}
};
return new SmartQueueConsumer(queueName, consumerConfig, consumerCreator);
} catch (Exception e) {
// If there is exception, nothing much can be done here besides propagating
Throwables.propagateIfPossible(e);
throw new IOException(e);
}
}
use of co.cask.cdap.data2.transaction.queue.hbase.ShardedHBaseQueueStrategy in project cdap by caskdata.
the class HBaseQueueDebugger method scanQueue.
private void scanQueue(TransactionExecutor txExecutor, HBaseConsumerStateStore stateStore, QueueName queueName, QueueBarrier start, @Nullable QueueBarrier end, final QueueStatistics outStats) throws Exception {
final byte[] queueRowPrefix = QueueEntryRow.getQueueRowPrefix(queueName);
ConsumerGroupConfig groupConfig = start.getGroupConfig();
printProgress("Got consumer group config: %s\n", groupConfig);
HBaseQueueAdmin admin = queueClientFactory.getQueueAdmin();
TableId tableId = admin.getDataTableId(queueName, QueueConstants.QueueType.SHARDED_QUEUE);
HTable hTable = queueClientFactory.createHTable(tableId);
printProgress("Looking at HBase table: %s\n", Bytes.toString(hTable.getTableName()));
final byte[] stateColumnName = Bytes.add(QueueEntryRow.STATE_COLUMN_PREFIX, Bytes.toBytes(groupConfig.getGroupId()));
int distributorBuckets = queueClientFactory.getDistributorBuckets(hTable.getTableDescriptor());
ShardedHBaseQueueStrategy queueStrategy = new ShardedHBaseQueueStrategy(tableUtil, distributorBuckets);
ScanBuilder scan = tableUtil.buildScan();
scan.setStartRow(start.getStartRow());
if (end != null) {
scan.setStopRow(end.getStartRow());
} else {
scan.setStopRow(QueueEntryRow.getQueueEntryRowKey(queueName, Long.MAX_VALUE, Integer.MAX_VALUE));
}
// Needs to include meta column for row that doesn't have state yet.
scan.addColumn(QueueEntryRow.COLUMN_FAMILY, QueueEntryRow.META_COLUMN);
scan.addColumn(QueueEntryRow.COLUMN_FAMILY, stateColumnName);
// Don't do block cache for debug tool. We don't want old blocks get cached
scan.setCacheBlocks(false);
scan.setMaxVersions(1);
printProgress("Scanning section with scan: %s\n", scan.toString());
List<Integer> instanceIds = Lists.newArrayList();
if (groupConfig.getDequeueStrategy() == DequeueStrategy.FIFO) {
instanceIds.add(0);
} else {
for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
instanceIds.add(instanceId);
}
}
final int rowsCache = Integer.parseInt(System.getProperty(PROP_ROWS_CACHE, "100000"));
for (final int instanceId : instanceIds) {
printProgress("Processing instance %d", instanceId);
ConsumerConfig consConfig = new ConsumerConfig(groupConfig, instanceId);
final QueueScanner scanner = queueStrategy.createScanner(consConfig, hTable, scan.build(), rowsCache);
try {
txExecutor.execute(new TransactionExecutor.Procedure<HBaseConsumerStateStore>() {
@Override
public void apply(HBaseConsumerStateStore input) throws Exception {
ImmutablePair<byte[], Map<byte[], byte[]>> result;
while ((result = scanner.next()) != null) {
byte[] rowKey = result.getFirst();
Map<byte[], byte[]> columns = result.getSecond();
visitRow(outStats, input.getTransaction(), rowKey, columns.get(stateColumnName), queueRowPrefix.length);
if (showProgress() && outStats.getTotal() % rowsCache == 0) {
System.out.printf("\rProcessing instance %d: %s", instanceId, outStats.getReport(showTxTimestampOnly()));
}
}
}
}, stateStore);
} catch (TransactionFailureException e) {
// Ignore transaction not in progress exception as it's caused by short TX timeout on commit
if (!(Throwables.getRootCause(e) instanceof TransactionNotInProgressException)) {
throw Throwables.propagate(e);
}
}
printProgress("\rProcessing instance %d: %s\n", instanceId, outStats.getReport(showTxTimestampOnly()));
}
}
use of co.cask.cdap.data2.transaction.queue.hbase.ShardedHBaseQueueStrategy in project cdap by caskdata.
the class HBaseQueueClientFactory method createProducer.
@Override
public QueueProducer createProducer(QueueName queueName, QueueMetrics queueMetrics) throws IOException {
HBaseQueueAdmin admin = ensureTableExists(queueName);
try {
final List<ConsumerGroupConfig> groupConfigs = Lists.newArrayList();
try (HBaseConsumerStateStore stateStore = admin.getConsumerStateStore(queueName)) {
Transactions.createTransactionExecutor(txExecutorFactory, stateStore).execute(new Subroutine() {
@Override
public void apply() throws Exception {
stateStore.getLatestConsumerGroups(groupConfigs);
}
});
}
Preconditions.checkState(!groupConfigs.isEmpty(), "Missing consumer group information for queue %s", queueName);
HTable hTable = createHTable(admin.getDataTableId(queueName, queueAdmin.getType()));
int distributorBuckets = getDistributorBuckets(hTable.getTableDescriptor());
return createProducer(hTable, queueName, queueMetrics, new ShardedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets), groupConfigs);
} catch (Exception e) {
Throwables.propagateIfPossible(e);
throw new IOException(e);
}
}
Aggregations