use of co.cask.cdap.data2.queue.ConsumerConfig in project cdap by caskdata.
the class HBaseStreamFileConsumerFactory method create.
@Override
protected StreamConsumer create(TableId tableId, StreamConfig streamConfig, ConsumerConfig consumerConfig, StreamConsumerStateStore stateStore, StreamConsumerState beginConsumerState, FileReader<StreamEventOffset, Iterable<StreamFileOffset>> reader, @Nullable ReadFilter extraFilter) throws IOException {
int splits = cConf.getInt(Constants.Stream.CONSUMER_TABLE_PRESPLITS);
AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix(new RowKeyDistributorByHashPrefix.OneByteSimpleHash(splits));
byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, distributor);
TableId hBaseTableId = tableUtil.createHTableId(new NamespaceId(tableId.getNamespace()), tableId.getTableName());
TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(hBaseTableId, cConf);
ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf);
tdBuilder.addColumnFamily(cfdBuilder.build());
tdBuilder.addProperty(QueueConstants.DISTRIBUTOR_BUCKETS, Integer.toString(splits));
try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys);
}
HTable hTable = tableUtil.createHTable(hConf, hBaseTableId);
hTable.setWriteBufferSize(Constants.Stream.HBASE_WRITE_BUFFER_SIZE);
hTable.setAutoFlushTo(false);
return new HBaseStreamFileConsumer(cConf, streamConfig, consumerConfig, tableUtil, hTable, reader, stateStore, beginConsumerState, extraFilter, createKeyDistributor(hTable.getTableDescriptor()));
}
use of co.cask.cdap.data2.queue.ConsumerConfig in project cdap by caskdata.
the class HBaseConsumerStateStore method getState.
/**
* Returns the consumer state as stored in the state store for the given consumer.
*/
HBaseConsumerState getState(long groupId, int instanceId) {
// Lookup the start row for the given instance, also search the barriers that bound the start row
ConsumerState consumerState = getConsumerState(groupId, instanceId);
QueueBarrier previousBarrier = consumerState.getPreviousBarrier();
QueueBarrier nextBarrier = consumerState.getNextBarrier();
if (previousBarrier == null && nextBarrier == null) {
throw new IllegalStateException(String.format("Unable to find barrier information for consumer. Queue: %s, GroupId: %d, InstanceId:%d", queueName, groupId, instanceId));
}
// There are three possible cases:
// 1. previousBarrier == null. It means in old compat mode. Since in old queue we didn't record the
// consumer group config, we assume it's the same as the one recorded by
// the nextBarrier (which was written when Flow start for the first time with new queue)
// 2. nextBarrier == null. It means pasted the last barrier. The consumer scan is unbounded
// 3. both not null. The scan is bounded by the nextBarrier and
// the consumer group config is described by the previousBarrier.
ConsumerGroupConfig groupConfig = previousBarrier != null ? previousBarrier.getGroupConfig() : nextBarrier.getGroupConfig();
ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
return new HBaseConsumerState(consumerConfig, consumerState.getConsumerStartRow(), previousBarrier == null ? null : previousBarrier.getStartRow(), nextBarrier == null ? null : nextBarrier.getStartRow());
}
use of co.cask.cdap.data2.queue.ConsumerConfig in project cdap by caskdata.
the class HBaseQueueClientFactory method createConsumer.
@Override
public QueueConsumer createConsumer(final QueueName queueName, final ConsumerConfig consumerConfig, int numGroups) throws IOException {
final HBaseQueueAdmin admin = ensureTableExists(queueName);
try {
final long groupId = consumerConfig.getGroupId();
// A callback for create a list of HBaseQueueConsumer
// based on the current queue consumer state of the given group
Callable<List<HBaseQueueConsumer>> consumerCreator = new Callable<List<HBaseQueueConsumer>>() {
@Override
public List<HBaseQueueConsumer> call() throws Exception {
List<HBaseConsumerState> states;
try (HBaseConsumerStateStore stateStore = admin.getConsumerStateStore(queueName)) {
TransactionExecutor txExecutor = Transactions.createTransactionExecutor(txExecutorFactory, stateStore);
// Find all consumer states for consumers that need to be created based on current state
states = txExecutor.execute(new Callable<List<HBaseConsumerState>>() {
@Override
public List<HBaseConsumerState> call() throws Exception {
List<HBaseConsumerState> consumerStates = Lists.newArrayList();
HBaseConsumerState state = stateStore.getState(groupId, consumerConfig.getInstanceId());
if (state.getPreviousBarrier() == null) {
// Old HBase consumer (Salted based, not sharded)
consumerStates.add(state);
return consumerStates;
}
// Find the smallest start barrier that has something to consume for this instance.
// It should always exists since we assume the queue is configured before this method is called
List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(groupId);
if (queueBarriers.isEmpty()) {
throw new IllegalStateException(String.format("No consumer information available. Queue: %s, GroupId: %d, InstanceId: %d", queueName, groupId, consumerConfig.getInstanceId()));
}
QueueBarrier startBarrier = Iterables.find(Lists.reverse(queueBarriers), new Predicate<QueueBarrier>() {
@Override
public boolean apply(QueueBarrier barrier) {
return barrier.getGroupConfig().getGroupSize() > consumerConfig.getInstanceId() && stateStore.isAllConsumed(consumerConfig, barrier.getStartRow());
}
}, queueBarriers.get(0));
int groupSize = startBarrier.getGroupConfig().getGroupSize();
for (int i = consumerConfig.getInstanceId(); i < groupSize; i += consumerConfig.getGroupSize()) {
consumerStates.add(stateStore.getState(groupId, i));
}
return consumerStates;
}
});
}
List<HBaseQueueConsumer> consumers = Lists.newArrayList();
for (HBaseConsumerState state : states) {
QueueType queueType = (state.getPreviousBarrier() == null) ? QueueType.QUEUE : QueueType.SHARDED_QUEUE;
HTable hTable = createHTable(admin.getDataTableId(queueName, queueType));
int distributorBuckets = getDistributorBuckets(hTable.getTableDescriptor());
HBaseQueueStrategy strategy = (state.getPreviousBarrier() == null) ? new SaltedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets) : new ShardedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets);
consumers.add(queueUtil.getQueueConsumer(cConf, hTable, queueName, state, admin.getConsumerStateStore(queueName), strategy));
}
return consumers;
}
};
return new SmartQueueConsumer(queueName, consumerConfig, consumerCreator);
} catch (Exception e) {
// If there is exception, nothing much can be done here besides propagating
Throwables.propagateIfPossible(e);
throw new IOException(e);
}
}
use of co.cask.cdap.data2.queue.ConsumerConfig in project cdap by caskdata.
the class AbstractStreamFileConsumerFactory method create.
@Override
public final StreamConsumer create(StreamId streamId, String namespace, ConsumerConfig consumerConfig) throws IOException {
StreamConfig streamConfig = StreamUtils.ensureExists(streamAdmin, streamId);
TableId tableId = getTableId(streamId, namespace);
StreamConsumerStateStore stateStore = stateStoreFactory.create(streamConfig);
StreamConsumerState consumerState = stateStore.get(consumerConfig.getGroupId(), consumerConfig.getInstanceId());
return create(tableId, streamConfig, consumerConfig, stateStore, consumerState, createReader(streamConfig, consumerState), new TTLReadFilter(streamConfig.getTTL()));
}
use of co.cask.cdap.data2.queue.ConsumerConfig in project cdap by caskdata.
the class HBaseQueueConsumer method updateStartRow.
@Override
protected void updateStartRow(byte[] startRow) {
if (canConsume && !completed) {
ConsumerConfig consumerConfig = getConfig();
stateStore.updateState(consumerConfig.getGroupId(), consumerConfig.getInstanceId(), startRow);
}
}
Aggregations