Examples with QueueType co.cask.cdap.data2.transaction.queue.QueueConstants.QueueType used on opensource projects

Example 1 with QueueType

use of co.cask.cdap.data2.transaction.queue.QueueConstants.QueueType in project cdap by caskdata.

the class HBaseQueueClientFactory method createConsumer.

@Override
public QueueConsumer createConsumer(final QueueName queueName, final ConsumerConfig consumerConfig, int numGroups) throws IOException {
    final HBaseQueueAdmin admin = ensureTableExists(queueName);
    try {
        final long groupId = consumerConfig.getGroupId();
        // A callback for create a list of HBaseQueueConsumer
        // based on the current queue consumer state of the given group
        Callable<List<HBaseQueueConsumer>> consumerCreator = new Callable<List<HBaseQueueConsumer>>() {

            @Override
            public List<HBaseQueueConsumer> call() throws Exception {
                List<HBaseConsumerState> states;
                try (HBaseConsumerStateStore stateStore = admin.getConsumerStateStore(queueName)) {
                    TransactionExecutor txExecutor = Transactions.createTransactionExecutor(txExecutorFactory, stateStore);
                    // Find all consumer states for consumers that need to be created based on current state
                    states = txExecutor.execute(new Callable<List<HBaseConsumerState>>() {

                        @Override
                        public List<HBaseConsumerState> call() throws Exception {
                            List<HBaseConsumerState> consumerStates = Lists.newArrayList();
                            HBaseConsumerState state = stateStore.getState(groupId, consumerConfig.getInstanceId());
                            if (state.getPreviousBarrier() == null) {
                                // Old HBase consumer (Salted based, not sharded)
                                consumerStates.add(state);
                                return consumerStates;
                            }
                            // Find the smallest start barrier that has something to consume for this instance.
                            // It should always exists since we assume the queue is configured before this method is called
                            List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(groupId);
                            if (queueBarriers.isEmpty()) {
                                throw new IllegalStateException(String.format("No consumer information available. Queue: %s, GroupId: %d, InstanceId: %d", queueName, groupId, consumerConfig.getInstanceId()));
                            }
                            QueueBarrier startBarrier = Iterables.find(Lists.reverse(queueBarriers), new Predicate<QueueBarrier>() {

                                @Override
                                public boolean apply(QueueBarrier barrier) {
                                    return barrier.getGroupConfig().getGroupSize() > consumerConfig.getInstanceId() && stateStore.isAllConsumed(consumerConfig, barrier.getStartRow());
                                }
                            }, queueBarriers.get(0));
                            int groupSize = startBarrier.getGroupConfig().getGroupSize();
                            for (int i = consumerConfig.getInstanceId(); i < groupSize; i += consumerConfig.getGroupSize()) {
                                consumerStates.add(stateStore.getState(groupId, i));
                            }
                            return consumerStates;
                        }
                    });
                }
                List<HBaseQueueConsumer> consumers = Lists.newArrayList();
                for (HBaseConsumerState state : states) {
                    QueueType queueType = (state.getPreviousBarrier() == null) ? QueueType.QUEUE : QueueType.SHARDED_QUEUE;
                    HTable hTable = createHTable(admin.getDataTableId(queueName, queueType));
                    int distributorBuckets = getDistributorBuckets(hTable.getTableDescriptor());
                    HBaseQueueStrategy strategy = (state.getPreviousBarrier() == null) ? new SaltedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets) : new ShardedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets);
                    consumers.add(queueUtil.getQueueConsumer(cConf, hTable, queueName, state, admin.getConsumerStateStore(queueName), strategy));
                }
                return consumers;
            }
        };
        return new SmartQueueConsumer(queueName, consumerConfig, consumerCreator);
    } catch (Exception e) {
        // If there is exception, nothing much can be done here besides propagating
        Throwables.propagateIfPossible(e);
        throw new IOException(e);
    }
}

Example 2 with QueueType

use of co.cask.cdap.data2.transaction.queue.QueueConstants.QueueType in project cdap by caskdata.

the class HBaseQueueAdmin method dropAllInNamespace.

@Override
public void dropAllInNamespace(NamespaceId namespaceId) throws Exception {
    Set<QueueConstants.QueueType> queueTypes = EnumSet.of(QueueConstants.QueueType.QUEUE, QueueConstants.QueueType.SHARDED_QUEUE);
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
        for (QueueConstants.QueueType queueType : queueTypes) {
            // Note: The trailing "." is crucial, since otherwise nsId could match nsId1, nsIdx etc
            // It's important to keep config table enabled while disabling and dropping  queue tables.
            final String queueTableNamePrefix = String.format("%s.%s.", NamespaceId.SYSTEM.getNamespace(), queueType);
            final String hbaseNamespace = tableUtil.getHBaseNamespace(namespaceId);
            final TableId configTableId = TableId.from(hbaseNamespace, getConfigTableName());
            tableUtil.deleteAllInNamespace(ddlExecutor, hbaseNamespace, hConf, new Predicate<TableId>() {

                @Override
                public boolean apply(TableId tableId) {
                    // It's a bit hacky here since we know how the Dataset System names tables
                    return (tableId.getTableName().startsWith(queueTableNamePrefix)) && !tableId.equals(configTableId);
                }
            });
        }
    }
    // Delete the state store in the namespace
    DatasetId id = getStateStoreId(namespaceId.getEntityName());
    if (datasetFramework.hasInstance(id)) {
        datasetFramework.deleteInstance(id);
    }
}