Examples with QueueBarrier co.cask.cdap.data2.transaction.queue.hbase.QueueBarrier used on opensource projects

Example 1 with QueueBarrier

use of co.cask.cdap.data2.transaction.queue.hbase.QueueBarrier in project cdap by caskdata.

the class HBaseConsumerStateStore method configureInstances.

@Override
public void configureInstances(long groupId, int instances) {
    // Find the last barrier info to get the existing group config
    List<QueueBarrier> queueBarriers = scanBarriers(groupId, new AllCollector<QueueBarrier>()).finish(new ArrayList<QueueBarrier>());
    Preconditions.checkState(!queueBarriers.isEmpty(), "No queue configuration found for group %s", groupId);
    QueueBarrier queueBarrier = queueBarriers.get(queueBarriers.size() - 1);
    ConsumerGroupConfig oldGroupConfig = queueBarrier.getGroupConfig();
    ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(groupId, instances, oldGroupConfig.getDequeueStrategy(), oldGroupConfig.getHashKey());
    byte[] startRow = QueueEntryRow.getQueueEntryRowKey(queueName, transaction.getWritePointer(), 0);
    Put put = new Put(Bytes.add(queueName.toBytes(), startRow));
    put.add(Bytes.toBytes(groupConfig.getGroupId()), GSON.toJson(groupConfig));
    table.put(put);
    // For instances that don't have start row, set the start row to barrier start row
    // We fetches all instances here for cleanup of barrier info later.
    Map<Integer, byte[]> startRows = fetchStartRows(groupId, Integer.MAX_VALUE);
    for (int instanceId = 0; instanceId < instances; instanceId++) {
        if (!startRows.containsKey(instanceId)) {
            table.put(queueName.toBytes(), getConsumerStateColumn(groupId, instanceId), startRow);
        }
    }
    // Remove barrier info that all instances has passed the start row it records
    Deque<byte[]> deletes = Lists.newLinkedList();
    for (QueueBarrier info : queueBarriers) {
        boolean allPassed = true;
        for (byte[] instanceStartRow : startRows.values()) {
            if (Bytes.compareTo(instanceStartRow, info.getStartRow()) <= 0) {
                allPassed = false;
                break;
            }
        }
        if (!allPassed) {
            break;
        }
        deletes.add(Bytes.add(queueName.toBytes(), info.getStartRow()));
    }
    // Retain the last barrier info
    if (deletes.size() > 1) {
        deletes.removeLast();
        byte[] column = Bytes.toBytes(groupId);
        for (byte[] delete : deletes) {
            table.delete(delete, column);
        }
    }
}

Example 2 with QueueBarrier

use of co.cask.cdap.data2.transaction.queue.hbase.QueueBarrier in project cdap by caskdata.

the class HBaseConsumerStateStore method configureGroups.

@Override
public void configureGroups(Iterable<? extends ConsumerGroupConfig> groupConfigs) {
    com.google.common.collect.Table<Long, Integer, byte[]> startRows = fetchAllStartRows();
    // Writes a new barrier info for all the groups
    byte[] startRow = QueueEntryRow.getQueueEntryRowKey(queueName, transaction.getWritePointer(), 0);
    Put put = new Put(Bytes.add(queueName.toBytes(), startRow));
    Set<Long> groupsIds = Sets.newHashSet();
    for (ConsumerGroupConfig groupConfig : groupConfigs) {
        long groupId = groupConfig.getGroupId();
        if (!groupsIds.add(groupId)) {
            throw new IllegalArgumentException("Same consumer group is provided multiple times");
        }
        put.add(Bytes.toBytes(groupId), GSON.toJson(groupConfig));
        // For new instance, set the start row to barrier start row
        for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
            if (!startRows.contains(groupId, instanceId)) {
                table.put(queueName.toBytes(), getConsumerStateColumn(groupId, instanceId), startRow);
            }
        }
    }
    // Remove all states for groups that are removed.
    deleteRemovedGroups(table.get(queueName.toBytes()), groupsIds);
    // Remove all barriers for groups that are removed.
    // Also remove barriers that have all consumers consumed pass that barrier
    // Multimap from groupId to barrier start rows. Ordering need to be maintained as the scan order.
    Multimap<Long, byte[]> deletes = LinkedHashMultimap.create();
    try (Scanner scanner = table.scan(barrierScanStartRow, barrierScanEndRow)) {
        Row row = scanner.next();
        while (row != null) {
            deleteRemovedGroups(row, groupsIds);
            // Check all instances in all groups
            for (Map.Entry<byte[], byte[]> entry : row.getColumns().entrySet()) {
                QueueBarrier barrier = decodeBarrierInfo(row.getRow(), entry.getValue());
                if (barrier == null) {
                    continue;
                }
                long groupId = barrier.getGroupConfig().getGroupId();
                boolean delete = true;
                // Check if all instances in a group has consumed passed the current barrier
                for (int instanceId = 0; instanceId < barrier.getGroupConfig().getGroupSize(); instanceId++) {
                    byte[] consumerStartRow = startRows.get(groupId, instanceId);
                    if (consumerStartRow == null || Bytes.compareTo(consumerStartRow, barrier.getStartRow()) < 0) {
                        delete = false;
                        break;
                    }
                }
                if (delete) {
                    deletes.put(groupId, row.getRow());
                }
            }
            row = scanner.next();
        }
    }
    // Remove barries that have all consumers consumed passed it
    for (Map.Entry<Long, Collection<byte[]>> entry : deletes.asMap().entrySet()) {
        // Retains the last barrier info
        if (entry.getValue().size() <= 1) {
            continue;
        }
        Deque<byte[]> rows = Lists.newLinkedList(entry.getValue());
        rows.removeLast();
        byte[] groupColumn = Bytes.toBytes(entry.getKey());
        for (byte[] rowKey : rows) {
            table.delete(rowKey, groupColumn);
        }
    }
    table.put(put);
}

Example 3 with QueueBarrier

use of co.cask.cdap.data2.transaction.queue.hbase.QueueBarrier in project cdap by caskdata.

the class HBaseConsumerStateStore method getState.

/**
 * Returns the consumer state as stored in the state store for the given consumer.
 */
HBaseConsumerState getState(long groupId, int instanceId) {
    // Lookup the start row for the given instance, also search the barriers that bound the start row
    ConsumerState consumerState = getConsumerState(groupId, instanceId);
    QueueBarrier previousBarrier = consumerState.getPreviousBarrier();
    QueueBarrier nextBarrier = consumerState.getNextBarrier();
    if (previousBarrier == null && nextBarrier == null) {
        throw new IllegalStateException(String.format("Unable to find barrier information for consumer. Queue: %s, GroupId: %d, InstanceId:%d", queueName, groupId, instanceId));
    }
    // There are three possible cases:
    // 1. previousBarrier == null. It means in old compat mode. Since in old queue we didn't record the
    // consumer group config, we assume it's the same as the one recorded by
    // the nextBarrier (which was written when Flow start for the first time with new queue)
    // 2. nextBarrier == null. It means pasted the last barrier. The consumer scan is unbounded
    // 3. both not null. The scan is bounded by the nextBarrier and
    // the consumer group config is described by the previousBarrier.
    ConsumerGroupConfig groupConfig = previousBarrier != null ? previousBarrier.getGroupConfig() : nextBarrier.getGroupConfig();
    ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
    return new HBaseConsumerState(consumerConfig, consumerState.getConsumerStartRow(), previousBarrier == null ? null : previousBarrier.getStartRow(), nextBarrier == null ? null : nextBarrier.getStartRow());
}

Example 4 with QueueBarrier

use of co.cask.cdap.data2.transaction.queue.hbase.QueueBarrier in project cdap by caskdata.

the class HBaseQueueClientFactory method createConsumer.

@Override
public QueueConsumer createConsumer(final QueueName queueName, final ConsumerConfig consumerConfig, int numGroups) throws IOException {
    final HBaseQueueAdmin admin = ensureTableExists(queueName);
    try {
        final long groupId = consumerConfig.getGroupId();
        // A callback for create a list of HBaseQueueConsumer
        // based on the current queue consumer state of the given group
        Callable<List<HBaseQueueConsumer>> consumerCreator = new Callable<List<HBaseQueueConsumer>>() {

            @Override
            public List<HBaseQueueConsumer> call() throws Exception {
                List<HBaseConsumerState> states;
                try (HBaseConsumerStateStore stateStore = admin.getConsumerStateStore(queueName)) {
                    TransactionExecutor txExecutor = Transactions.createTransactionExecutor(txExecutorFactory, stateStore);
                    // Find all consumer states for consumers that need to be created based on current state
                    states = txExecutor.execute(new Callable<List<HBaseConsumerState>>() {

                        @Override
                        public List<HBaseConsumerState> call() throws Exception {
                            List<HBaseConsumerState> consumerStates = Lists.newArrayList();
                            HBaseConsumerState state = stateStore.getState(groupId, consumerConfig.getInstanceId());
                            if (state.getPreviousBarrier() == null) {
                                // Old HBase consumer (Salted based, not sharded)
                                consumerStates.add(state);
                                return consumerStates;
                            }
                            // Find the smallest start barrier that has something to consume for this instance.
                            // It should always exists since we assume the queue is configured before this method is called
                            List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(groupId);
                            if (queueBarriers.isEmpty()) {
                                throw new IllegalStateException(String.format("No consumer information available. Queue: %s, GroupId: %d, InstanceId: %d", queueName, groupId, consumerConfig.getInstanceId()));
                            }
                            QueueBarrier startBarrier = Iterables.find(Lists.reverse(queueBarriers), new Predicate<QueueBarrier>() {

                                @Override
                                public boolean apply(QueueBarrier barrier) {
                                    return barrier.getGroupConfig().getGroupSize() > consumerConfig.getInstanceId() && stateStore.isAllConsumed(consumerConfig, barrier.getStartRow());
                                }
                            }, queueBarriers.get(0));
                            int groupSize = startBarrier.getGroupConfig().getGroupSize();
                            for (int i = consumerConfig.getInstanceId(); i < groupSize; i += consumerConfig.getGroupSize()) {
                                consumerStates.add(stateStore.getState(groupId, i));
                            }
                            return consumerStates;
                        }
                    });
                }
                List<HBaseQueueConsumer> consumers = Lists.newArrayList();
                for (HBaseConsumerState state : states) {
                    QueueType queueType = (state.getPreviousBarrier() == null) ? QueueType.QUEUE : QueueType.SHARDED_QUEUE;
                    HTable hTable = createHTable(admin.getDataTableId(queueName, queueType));
                    int distributorBuckets = getDistributorBuckets(hTable.getTableDescriptor());
                    HBaseQueueStrategy strategy = (state.getPreviousBarrier() == null) ? new SaltedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets) : new ShardedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets);
                    consumers.add(queueUtil.getQueueConsumer(cConf, hTable, queueName, state, admin.getConsumerStateStore(queueName), strategy));
                }
                return consumers;
            }
        };
        return new SmartQueueConsumer(queueName, consumerConfig, consumerCreator);
    } catch (Exception e) {
        // If there is exception, nothing much can be done here besides propagating
        Throwables.propagateIfPossible(e);
        throw new IOException(e);
    }
}

Example 5 with QueueBarrier

use of co.cask.cdap.data2.transaction.queue.hbase.QueueBarrier in project cdap by caskdata.

the class HBaseQueueTest method configTest.

@Test
public void configTest() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "configure");
    queueAdmin.create(queueName);
    final List<ConsumerGroupConfig> groupConfigs = ImmutableList.of(new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(2L, 2, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(3L, 3, DequeueStrategy.FIFO, null));
    try (HBaseConsumerStateStore stateStore = ((HBaseQueueAdmin) queueAdmin).getConsumerStateStore(queueName)) {
        TransactionExecutor txExecutor = Transactions.createTransactionExecutor(executorFactory, stateStore);
        // Intentionally set a row state for group 2, instance 0. It's for testing upgrade of config.
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                stateStore.updateState(2L, 0, QueueEntryRow.getQueueEntryRowKey(queueName, 10L, 0));
            }
        });
        // Set the group info
        configureGroups(queueName, groupConfigs);
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                for (ConsumerGroupConfig groupConfig : groupConfigs) {
                    long groupId = groupConfig.getGroupId();
                    List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(groupId);
                    Assert.assertEquals(1, queueBarriers.size());
                    for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
                        HBaseConsumerState state = stateStore.getState(groupId, instanceId);
                        if (groupId == 2L && instanceId == 0) {
                            // For group 2L instance 0, the start row shouldn't be changed.
                            // End row should be the same as the first barrier
                            Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), QueueEntryRow.getQueueEntryRowKey(queueName, 10L, 0)));
                            Assert.assertEquals(0, Bytes.compareTo(state.getNextBarrier(), queueBarriers.get(0).getStartRow()));
                        } else {
                            // For other group, they should have the start row the same as the first barrier info
                            Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(0).getStartRow()));
                        }
                    }
                }
            }
        });
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // Check consumers are all processed up to the barrier boundary
                for (long groupId = 1L; groupId <= 3L; groupId++) {
                    List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(groupId);
                    boolean allConsumed = stateStore.isAllConsumed(groupId, queueBarriers.get(0).getStartRow());
                    // For group 2, instance 0 is not consumed up to the boundary yet
                    Assert.assertTrue((groupId == 2L) != allConsumed);
                    if (groupId == 2L) {
                        // Mark group 2, instance 0 as completed the barrier.
                        stateStore.completed(groupId, 0);
                    }
                }
            }
        });
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // After group 2, instance 0 completed the current barrier, all consumers in group 2 should be able to
                // proceed
                List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(2L);
                byte[] startRow = stateStore.getState(2L, 0).getStartRow();
                Assert.assertEquals(0, Bytes.compareTo(startRow, queueBarriers.get(0).getStartRow()));
                Assert.assertTrue(stateStore.isAllConsumed(2L, startRow));
            }
        });
        // Add instance to group 2
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                stateStore.configureInstances(2L, 3);
            }
        });
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(2L);
                Assert.assertEquals(2, queueBarriers.size());
                // For existing instances, the start row shouldn't changed.
                for (int instanceId = 0; instanceId < 2; instanceId++) {
                    HBaseConsumerState state = stateStore.getState(2L, instanceId);
                    Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(0).getStartRow()));
                    Assert.assertEquals(0, Bytes.compareTo(state.getNextBarrier(), queueBarriers.get(1).getStartRow()));
                    // Complete the existing instance
                    stateStore.completed(2L, instanceId);
                }
                // For new instances, the start row should be the same as the new barrier
                HBaseConsumerState state = stateStore.getState(2L, 2);
                Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(1).getStartRow()));
                Assert.assertNull(state.getNextBarrier());
                // All instances should be consumed up to the beginning of the last barrier info
                Assert.assertTrue(stateStore.isAllConsumed(2L, queueBarriers.get(1).getStartRow()));
            }
        });
        // Reduce instances of group 2 through group reconfiguration, remove group 1 and 3, add group 4.
        configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(2L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(4L, 1, DequeueStrategy.FIFO, null)));
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // States and barrier info for removed groups should be gone
                try {
                    // There should be no barrier info for group 1
                    List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(1L);
                    Assert.assertTrue(queueBarriers.isEmpty());
                    stateStore.getState(1L, 0);
                    Assert.fail("Not expected to get state for group 1");
                } catch (Exception e) {
                // Expected
                }
                try {
                    // There should be no barrier info for group 3
                    List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(3L);
                    Assert.assertTrue(queueBarriers.isEmpty());
                    stateStore.getState(3L, 0);
                    Assert.fail("Not expected to get state for group 3");
                } catch (Exception e) {
                // Expected
                }
                // For group 2, there should be two barrier infos,
                // since all consumers passed the first barrier (groupSize = 2). Only the size = 3 and size = 1 left
                List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(2L);
                Assert.assertEquals(2, queueBarriers.size());
                // Make all consumers (3 of them before reconfigure) in group 2 consumes everything
                for (int instanceId = 0; instanceId < 3; instanceId++) {
                    stateStore.completed(2L, instanceId);
                }
                // For the remaining consumer, it should start consuming from the latest barrier
                HBaseConsumerState state = stateStore.getState(2L, 0);
                Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(1).getStartRow()));
                Assert.assertNull(state.getNextBarrier());
                // For removed instances, they should throw exception when retrieving their states
                for (int i = 1; i < 3; i++) {
                    try {
                        stateStore.getState(2L, i);
                        Assert.fail("Not expected to get state for group 2, instance " + i);
                    } catch (Exception e) {
                    // Expected
                    }
                }
            }
        });
    } finally {
        queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
    }
}