Examples with ConsumerGroupConfig co.cask.cdap.data2.queue.ConsumerGroupConfig used on opensource projects

Example 1 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class HBaseConsumerStateStore method configureInstances.

@Override
public void configureInstances(long groupId, int instances) {
    // Find the last barrier info to get the existing group config
    List<QueueBarrier> queueBarriers = scanBarriers(groupId, new AllCollector<QueueBarrier>()).finish(new ArrayList<QueueBarrier>());
    Preconditions.checkState(!queueBarriers.isEmpty(), "No queue configuration found for group %s", groupId);
    QueueBarrier queueBarrier = queueBarriers.get(queueBarriers.size() - 1);
    ConsumerGroupConfig oldGroupConfig = queueBarrier.getGroupConfig();
    ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(groupId, instances, oldGroupConfig.getDequeueStrategy(), oldGroupConfig.getHashKey());
    byte[] startRow = QueueEntryRow.getQueueEntryRowKey(queueName, transaction.getWritePointer(), 0);
    Put put = new Put(Bytes.add(queueName.toBytes(), startRow));
    put.add(Bytes.toBytes(groupConfig.getGroupId()), GSON.toJson(groupConfig));
    table.put(put);
    // For instances that don't have start row, set the start row to barrier start row
    // We fetches all instances here for cleanup of barrier info later.
    Map<Integer, byte[]> startRows = fetchStartRows(groupId, Integer.MAX_VALUE);
    for (int instanceId = 0; instanceId < instances; instanceId++) {
        if (!startRows.containsKey(instanceId)) {
            table.put(queueName.toBytes(), getConsumerStateColumn(groupId, instanceId), startRow);
        }
    }
    // Remove barrier info that all instances has passed the start row it records
    Deque<byte[]> deletes = Lists.newLinkedList();
    for (QueueBarrier info : queueBarriers) {
        boolean allPassed = true;
        for (byte[] instanceStartRow : startRows.values()) {
            if (Bytes.compareTo(instanceStartRow, info.getStartRow()) <= 0) {
                allPassed = false;
                break;
            }
        }
        if (!allPassed) {
            break;
        }
        deletes.add(Bytes.add(queueName.toBytes(), info.getStartRow()));
    }
    // Retain the last barrier info
    if (deletes.size() > 1) {
        deletes.removeLast();
        byte[] column = Bytes.toBytes(groupId);
        for (byte[] delete : deletes) {
            table.delete(delete, column);
        }
    }
}

Example 2 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class HBaseConsumerStateStore method getLatestConsumerGroups.

void getLatestConsumerGroups(Collection<? super ConsumerGroupConfig> result) {
    try (Scanner scanner = table.scan(barrierScanStartRow, barrierScanEndRow)) {
        // Get the last row
        Row lastRow = null;
        Row row = scanner.next();
        while (row != null) {
            lastRow = row;
            row = scanner.next();
        }
        if (lastRow == null) {
            throw new IllegalStateException("No consumer group information. Queue: " + queueName);
        }
        for (Map.Entry<byte[], byte[]> entry : lastRow.getColumns().entrySet()) {
            result.add(GSON.fromJson(new String(entry.getValue(), Charsets.UTF_8), ConsumerGroupConfig.class));
        }
    }
}

Example 3 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class HBaseConsumerStateStore method configureGroups.

@Override
public void configureGroups(Iterable<? extends ConsumerGroupConfig> groupConfigs) {
    com.google.common.collect.Table<Long, Integer, byte[]> startRows = fetchAllStartRows();
    // Writes a new barrier info for all the groups
    byte[] startRow = QueueEntryRow.getQueueEntryRowKey(queueName, transaction.getWritePointer(), 0);
    Put put = new Put(Bytes.add(queueName.toBytes(), startRow));
    Set<Long> groupsIds = Sets.newHashSet();
    for (ConsumerGroupConfig groupConfig : groupConfigs) {
        long groupId = groupConfig.getGroupId();
        if (!groupsIds.add(groupId)) {
            throw new IllegalArgumentException("Same consumer group is provided multiple times");
        }
        put.add(Bytes.toBytes(groupId), GSON.toJson(groupConfig));
        // For new instance, set the start row to barrier start row
        for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
            if (!startRows.contains(groupId, instanceId)) {
                table.put(queueName.toBytes(), getConsumerStateColumn(groupId, instanceId), startRow);
            }
        }
    }
    // Remove all states for groups that are removed.
    deleteRemovedGroups(table.get(queueName.toBytes()), groupsIds);
    // Remove all barriers for groups that are removed.
    // Also remove barriers that have all consumers consumed pass that barrier
    // Multimap from groupId to barrier start rows. Ordering need to be maintained as the scan order.
    Multimap<Long, byte[]> deletes = LinkedHashMultimap.create();
    try (Scanner scanner = table.scan(barrierScanStartRow, barrierScanEndRow)) {
        Row row = scanner.next();
        while (row != null) {
            deleteRemovedGroups(row, groupsIds);
            // Check all instances in all groups
            for (Map.Entry<byte[], byte[]> entry : row.getColumns().entrySet()) {
                QueueBarrier barrier = decodeBarrierInfo(row.getRow(), entry.getValue());
                if (barrier == null) {
                    continue;
                }
                long groupId = barrier.getGroupConfig().getGroupId();
                boolean delete = true;
                // Check if all instances in a group has consumed passed the current barrier
                for (int instanceId = 0; instanceId < barrier.getGroupConfig().getGroupSize(); instanceId++) {
                    byte[] consumerStartRow = startRows.get(groupId, instanceId);
                    if (consumerStartRow == null || Bytes.compareTo(consumerStartRow, barrier.getStartRow()) < 0) {
                        delete = false;
                        break;
                    }
                }
                if (delete) {
                    deletes.put(groupId, row.getRow());
                }
            }
            row = scanner.next();
        }
    }
    // Remove barries that have all consumers consumed passed it
    for (Map.Entry<Long, Collection<byte[]>> entry : deletes.asMap().entrySet()) {
        // Retains the last barrier info
        if (entry.getValue().size() <= 1) {
            continue;
        }
        Deque<byte[]> rows = Lists.newLinkedList(entry.getValue());
        rows.removeLast();
        byte[] groupColumn = Bytes.toBytes(entry.getKey());
        for (byte[] rowKey : rows) {
            table.delete(rowKey, groupColumn);
        }
    }
    table.put(put);
}

Example 4 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class HBaseConsumerStateStore method getState.

/**
 * Returns the consumer state as stored in the state store for the given consumer.
 */
HBaseConsumerState getState(long groupId, int instanceId) {
    // Lookup the start row for the given instance, also search the barriers that bound the start row
    ConsumerState consumerState = getConsumerState(groupId, instanceId);
    QueueBarrier previousBarrier = consumerState.getPreviousBarrier();
    QueueBarrier nextBarrier = consumerState.getNextBarrier();
    if (previousBarrier == null && nextBarrier == null) {
        throw new IllegalStateException(String.format("Unable to find barrier information for consumer. Queue: %s, GroupId: %d, InstanceId:%d", queueName, groupId, instanceId));
    }
    // There are three possible cases:
    // 1. previousBarrier == null. It means in old compat mode. Since in old queue we didn't record the
    // consumer group config, we assume it's the same as the one recorded by
    // the nextBarrier (which was written when Flow start for the first time with new queue)
    // 2. nextBarrier == null. It means pasted the last barrier. The consumer scan is unbounded
    // 3. both not null. The scan is bounded by the nextBarrier and
    // the consumer group config is described by the previousBarrier.
    ConsumerGroupConfig groupConfig = previousBarrier != null ? previousBarrier.getGroupConfig() : nextBarrier.getGroupConfig();
    ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
    return new HBaseConsumerState(consumerConfig, consumerState.getConsumerStartRow(), previousBarrier == null ? null : previousBarrier.getStartRow(), nextBarrier == null ? null : nextBarrier.getStartRow());
}

Example 5 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class ShardedHBaseQueueStrategy method getRowKeys.

@Override
public void getRowKeys(Iterable<ConsumerGroupConfig> consumerGroupConfigs, QueueEntry queueEntry, byte[] rowKeyPrefix, long writePointer, int counter, Collection<byte[]> rowKeys) {
    byte[] rowKey = new byte[rowKeyPrefix.length + Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT];
    Bytes.putBytes(rowKey, 0, rowKeyPrefix, 0, rowKeyPrefix.length);
    Bytes.putLong(rowKey, rowKeyPrefix.length, writePointer);
    Bytes.putInt(rowKey, rowKey.length - Bytes.SIZEOF_INT, counter);
    // Generates all row keys, one per consumer group.
    for (ConsumerGroupConfig config : consumerGroupConfigs) {
        DequeueStrategy dequeueStrategy = config.getDequeueStrategy();
        // Default for FIFO
        int instanceId = -1;
        if (dequeueStrategy != DequeueStrategy.FIFO) {
            if (dequeueStrategy == DequeueStrategy.ROUND_ROBIN) {
                instanceId = QueueEntryRow.getRoundRobinConsumerInstance(writePointer, counter, config.getGroupSize());
            } else if (dequeueStrategy == DequeueStrategy.HASH) {
                instanceId = QueueEntryRow.getHashConsumerInstance(queueEntry.getHashKeys(), config.getHashKey(), config.getGroupSize());
            } else {
                throw new IllegalArgumentException("Unsupported consumer strategy: " + dequeueStrategy);
            }
        }
        rowKeys.add(rowKeyDistributor.getDistributedKey(getShardedKey(config, instanceId, rowKey)));
    }
}