Examples with DequeueStrategy co.cask.cdap.data2.queue.DequeueStrategy used on opensource projects

Example 1 with DequeueStrategy

use of co.cask.cdap.data2.queue.DequeueStrategy in project cdap by caskdata.

the class ShardedHBaseQueueStrategy method getRowKeys.

@Override
public void getRowKeys(Iterable<ConsumerGroupConfig> consumerGroupConfigs, QueueEntry queueEntry, byte[] rowKeyPrefix, long writePointer, int counter, Collection<byte[]> rowKeys) {
    byte[] rowKey = new byte[rowKeyPrefix.length + Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT];
    Bytes.putBytes(rowKey, 0, rowKeyPrefix, 0, rowKeyPrefix.length);
    Bytes.putLong(rowKey, rowKeyPrefix.length, writePointer);
    Bytes.putInt(rowKey, rowKey.length - Bytes.SIZEOF_INT, counter);
    // Generates all row keys, one per consumer group.
    for (ConsumerGroupConfig config : consumerGroupConfigs) {
        DequeueStrategy dequeueStrategy = config.getDequeueStrategy();
        // Default for FIFO
        int instanceId = -1;
        if (dequeueStrategy != DequeueStrategy.FIFO) {
            if (dequeueStrategy == DequeueStrategy.ROUND_ROBIN) {
                instanceId = QueueEntryRow.getRoundRobinConsumerInstance(writePointer, counter, config.getGroupSize());
            } else if (dequeueStrategy == DequeueStrategy.HASH) {
                instanceId = QueueEntryRow.getHashConsumerInstance(queueEntry.getHashKeys(), config.getHashKey(), config.getGroupSize());
            } else {
                throw new IllegalArgumentException("Unsupported consumer strategy: " + dequeueStrategy);
            }
        }
        rowKeys.add(rowKeyDistributor.getDistributedKey(getShardedKey(config, instanceId, rowKey)));
    }
}

Example 2 with DequeueStrategy

use of co.cask.cdap.data2.queue.DequeueStrategy in project cdap by caskdata.

the class QueueTest method testOneEnqueueDequeue.

private void testOneEnqueueDequeue(DequeueStrategy strategy) throws Exception {
    // since this is used by more than one test method, ensure uniqueness of the queue name by adding strategy
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queue1" + strategy.toString());
    configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, strategy, null), new ConsumerGroupConfig(1L, 1, strategy, null)));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0L, 0, 1, strategy, null), new ConsumerConfig(1L, 0, 1, strategy, null));
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        TransactionContext txContext = createTxContext(producer);
        txContext.start();
        producer.enqueue(new QueueEntry(Bytes.toBytes(55)));
        txContext.finish();
        try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2)) {
            txContext = createTxContext(consumer);
            txContext.start();
            Assert.assertEquals(55, Bytes.toInt(consumer.dequeue().iterator().next()));
            txContext.finish();
        }
    }
    forceEviction(queueName, 2);
    // verifying that consumer of the 2nd group can process items: they were not evicted
    try (QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
        TransactionContext txContext = createTxContext(consumer2);
        txContext.start();
        Assert.assertEquals(55, Bytes.toInt(consumer2.dequeue().iterator().next()));
        txContext.finish();
    }
    // now all should be evicted
    verifyQueueIsEmpty(queueName, consumerConfigs);
}

Example 3 with DequeueStrategy

use of co.cask.cdap.data2.queue.DequeueStrategy in project cdap by caskdata.

the class FlowUtils method createConsumerGroupConfig.

/**
 * Creates a {@link ConsumerGroupConfig} by inspecting the given process method.
 */
public static ConsumerGroupConfig createConsumerGroupConfig(long groupId, int groupSize, Method processMethod) {
    // Determine input queue partition type
    HashPartition hashPartition = processMethod.getAnnotation(HashPartition.class);
    RoundRobin roundRobin = processMethod.getAnnotation(RoundRobin.class);
    DequeueStrategy strategy = DequeueStrategy.FIFO;
    String hashKey = null;
    Preconditions.checkArgument(!(hashPartition != null && roundRobin != null), "Only one strategy allowed for process() method: %s", processMethod.getName());
    if (hashPartition != null) {
        strategy = DequeueStrategy.HASH;
        hashKey = hashPartition.value();
        Preconditions.checkArgument(!hashKey.isEmpty(), "Partition key cannot be empty: %s", processMethod.getName());
    } else if (roundRobin != null) {
        strategy = DequeueStrategy.ROUND_ROBIN;
    }
    return new ConsumerGroupConfig(groupId, groupSize, strategy, hashKey);
}

Example 4 with DequeueStrategy

use of co.cask.cdap.data2.queue.DequeueStrategy in project cdap by caskdata.

the class AbstractStreamFileConsumer method createBaseReadFilter.

private ReadFilter createBaseReadFilter(final ConsumerConfig consumerConfig) {
    final int groupSize = consumerConfig.getGroupSize();
    final DequeueStrategy strategy = consumerConfig.getDequeueStrategy();
    if (groupSize == 1 || strategy == DequeueStrategy.FIFO) {
        return ReadFilter.ALWAYS_ACCEPT;
    }
    // For RoundRobin and Hash partition, the claim is done by matching hashCode to instance id.
    // For Hash, to preserve existing behavior, everything route to instance 0.
    // For RoundRobin, the idea is to scatter the events across consumers evenly. Since there is no way to known
    // about the absolute starting point to do true round robin, we employ a good enough hash function on the
    // file offset as a way to spread events across consumers
    final int instanceId = consumerConfig.getInstanceId();
    return new ReadFilter() {

        @Override
        public boolean acceptOffset(long offset) {
            int hashValue = Math.abs(strategy == DequeueStrategy.HASH ? 0 : ROUND_ROBIN_HASHER.hashLong(offset).hashCode());
            return instanceId == (hashValue % groupSize);
        }
    };
}

Example 5 with DequeueStrategy

use of co.cask.cdap.data2.queue.DequeueStrategy in project cdap by caskdata.

the class DequeueScanAttributes method readConsumerConfig.

public static ConsumerConfig readConsumerConfig(DataInput dataInput) throws IOException {
    long groupId = dataInput.readLong();
    int groupSize = dataInput.readInt();
    int instanceId = dataInput.readInt();
    DequeueStrategy strategy = WritableUtils.readEnum(dataInput, DequeueStrategy.class);
    String hashKey = WritableUtils.readString(dataInput);
    return new ConsumerConfig(groupId, instanceId, groupSize, strategy, hashKey);
}