use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.
the class QueueTest method testQueueAbortRetrySkip.
@Test(timeout = TIMEOUT_MS)
public void testQueueAbortRetrySkip() throws Exception {
QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queuefailure");
configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.HASH, "key")));
List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null), new ConsumerConfig(1, 0, 1, DequeueStrategy.HASH, "key"));
createEnqueueRunnable(queueName, 5, 1, null).run();
try (QueueConsumer fifoConsumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2);
QueueConsumer hashConsumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
TransactionContext txContext = createTxContext(fifoConsumer, hashConsumer);
txContext.start();
Assert.assertEquals(0, Bytes.toInt(fifoConsumer.dequeue().iterator().next()));
Assert.assertEquals(0, Bytes.toInt(hashConsumer.dequeue().iterator().next()));
// Abort the consumer transaction
txContext.abort();
// Dequeue again in a new transaction, should see the same entries
txContext.start();
Assert.assertEquals(0, Bytes.toInt(fifoConsumer.dequeue().iterator().next()));
Assert.assertEquals(0, Bytes.toInt(hashConsumer.dequeue().iterator().next()));
txContext.finish();
// Dequeue again, now should get next entry
txContext.start();
Assert.assertEquals(1, Bytes.toInt(fifoConsumer.dequeue().iterator().next()));
Assert.assertEquals(1, Bytes.toInt(hashConsumer.dequeue().iterator().next()));
txContext.finish();
// Dequeue a result and abort.
txContext.start();
DequeueResult<byte[]> fifoResult = fifoConsumer.dequeue();
DequeueResult<byte[]> hashResult = hashConsumer.dequeue();
Assert.assertEquals(2, Bytes.toInt(fifoResult.iterator().next()));
Assert.assertEquals(2, Bytes.toInt(hashResult.iterator().next()));
txContext.abort();
// Now skip the result with a new transaction.
txContext.start();
fifoResult.reclaim();
hashResult.reclaim();
txContext.finish();
// Dequeue again, it should see a new entry
txContext.start();
Assert.assertEquals(3, Bytes.toInt(fifoConsumer.dequeue().iterator().next()));
Assert.assertEquals(3, Bytes.toInt(hashConsumer.dequeue().iterator().next()));
txContext.finish();
// Dequeue again, it should see a new entry
txContext.start();
Assert.assertEquals(4, Bytes.toInt(fifoConsumer.dequeue().iterator().next()));
Assert.assertEquals(4, Bytes.toInt(hashConsumer.dequeue().iterator().next()));
txContext.finish();
}
verifyQueueIsEmpty(queueName, consumerConfigs);
}
use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.
the class QueueTest method testConcurrentEnqueue.
@Category(SlowTests.class)
@Test
public void testConcurrentEnqueue() throws Exception {
// This test is for testing multiple producers that writes with a delay after a transaction started.
// This is for verifying consumer advances the startKey correctly.
final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "concurrent");
configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0, 1, DequeueStrategy.FIFO, null)));
final CyclicBarrier barrier = new CyclicBarrier(4);
ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
// Starts three producers to enqueue concurrently. For each entry, starts a TX, sleep, enqueue, commit.
ExecutorService executor = Executors.newFixedThreadPool(3);
final int entryCount = 50;
for (int i = 0; i < 3; i++) {
final QueueProducer producer = queueClientFactory.createProducer(queueName);
final int producerId = i + 1;
executor.execute(new Runnable() {
@Override
public void run() {
try {
barrier.await();
for (int i = 0; i < entryCount; i++) {
TransactionContext txContext = createTxContext(producer);
txContext.start();
// Sleeps at different rate to make the scan in consumer has higher change to see
// the transaction but not the entry (as not yet written)
TimeUnit.MILLISECONDS.sleep(producerId * 50);
producer.enqueue(new QueueEntry(Bytes.toBytes(i)));
txContext.finish();
}
} catch (Exception e) {
LOG.error(e.getMessage(), e);
} finally {
Closeables.closeQuietly(producer);
}
}
});
}
// sum(0..entryCount) * 3
int expectedSum = entryCount * (entryCount - 1) / 2 * 3;
try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
// Trigger starts of producer
barrier.await();
int dequeueSum = 0;
int noProgress = 0;
while (dequeueSum != expectedSum && noProgress < 200) {
TransactionContext txContext = createTxContext(consumer);
txContext.start();
DequeueResult<byte[]> result = consumer.dequeue();
if (!result.isEmpty()) {
noProgress = 0;
int value = Bytes.toInt(result.iterator().next());
dequeueSum += value;
} else {
noProgress++;
TimeUnit.MILLISECONDS.sleep(10);
}
txContext.finish();
}
Assert.assertEquals(expectedSum, dequeueSum);
}
}
use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.
the class QueueTest method testDropAllForNamespace.
@Test
public void testDropAllForNamespace() throws Exception {
// create 4 queues
QueueName myQueue1 = QueueName.fromFlowlet(NAMESPACE_ID.getEntityName(), "myapp1", "myflow1", "myflowlet1", "myout1");
QueueName myQueue2 = QueueName.fromFlowlet(NAMESPACE_ID.getEntityName(), "myapp2", "myflow2", "myflowlet2", "myout2");
QueueName yourQueue1 = QueueName.fromFlowlet(NAMESPACE_ID1.getEntityName(), "yourapp1", "yourflow1", "yourflowlet1", "yourout1");
QueueName yourQueue2 = QueueName.fromFlowlet(NAMESPACE_ID1.getEntityName(), "yourapp2", "yourflow2", "yourflowlet2", "yourout2");
queueAdmin.create(myQueue1);
queueAdmin.create(myQueue2);
queueAdmin.create(yourQueue1);
queueAdmin.create(yourQueue2);
// verify that queues got created
Assert.assertTrue(queueAdmin.exists(myQueue1) && queueAdmin.exists(myQueue2) && queueAdmin.exists(yourQueue1) && queueAdmin.exists(yourQueue2));
// create some consumer configurations for all queues
List<ConsumerGroupConfig> groupConfigs = ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null));
configureGroups(myQueue1, groupConfigs);
configureGroups(myQueue2, groupConfigs);
configureGroups(yourQueue1, groupConfigs);
configureGroups(yourQueue2, groupConfigs);
// verify that the consumer config exists
verifyConsumerConfigExists(myQueue1, myQueue2, yourQueue1, yourQueue2);
// drop queues in NAMESPACE_ID
queueAdmin.dropAllInNamespace(NAMESPACE_ID);
// verify queues in NAMESPACE_ID are dropped
Assert.assertFalse(queueAdmin.exists(myQueue1) || queueAdmin.exists(myQueue2));
// also verify that consumer config of all queues in 'myspace' is deleted
verifyConsumerConfigIsDeleted(myQueue1, myQueue2);
// but the ones in NAMESPACE_ID1 still exist
Assert.assertTrue(queueAdmin.exists(yourQueue1) && queueAdmin.exists(yourQueue2));
// consumer config for queues in 'namespace1' should also still exist
verifyConsumerConfigExists(yourQueue1, yourQueue2);
// drop queues in NAMESPACE_ID1
queueAdmin.dropAllInNamespace(NAMESPACE_ID1);
// verify queues in NAMESPACE_ID are dropped
Assert.assertFalse(queueAdmin.exists(yourQueue1) || queueAdmin.exists(yourQueue2));
// verify that the consumer config of all queues in 'namespace1' is deleted
verifyConsumerConfigIsDeleted(yourQueue1, yourQueue2);
}
use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.
the class QueueTest method testOneEnqueueDequeue.
private void testOneEnqueueDequeue(DequeueStrategy strategy) throws Exception {
// since this is used by more than one test method, ensure uniqueness of the queue name by adding strategy
QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queue1" + strategy.toString());
configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, strategy, null), new ConsumerGroupConfig(1L, 1, strategy, null)));
List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0L, 0, 1, strategy, null), new ConsumerConfig(1L, 0, 1, strategy, null));
try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
TransactionContext txContext = createTxContext(producer);
txContext.start();
producer.enqueue(new QueueEntry(Bytes.toBytes(55)));
txContext.finish();
try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2)) {
txContext = createTxContext(consumer);
txContext.start();
Assert.assertEquals(55, Bytes.toInt(consumer.dequeue().iterator().next()));
txContext.finish();
}
}
forceEviction(queueName, 2);
// verifying that consumer of the 2nd group can process items: they were not evicted
try (QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
TransactionContext txContext = createTxContext(consumer2);
txContext.start();
Assert.assertEquals(55, Bytes.toInt(consumer2.dequeue().iterator().next()));
txContext.finish();
}
// now all should be evicted
verifyQueueIsEmpty(queueName, consumerConfigs);
}
use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.
the class HBaseQueueTest method testReconfigure.
@Test(timeout = 30000L)
public void testReconfigure() throws Exception {
final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "changeinstances");
ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(0L, 2, DequeueStrategy.HASH, "key");
configureGroups(queueName, ImmutableList.of(groupConfig));
// Enqueue 10 items
createEnqueueRunnable(queueName, 10, 1, null).run();
// Map from instance id to items dequeued
final Multimap<Integer, Integer> dequeued = ArrayListMultimap.create();
// Consume 2 items for each consumer instances
for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
final ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
DequeueResult<byte[]> result = consumer.dequeue(2);
Assert.assertEquals(2, result.size());
for (byte[] data : result) {
dequeued.put(consumerConfig.getInstanceId(), Bytes.toInt(data));
}
}
});
}
}
// Increase number of instances to 3
changeInstances(queueName, 0L, 3);
// Enqueue 10 more items
createEnqueueRunnable(queueName, 10, 1, null).run();
groupConfig = new ConsumerGroupConfig(0L, 3, DequeueStrategy.HASH, "key");
// Dequeue everything
while (dequeued.size() != 20) {
for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
final ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
for (byte[] data : consumer.dequeue(20)) {
dequeued.put(consumerConfig.getInstanceId(), Bytes.toInt(data));
}
}
});
}
}
}
// Instance 0 should see all evens before change instances
Assert.assertEquals(ImmutableList.of(0, 2, 4, 6, 8, 0, 3, 6, 9), dequeued.get(0));
// Instance 1 should see all odds before change instances
Assert.assertEquals(ImmutableList.of(1, 3, 5, 7, 9, 1, 4, 7), dequeued.get(1));
// Instance 2 should only see entries after change instances
Assert.assertEquals(ImmutableList.of(2, 5, 8), dequeued.get(2));
// All consumers should have empty dequeue now
for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
final ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
DequeueResult<byte[]> result = consumer.dequeue(20);
Assert.assertTrue(result.isEmpty());
}
});
}
}
// Enqueue 6 more items for the 3 instances
createEnqueueRunnable(queueName, 6, 1, null).run();
// Reduce to 1 consumer
changeInstances(queueName, 0L, 1);
// The consumer 0 should be able to consume all 10 new items
dequeued.clear();
final ConsumerConfig consumerConfig = new ConsumerConfig(0L, 0, 1, DequeueStrategy.HASH, "key");
try (final QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
while (dequeued.size() != 6) {
Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
for (byte[] data : consumer.dequeue(1)) {
dequeued.put(consumerConfig.getInstanceId(), Bytes.toInt(data));
}
}
});
}
}
Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 4, 5), dequeued.get(0));
}
Aggregations