use of co.cask.cdap.data2.queue.QueueConsumer in project cdap by caskdata.
the class QueueTest method testClearOrDropAllForFlow.
private void testClearOrDropAllForFlow(boolean doDrop) throws Exception {
// this test is the same for clear and drop, except for two small places...
// using a different app name for each case as this test leaves some entries
String app = doDrop ? "tDAFF" : "tCAFF";
QueueName queueName1 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow1", "flowlet1", "out1");
QueueName queueName2 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow1", "flowlet2", "out2");
QueueName queueName3 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow2", "flowlet1", "out");
List<ConsumerGroupConfig> groupConfigs = ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null));
configureGroups(queueName1, groupConfigs);
configureGroups(queueName2, groupConfigs);
configureGroups(queueName3, groupConfigs);
try (QueueProducer producer1 = queueClientFactory.createProducer(queueName1);
QueueProducer producer2 = queueClientFactory.createProducer(queueName2);
QueueProducer producer3 = queueClientFactory.createProducer(queueName3)) {
TransactionContext txContext = createTxContext(producer1, producer2, producer3);
txContext.start();
for (int i = 0; i < 10; i++) {
for (QueueProducer producer : Arrays.asList(producer1, producer2, producer3)) {
producer.enqueue(new QueueEntry(Bytes.toBytes(i)));
}
}
txContext.finish();
}
// consume 1 element from each queue
ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName1, consumerConfig, 1);
QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName2, consumerConfig, 1);
QueueConsumer consumer3 = queueClientFactory.createConsumer(queueName3, consumerConfig, 1)) {
TransactionContext txContext = createTxContext(consumer1, consumer2, consumer3);
txContext.start();
for (QueueConsumer consumer : Arrays.asList(consumer1, consumer2, consumer3)) {
DequeueResult<byte[]> result = consumer.dequeue(1);
Assert.assertFalse(result.isEmpty());
Assert.assertArrayEquals(Bytes.toBytes(0), result.iterator().next());
}
txContext.finish();
}
// verify the consumer config was deleted
verifyConsumerConfigExists(queueName1, queueName2);
// clear/drop all queues for flow1
FlowId flow1Id = NamespaceId.DEFAULT.app(app).flow("flow1");
if (doDrop) {
queueAdmin.dropAllForFlow(flow1Id);
} else {
queueAdmin.clearAllForFlow(flow1Id);
}
if (doDrop) {
// verify that only flow2's queues still exist
Assert.assertFalse(queueAdmin.exists(queueName1));
Assert.assertFalse(queueAdmin.exists(queueName2));
Assert.assertTrue(queueAdmin.exists(queueName3));
} else {
// verify all queues still exist
Assert.assertTrue(queueAdmin.exists(queueName1));
Assert.assertTrue(queueAdmin.exists(queueName2));
Assert.assertTrue(queueAdmin.exists(queueName3));
}
// verify the consumer config was deleted
verifyConsumerConfigIsDeleted(queueName1, queueName2);
// create new consumers because existing ones may have pre-fetched and cached some entries
configureGroups(queueName1, groupConfigs);
configureGroups(queueName2, groupConfigs);
try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName1, consumerConfig, 1);
QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName2, consumerConfig, 1);
QueueConsumer consumer3 = queueClientFactory.createConsumer(queueName3, consumerConfig, 1)) {
TransactionContext txContext = createTxContext(consumer1, consumer2, consumer3);
txContext.start();
// attempt to consume from flow1's queues, should be empty
for (QueueConsumer consumer : Arrays.asList(consumer1, consumer2)) {
DequeueResult<byte[]> result = consumer.dequeue(1);
Assert.assertTrue(result.isEmpty());
}
// but flow2 was not deleted -> consumer 3 should get another entry
DequeueResult<byte[]> result = consumer3.dequeue(1);
Assert.assertFalse(result.isEmpty());
Assert.assertArrayEquals(Bytes.toBytes(1), result.iterator().next());
txContext.finish();
}
}
use of co.cask.cdap.data2.queue.QueueConsumer in project cdap by caskdata.
the class QueueTest method testReset.
@Test
public void testReset() throws Exception {
// NOTE: using different name of the queue from other unit-tests because this test leaves entries
QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queueReset");
configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null), new ConsumerConfig(1, 0, 1, DequeueStrategy.FIFO, null));
try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
TransactionContext txContext = createTxContext(producer);
txContext.start();
producer.enqueue(new QueueEntry(Bytes.toBytes(0)));
producer.enqueue(new QueueEntry(Bytes.toBytes(1)));
producer.enqueue(new QueueEntry(Bytes.toBytes(2)));
producer.enqueue(new QueueEntry(Bytes.toBytes(3)));
producer.enqueue(new QueueEntry(Bytes.toBytes(4)));
txContext.finish();
try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2)) {
// Check that there's smth in the queue, but do not consume: abort tx after check
txContext = createTxContext(consumer1);
txContext.start();
Assert.assertEquals(0, Bytes.toInt(consumer1.dequeue().iterator().next()));
txContext.finish();
}
// Reset queues
queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
// we gonna need another one to check again to avoid caching side-affects
configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
try (QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
txContext = createTxContext(consumer2);
// Check again: should be nothing in the queue
txContext.start();
Assert.assertTrue(consumer2.dequeue().isEmpty());
txContext.finish();
// add another entry
txContext = createTxContext(producer);
txContext.start();
producer.enqueue(new QueueEntry(Bytes.toBytes(5)));
txContext.finish();
txContext = createTxContext(consumer2);
// Check again: consumer should see new entry
txContext.start();
Assert.assertEquals(5, Bytes.toInt(consumer2.dequeue().iterator().next()));
txContext.finish();
}
}
}
use of co.cask.cdap.data2.queue.QueueConsumer in project cdap by caskdata.
the class QueueTest method testDropAllQueues.
@Test
public void testDropAllQueues() throws Exception {
// create a queue and a stream and enqueue one entry each
QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "myApp", "myFlow", "myFlowlet", "tDAQ");
ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
configureGroups(queueName, ImmutableList.of(consumerConfig));
try (final QueueProducer qProducer = queueClientFactory.createProducer(queueName)) {
executorFactory.createExecutor(Lists.newArrayList((TransactionAware) qProducer)).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
qProducer.enqueue(new QueueEntry(Bytes.toBytes("q42")));
}
});
// drop all queues
queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
// verify that queue is gone and stream is still there
configureGroups(queueName, ImmutableList.of(consumerConfig));
try (final QueueConsumer qConsumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
executorFactory.createExecutor(Lists.newArrayList((TransactionAware) qConsumer)).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
DequeueResult<byte[]> dequeue = qConsumer.dequeue();
Assert.assertTrue(dequeue.isEmpty());
}
});
}
}
}
use of co.cask.cdap.data2.queue.QueueConsumer in project cdap by caskdata.
the class ProgramLifecycleHttpHandlerTest method dequeueOne.
private boolean dequeueOne(QueueName queueName) throws Exception {
QueueClientFactory queueClientFactory = AppFabricTestBase.getInjector().getInstance(QueueClientFactory.class);
final QueueConsumer consumer = queueClientFactory.createConsumer(queueName, new ConsumerConfig(1L, 0, 1, DequeueStrategy.ROUND_ROBIN, null), 1);
// doing inside tx
TransactionExecutorFactory txExecutorFactory = AppFabricTestBase.getInjector().getInstance(TransactionExecutorFactory.class);
return txExecutorFactory.createExecutor(ImmutableList.of((TransactionAware) consumer)).execute(new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return !consumer.dequeue(1).isEmpty();
}
});
}
Aggregations