Examples with ConsumerGroupConfig co.cask.cdap.data2.queue.ConsumerGroupConfig used on opensource projects

Example 16 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class QueueTest method testClearOrDropAllForFlow.

private void testClearOrDropAllForFlow(boolean doDrop) throws Exception {
    // this test is the same for clear and drop, except for two small places...
    // using a different app name for each case as this test leaves some entries
    String app = doDrop ? "tDAFF" : "tCAFF";
    QueueName queueName1 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow1", "flowlet1", "out1");
    QueueName queueName2 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow1", "flowlet2", "out2");
    QueueName queueName3 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow2", "flowlet1", "out");
    List<ConsumerGroupConfig> groupConfigs = ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null));
    configureGroups(queueName1, groupConfigs);
    configureGroups(queueName2, groupConfigs);
    configureGroups(queueName3, groupConfigs);
    try (QueueProducer producer1 = queueClientFactory.createProducer(queueName1);
        QueueProducer producer2 = queueClientFactory.createProducer(queueName2);
        QueueProducer producer3 = queueClientFactory.createProducer(queueName3)) {
        TransactionContext txContext = createTxContext(producer1, producer2, producer3);
        txContext.start();
        for (int i = 0; i < 10; i++) {
            for (QueueProducer producer : Arrays.asList(producer1, producer2, producer3)) {
                producer.enqueue(new QueueEntry(Bytes.toBytes(i)));
            }
        }
        txContext.finish();
    }
    // consume 1 element from each queue
    ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
    try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName1, consumerConfig, 1);
        QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName2, consumerConfig, 1);
        QueueConsumer consumer3 = queueClientFactory.createConsumer(queueName3, consumerConfig, 1)) {
        TransactionContext txContext = createTxContext(consumer1, consumer2, consumer3);
        txContext.start();
        for (QueueConsumer consumer : Arrays.asList(consumer1, consumer2, consumer3)) {
            DequeueResult<byte[]> result = consumer.dequeue(1);
            Assert.assertFalse(result.isEmpty());
            Assert.assertArrayEquals(Bytes.toBytes(0), result.iterator().next());
        }
        txContext.finish();
    }
    // verify the consumer config was deleted
    verifyConsumerConfigExists(queueName1, queueName2);
    // clear/drop all queues for flow1
    FlowId flow1Id = NamespaceId.DEFAULT.app(app).flow("flow1");
    if (doDrop) {
        queueAdmin.dropAllForFlow(flow1Id);
    } else {
        queueAdmin.clearAllForFlow(flow1Id);
    }
    if (doDrop) {
        // verify that only flow2's queues still exist
        Assert.assertFalse(queueAdmin.exists(queueName1));
        Assert.assertFalse(queueAdmin.exists(queueName2));
        Assert.assertTrue(queueAdmin.exists(queueName3));
    } else {
        // verify all queues still exist
        Assert.assertTrue(queueAdmin.exists(queueName1));
        Assert.assertTrue(queueAdmin.exists(queueName2));
        Assert.assertTrue(queueAdmin.exists(queueName3));
    }
    // verify the consumer config was deleted
    verifyConsumerConfigIsDeleted(queueName1, queueName2);
    // create new consumers because existing ones may have pre-fetched and cached some entries
    configureGroups(queueName1, groupConfigs);
    configureGroups(queueName2, groupConfigs);
    try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName1, consumerConfig, 1);
        QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName2, consumerConfig, 1);
        QueueConsumer consumer3 = queueClientFactory.createConsumer(queueName3, consumerConfig, 1)) {
        TransactionContext txContext = createTxContext(consumer1, consumer2, consumer3);
        txContext.start();
        // attempt to consume from flow1's queues, should be empty
        for (QueueConsumer consumer : Arrays.asList(consumer1, consumer2)) {
            DequeueResult<byte[]> result = consumer.dequeue(1);
            Assert.assertTrue(result.isEmpty());
        }
        // but flow2 was not deleted -> consumer 3 should get another entry
        DequeueResult<byte[]> result = consumer3.dequeue(1);
        Assert.assertFalse(result.isEmpty());
        Assert.assertArrayEquals(Bytes.toBytes(1), result.iterator().next());
        txContext.finish();
    }
}

Example 17 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class QueueTest method enqueueDequeue.

private void enqueueDequeue(final QueueName queueName, int preEnqueueCount, int concurrentCount, int enqueueBatchSize, int consumerSize, DequeueStrategy dequeueStrategy, final int dequeueBatchSize) throws Exception {
    ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(0L, consumerSize, dequeueStrategy, "key");
    configureGroups(queueName, ImmutableList.of(groupConfig));
    Preconditions.checkArgument(preEnqueueCount % enqueueBatchSize == 0, "Count must be divisible by enqueueBatchSize");
    Preconditions.checkArgument(concurrentCount % enqueueBatchSize == 0, "Count must be divisible by enqueueBatchSize");
    final List<ConsumerConfig> consumerConfigs = Lists.newArrayList();
    for (int i = 0; i < consumerSize; i++) {
        consumerConfigs.add(new ConsumerConfig(groupConfig, i));
    }
    createEnqueueRunnable(queueName, preEnqueueCount, enqueueBatchSize, null).run();
    final CyclicBarrier startBarrier = new CyclicBarrier(consumerSize + 2);
    ExecutorService executor = Executors.newFixedThreadPool(consumerSize + 1);
    // Enqueue thread
    executor.submit(createEnqueueRunnable(queueName, concurrentCount, enqueueBatchSize, startBarrier));
    // Dequeue
    final long expectedSum = ((long) preEnqueueCount / 2 * ((long) preEnqueueCount - 1)) + ((long) concurrentCount / 2 * ((long) concurrentCount - 1));
    final AtomicLong valueSum = new AtomicLong();
    final CountDownLatch completeLatch = new CountDownLatch(consumerSize);
    for (int i = 0; i < consumerSize; i++) {
        final int instanceId = i;
        executor.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    startBarrier.await();
                    LOG.info("Consumer {} starts consuming {}", instanceId, queueName.getSimpleName());
                    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(instanceId), 1)) {
                        TransactionContext txContext = createTxContext(consumer);
                        Stopwatch stopwatch = new Stopwatch();
                        stopwatch.start();
                        int dequeueCount = 0;
                        while (valueSum.get() < expectedSum) {
                            txContext.start();
                            try {
                                DequeueResult<byte[]> result = consumer.dequeue(dequeueBatchSize);
                                txContext.finish();
                                if (result.isEmpty()) {
                                    continue;
                                }
                                for (byte[] data : result) {
                                    valueSum.addAndGet(Bytes.toInt(data));
                                    dequeueCount++;
                                }
                            } catch (TransactionFailureException e) {
                                LOG.error("Operation error", e);
                                txContext.abort();
                                throw Throwables.propagate(e);
                            }
                        }
                        long elapsed = stopwatch.elapsedTime(TimeUnit.MILLISECONDS);
                        LOG.info("Dequeue {} entries in {} ms for {}", dequeueCount, elapsed, queueName.getSimpleName());
                        LOG.info("Dequeue avg {} entries per seconds for {}", (double) dequeueCount * 1000 / elapsed, queueName.getSimpleName());
                        consumer.close();
                        completeLatch.countDown();
                    }
                } catch (Exception e) {
                    LOG.error(e.getMessage(), e);
                }
            }
        });
    }
    startBarrier.await();
    completeLatch.await();
    Assert.assertEquals(expectedSum, valueSum.get());
    // Only check eviction for queue.
    if (!queueName.isStream()) {
        verifyQueueIsEmpty(queueName, consumerConfigs);
    }
    executor.shutdownNow();
}

Example 18 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class QueueTest method verifyQueueIsEmpty.

protected void verifyQueueIsEmpty(QueueName queueName, List<ConsumerConfig> consumerConfigs) throws Exception {
    // Verify the queue is empty
    Set<ConsumerGroupConfig> groupConfigs = Sets.newHashSet();
    for (ConsumerConfig consumerConfig : consumerConfigs) {
        try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, -1)) {
            groupConfigs.add(new ConsumerGroupConfig(consumerConfig));
            TransactionContext txContext = createTxContext(consumer);
            try {
                txContext.start();
                Assert.assertTrue(consumer.dequeue().isEmpty());
                txContext.finish();
            } catch (TransactionFailureException e) {
                txContext.abort();
                throw Throwables.propagate(e);
            }
        }
    }
    forceEviction(queueName, groupConfigs.size());
    long newGroupId = groupConfigs.size();
    groupConfigs.add(new ConsumerGroupConfig(newGroupId, 1, DequeueStrategy.FIFO, null));
    configureGroups(queueName, groupConfigs);
    // the queue has been consumed by n consumers. Use a consumerId greater than n to make sure it can dequeue.
    ConsumerConfig consumerConfig = new ConsumerConfig(newGroupId, 0, 1, DequeueStrategy.FIFO, null);
    resetConsumerState(queueName, consumerConfig);
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, -1)) {
        TransactionContext txContext = createTxContext(consumer);
        txContext.start();
        DequeueResult<byte[]> result = consumer.dequeue();
        if (!result.isEmpty()) {
            StringBuilder resultString = new StringBuilder();
            for (byte[] aResult : result) {
                if (resultString.length() > 0) {
                    resultString.append(", ");
                }
                resultString.append(Bytes.toInt(aResult));
            }
            LOG.info("Queue should be empty but returned result: {}, value = ", result, resultString);
        }
        Assert.assertTrue("Entire queue should be evicted after test but dequeue succeeds.", result.isEmpty());
        txContext.abort();
    }
}

Example 19 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class QueueTest method testReset.

@Test
public void testReset() throws Exception {
    // NOTE: using different name of the queue from other unit-tests because this test leaves entries
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queueReset");
    configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null), new ConsumerConfig(1, 0, 1, DequeueStrategy.FIFO, null));
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        TransactionContext txContext = createTxContext(producer);
        txContext.start();
        producer.enqueue(new QueueEntry(Bytes.toBytes(0)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(1)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(2)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(3)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(4)));
        txContext.finish();
        try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2)) {
            // Check that there's smth in the queue, but do not consume: abort tx after check
            txContext = createTxContext(consumer1);
            txContext.start();
            Assert.assertEquals(0, Bytes.toInt(consumer1.dequeue().iterator().next()));
            txContext.finish();
        }
        // Reset queues
        queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
        // we gonna need another one to check again to avoid caching side-affects
        configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
        try (QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
            txContext = createTxContext(consumer2);
            // Check again: should be nothing in the queue
            txContext.start();
            Assert.assertTrue(consumer2.dequeue().isEmpty());
            txContext.finish();
            // add another entry
            txContext = createTxContext(producer);
            txContext.start();
            producer.enqueue(new QueueEntry(Bytes.toBytes(5)));
            txContext.finish();
            txContext = createTxContext(consumer2);
            // Check again: consumer should see new entry
            txContext.start();
            Assert.assertEquals(5, Bytes.toInt(consumer2.dequeue().iterator().next()));
            txContext.finish();
        }
    }
}

Example 20 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class QueueTest method testMultiStageConsumer.

@Test
public void testMultiStageConsumer() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "multistage");
    ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(0L, 2, DequeueStrategy.HASH, "key");
    configureGroups(queueName, ImmutableList.of(groupConfig));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(groupConfig, 0), new ConsumerConfig(groupConfig, 1));
    // Enqueue 10 items
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        for (int i = 0; i < 10; i++) {
            TransactionContext txContext = createTxContext(producer);
            txContext.start();
            producer.enqueue(new QueueEntry("key", i, Bytes.toBytes(i)));
            txContext.finish();
        }
    }
    // Consumer all even entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 1)) {
        for (int i = 0; i < 5; i++) {
            TransactionContext txContext = createTxContext(consumer);
            txContext.start();
            DequeueResult<byte[]> result = consumer.dequeue();
            Assert.assertTrue(!result.isEmpty());
            Assert.assertEquals(i * 2, Bytes.toInt(result.iterator().next()));
            txContext.finish();
        }
    }
    // Consume 2 odd entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 1)) {
        TransactionContext txContext = createTxContext(consumer);
        txContext.start();
        DequeueResult<byte[]> result = consumer.dequeue(2);
        Assert.assertEquals(2, result.size());
        Iterator<byte[]> iter = result.iterator();
        for (int i = 0; i < 2; i++) {
            Assert.assertEquals(i * 2 + 1, Bytes.toInt(iter.next()));
        }
        txContext.finish();
    }
    // Consume the rest odd entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 1)) {
        for (int i = 2; i < 5; i++) {
            TransactionContext txContext = createTxContext(consumer);
            txContext.start();
            DequeueResult<byte[]> result = consumer.dequeue();
            Assert.assertTrue(!result.isEmpty());
            Assert.assertEquals(i * 2 + 1, Bytes.toInt(result.iterator().next()));
            txContext.finish();
        }
    }
}