Examples with QueueConsumer co.cask.cdap.data2.queue.QueueConsumer used on opensource projects

Example 11 with QueueConsumer

use of co.cask.cdap.data2.queue.QueueConsumer in project cdap by caskdata.

the class InMemoryStreamConsumerFactory method create.

@Override
public StreamConsumer create(StreamId streamId, String namespace, ConsumerConfig consumerConfig) throws IOException {
    QueueName queueName = QueueName.fromStream(streamId);
    QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, -1);
    return new QueueToStreamConsumer(streamId, consumerConfig, consumer);
}

Example 12 with QueueConsumer

use of co.cask.cdap.data2.queue.QueueConsumer in project cdap by caskdata.

the class QueueTest method verifyQueueIsEmpty.

protected void verifyQueueIsEmpty(QueueName queueName, List<ConsumerConfig> consumerConfigs) throws Exception {
    // Verify the queue is empty
    Set<ConsumerGroupConfig> groupConfigs = Sets.newHashSet();
    for (ConsumerConfig consumerConfig : consumerConfigs) {
        try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, -1)) {
            groupConfigs.add(new ConsumerGroupConfig(consumerConfig));
            TransactionContext txContext = createTxContext(consumer);
            try {
                txContext.start();
                Assert.assertTrue(consumer.dequeue().isEmpty());
                txContext.finish();
            } catch (TransactionFailureException e) {
                txContext.abort();
                throw Throwables.propagate(e);
            }
        }
    }
    forceEviction(queueName, groupConfigs.size());
    long newGroupId = groupConfigs.size();
    groupConfigs.add(new ConsumerGroupConfig(newGroupId, 1, DequeueStrategy.FIFO, null));
    configureGroups(queueName, groupConfigs);
    // the queue has been consumed by n consumers. Use a consumerId greater than n to make sure it can dequeue.
    ConsumerConfig consumerConfig = new ConsumerConfig(newGroupId, 0, 1, DequeueStrategy.FIFO, null);
    resetConsumerState(queueName, consumerConfig);
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, -1)) {
        TransactionContext txContext = createTxContext(consumer);
        txContext.start();
        DequeueResult<byte[]> result = consumer.dequeue();
        if (!result.isEmpty()) {
            StringBuilder resultString = new StringBuilder();
            for (byte[] aResult : result) {
                if (resultString.length() > 0) {
                    resultString.append(", ");
                }
                resultString.append(Bytes.toInt(aResult));
            }
            LOG.info("Queue should be empty but returned result: {}, value = ", result, resultString);
        }
        Assert.assertTrue("Entire queue should be evicted after test but dequeue succeeds.", result.isEmpty());
        txContext.abort();
    }
}

Example 13 with QueueConsumer

use of co.cask.cdap.data2.queue.QueueConsumer in project cdap by caskdata.

the class QueueTest method enqueueDequeue.

private void enqueueDequeue(final QueueName queueName, int preEnqueueCount, int concurrentCount, int enqueueBatchSize, int consumerSize, DequeueStrategy dequeueStrategy, final int dequeueBatchSize) throws Exception {
    ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(0L, consumerSize, dequeueStrategy, "key");
    configureGroups(queueName, ImmutableList.of(groupConfig));
    Preconditions.checkArgument(preEnqueueCount % enqueueBatchSize == 0, "Count must be divisible by enqueueBatchSize");
    Preconditions.checkArgument(concurrentCount % enqueueBatchSize == 0, "Count must be divisible by enqueueBatchSize");
    final List<ConsumerConfig> consumerConfigs = Lists.newArrayList();
    for (int i = 0; i < consumerSize; i++) {
        consumerConfigs.add(new ConsumerConfig(groupConfig, i));
    }
    createEnqueueRunnable(queueName, preEnqueueCount, enqueueBatchSize, null).run();
    final CyclicBarrier startBarrier = new CyclicBarrier(consumerSize + 2);
    ExecutorService executor = Executors.newFixedThreadPool(consumerSize + 1);
    // Enqueue thread
    executor.submit(createEnqueueRunnable(queueName, concurrentCount, enqueueBatchSize, startBarrier));
    // Dequeue
    final long expectedSum = ((long) preEnqueueCount / 2 * ((long) preEnqueueCount - 1)) + ((long) concurrentCount / 2 * ((long) concurrentCount - 1));
    final AtomicLong valueSum = new AtomicLong();
    final CountDownLatch completeLatch = new CountDownLatch(consumerSize);
    for (int i = 0; i < consumerSize; i++) {
        final int instanceId = i;
        executor.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    startBarrier.await();
                    LOG.info("Consumer {} starts consuming {}", instanceId, queueName.getSimpleName());
                    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(instanceId), 1)) {
                        TransactionContext txContext = createTxContext(consumer);
                        Stopwatch stopwatch = new Stopwatch();
                        stopwatch.start();
                        int dequeueCount = 0;
                        while (valueSum.get() < expectedSum) {
                            txContext.start();
                            try {
                                DequeueResult<byte[]> result = consumer.dequeue(dequeueBatchSize);
                                txContext.finish();
                                if (result.isEmpty()) {
                                    continue;
                                }
                                for (byte[] data : result) {
                                    valueSum.addAndGet(Bytes.toInt(data));
                                    dequeueCount++;
                                }
                            } catch (TransactionFailureException e) {
                                LOG.error("Operation error", e);
                                txContext.abort();
                                throw Throwables.propagate(e);
                            }
                        }
                        long elapsed = stopwatch.elapsedTime(TimeUnit.MILLISECONDS);
                        LOG.info("Dequeue {} entries in {} ms for {}", dequeueCount, elapsed, queueName.getSimpleName());
                        LOG.info("Dequeue avg {} entries per seconds for {}", (double) dequeueCount * 1000 / elapsed, queueName.getSimpleName());
                        consumer.close();
                        completeLatch.countDown();
                    }
                } catch (Exception e) {
                    LOG.error(e.getMessage(), e);
                }
            }
        });
    }
    startBarrier.await();
    completeLatch.await();
    Assert.assertEquals(expectedSum, valueSum.get());
    // Only check eviction for queue.
    if (!queueName.isStream()) {
        verifyQueueIsEmpty(queueName, consumerConfigs);
    }
    executor.shutdownNow();
}

Example 14 with QueueConsumer

use of co.cask.cdap.data2.queue.QueueConsumer in project cdap by caskdata.

the class QueueTest method testMultiStageConsumer.

@Test
public void testMultiStageConsumer() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "multistage");
    ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(0L, 2, DequeueStrategy.HASH, "key");
    configureGroups(queueName, ImmutableList.of(groupConfig));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(groupConfig, 0), new ConsumerConfig(groupConfig, 1));
    // Enqueue 10 items
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        for (int i = 0; i < 10; i++) {
            TransactionContext txContext = createTxContext(producer);
            txContext.start();
            producer.enqueue(new QueueEntry("key", i, Bytes.toBytes(i)));
            txContext.finish();
        }
    }
    // Consumer all even entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 1)) {
        for (int i = 0; i < 5; i++) {
            TransactionContext txContext = createTxContext(consumer);
            txContext.start();
            DequeueResult<byte[]> result = consumer.dequeue();
            Assert.assertTrue(!result.isEmpty());
            Assert.assertEquals(i * 2, Bytes.toInt(result.iterator().next()));
            txContext.finish();
        }
    }
    // Consume 2 odd entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 1)) {
        TransactionContext txContext = createTxContext(consumer);
        txContext.start();
        DequeueResult<byte[]> result = consumer.dequeue(2);
        Assert.assertEquals(2, result.size());
        Iterator<byte[]> iter = result.iterator();
        for (int i = 0; i < 2; i++) {
            Assert.assertEquals(i * 2 + 1, Bytes.toInt(iter.next()));
        }
        txContext.finish();
    }
    // Consume the rest odd entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 1)) {
        for (int i = 2; i < 5; i++) {
            TransactionContext txContext = createTxContext(consumer);
            txContext.start();
            DequeueResult<byte[]> result = consumer.dequeue();
            Assert.assertTrue(!result.isEmpty());
            Assert.assertEquals(i * 2 + 1, Bytes.toInt(result.iterator().next()));
            txContext.finish();
        }
    }
}

Example 15 with QueueConsumer

use of co.cask.cdap.data2.queue.QueueConsumer in project cdap by caskdata.

the class HBaseQueueTest method testQueueUpgrade.

// This test upgrade from old queue (salted base) to new queue (sharded base)
@Test(timeout = 30000L)
public void testQueueUpgrade() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "upgrade");
    HBaseQueueAdmin hbaseQueueAdmin = (HBaseQueueAdmin) queueAdmin;
    HBaseQueueClientFactory hBaseQueueClientFactory = (HBaseQueueClientFactory) queueClientFactory;
    // Create the old queue table explicitly
    HBaseQueueAdmin oldQueueAdmin = new HBaseQueueAdmin(hConf, cConf, injector.getInstance(LocationFactory.class), injector.getInstance(HBaseTableUtil.class), injector.getInstance(DatasetFramework.class), injector.getInstance(TransactionExecutorFactory.class), QueueConstants.QueueType.QUEUE, injector.getInstance(NamespaceQueryAdmin.class), injector.getInstance(Impersonator.class));
    oldQueueAdmin.create(queueName);
    int buckets = cConf.getInt(QueueConstants.ConfigKeys.QUEUE_TABLE_PRESPLITS);
    try (final HBaseQueueProducer oldProducer = hBaseQueueClientFactory.createProducer(oldQueueAdmin, queueName, QueueConstants.QueueType.QUEUE, QueueMetrics.NOOP_QUEUE_METRICS, new SaltedHBaseQueueStrategy(tableUtil, buckets), new ArrayList<ConsumerGroupConfig>())) {
        // Enqueue 10 items to old queue table
        Transactions.createTransactionExecutor(executorFactory, oldProducer).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                for (int i = 0; i < 10; i++) {
                    oldProducer.enqueue(new QueueEntry("key", i, Bytes.toBytes("Message " + i)));
                }
            }
        });
    }
    // Configure the consumer
    final ConsumerConfig consumerConfig = new ConsumerConfig(0L, 0, 1, DequeueStrategy.HASH, "key");
    try (QueueConfigurer configurer = queueAdmin.getQueueConfigurer(queueName)) {
        Transactions.createTransactionExecutor(executorFactory, configurer).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                configurer.configureGroups(ImmutableList.of(consumerConfig));
            }
        });
    }
    // explicit set the consumer state to be the lowest start row
    try (HBaseConsumerStateStore stateStore = hbaseQueueAdmin.getConsumerStateStore(queueName)) {
        Transactions.createTransactionExecutor(executorFactory, stateStore).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                stateStore.updateState(consumerConfig.getGroupId(), consumerConfig.getInstanceId(), QueueEntryRow.getQueueEntryRowKey(queueName, 0L, 0));
            }
        });
    }
    // Enqueue 10 more items to new queue table
    createEnqueueRunnable(queueName, 10, 1, null).run();
    // Verify both old and new table have 10 rows each
    Assert.assertEquals(10, countRows(hbaseQueueAdmin.getDataTableId(queueName, QueueConstants.QueueType.QUEUE)));
    Assert.assertEquals(10, countRows(hbaseQueueAdmin.getDataTableId(queueName, QueueConstants.QueueType.SHARDED_QUEUE)));
    // Create a consumer. It should see all 20 items
    final List<String> messages = Lists.newArrayList();
    try (final QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
        while (messages.size() != 20) {
            Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    DequeueResult<byte[]> result = consumer.dequeue(20);
                    for (byte[] data : result) {
                        messages.add(Bytes.toString(data));
                    }
                }
            });
        }
    }
    verifyQueueIsEmpty(queueName, ImmutableList.of(consumerConfig));
}