Examples with QueueProducer co.cask.cdap.data2.queue.QueueProducer used on opensource projects

Example 11 with QueueProducer

use of co.cask.cdap.data2.queue.QueueProducer in project cdap by caskdata.

the class QueueTest method testReset.

@Test
public void testReset() throws Exception {
    // NOTE: using different name of the queue from other unit-tests because this test leaves entries
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queueReset");
    configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null), new ConsumerConfig(1, 0, 1, DequeueStrategy.FIFO, null));
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        TransactionContext txContext = createTxContext(producer);
        txContext.start();
        producer.enqueue(new QueueEntry(Bytes.toBytes(0)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(1)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(2)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(3)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(4)));
        txContext.finish();
        try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2)) {
            // Check that there's smth in the queue, but do not consume: abort tx after check
            txContext = createTxContext(consumer1);
            txContext.start();
            Assert.assertEquals(0, Bytes.toInt(consumer1.dequeue().iterator().next()));
            txContext.finish();
        }
        // Reset queues
        queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
        // we gonna need another one to check again to avoid caching side-affects
        configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
        try (QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
            txContext = createTxContext(consumer2);
            // Check again: should be nothing in the queue
            txContext.start();
            Assert.assertTrue(consumer2.dequeue().isEmpty());
            txContext.finish();
            // add another entry
            txContext = createTxContext(producer);
            txContext.start();
            producer.enqueue(new QueueEntry(Bytes.toBytes(5)));
            txContext.finish();
            txContext = createTxContext(consumer2);
            // Check again: consumer should see new entry
            txContext.start();
            Assert.assertEquals(5, Bytes.toInt(consumer2.dequeue().iterator().next()));
            txContext.finish();
        }
    }
}

Example 12 with QueueProducer

use of co.cask.cdap.data2.queue.QueueProducer in project cdap by caskdata.

the class QueueTest method testDropAllQueues.

@Test
public void testDropAllQueues() throws Exception {
    // create a queue and a stream and enqueue one entry each
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "myApp", "myFlow", "myFlowlet", "tDAQ");
    ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
    configureGroups(queueName, ImmutableList.of(consumerConfig));
    try (final QueueProducer qProducer = queueClientFactory.createProducer(queueName)) {
        executorFactory.createExecutor(Lists.newArrayList((TransactionAware) qProducer)).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                qProducer.enqueue(new QueueEntry(Bytes.toBytes("q42")));
            }
        });
        // drop all queues
        queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
        // verify that queue is gone and stream is still there
        configureGroups(queueName, ImmutableList.of(consumerConfig));
        try (final QueueConsumer qConsumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
            executorFactory.createExecutor(Lists.newArrayList((TransactionAware) qConsumer)).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    DequeueResult<byte[]> dequeue = qConsumer.dequeue();
                    Assert.assertTrue(dequeue.isEmpty());
                }
            });
        }
    }
}

Example 13 with QueueProducer

use of co.cask.cdap.data2.queue.QueueProducer in project cdap by caskdata.

the class ProgramLifecycleHttpHandlerTest method enqueue.

// TODO: Duplicate from AppFabricHttpHandlerTest, remove the AppFabricHttpHandlerTest method after deprecating v2 APIs
private void enqueue(QueueName queueName, final QueueEntry queueEntry) throws Exception {
    QueueClientFactory queueClientFactory = AppFabricTestBase.getInjector().getInstance(QueueClientFactory.class);
    final QueueProducer producer = queueClientFactory.createProducer(queueName);
    // doing inside tx
    TransactionExecutorFactory txExecutorFactory = AppFabricTestBase.getInjector().getInstance(TransactionExecutorFactory.class);
    txExecutorFactory.createExecutor(ImmutableList.of((TransactionAware) producer)).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // write more than one so that we can dequeue multiple times for multiple checks
            // we only dequeue twice, but ensure that the drop queues call drops the rest of the entries as well
            int numEntries = 0;
            while (numEntries++ < 5) {
                producer.enqueue(queueEntry);
            }
        }
    });
}

Example 14 with QueueProducer

use of co.cask.cdap.data2.queue.QueueProducer in project cdap by caskdata.

the class HBaseQueueClientFactory method createProducer.

@Override
public QueueProducer createProducer(QueueName queueName, QueueMetrics queueMetrics) throws IOException {
    HBaseQueueAdmin admin = ensureTableExists(queueName);
    try {
        final List<ConsumerGroupConfig> groupConfigs = Lists.newArrayList();
        try (HBaseConsumerStateStore stateStore = admin.getConsumerStateStore(queueName)) {
            Transactions.createTransactionExecutor(txExecutorFactory, stateStore).execute(new Subroutine() {

                @Override
                public void apply() throws Exception {
                    stateStore.getLatestConsumerGroups(groupConfigs);
                }
            });
        }
        Preconditions.checkState(!groupConfigs.isEmpty(), "Missing consumer group information for queue %s", queueName);
        HTable hTable = createHTable(admin.getDataTableId(queueName, queueAdmin.getType()));
        int distributorBuckets = getDistributorBuckets(hTable.getTableDescriptor());
        return createProducer(hTable, queueName, queueMetrics, new ShardedHBaseQueueStrategy(hBaseTableUtil, distributorBuckets), groupConfigs);
    } catch (Exception e) {
        Throwables.propagateIfPossible(e);
        throw new IOException(e);
    }
}