Search in sources :

Example 6 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class QueueTest method testQueueAbortRetrySkip.

@Test(timeout = TIMEOUT_MS)
public void testQueueAbortRetrySkip() throws Exception {
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queuefailure");
    configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.HASH, "key")));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null), new ConsumerConfig(1, 0, 1, DequeueStrategy.HASH, "key"));
    createEnqueueRunnable(queueName, 5, 1, null).run();
    try (QueueConsumer fifoConsumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2);
        QueueConsumer hashConsumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
        TransactionContext txContext = createTxContext(fifoConsumer, hashConsumer);
        txContext.start();
        Assert.assertEquals(0, Bytes.toInt(fifoConsumer.dequeue().iterator().next()));
        Assert.assertEquals(0, Bytes.toInt(hashConsumer.dequeue().iterator().next()));
        // Abort the consumer transaction
        txContext.abort();
        // Dequeue again in a new transaction, should see the same entries
        txContext.start();
        Assert.assertEquals(0, Bytes.toInt(fifoConsumer.dequeue().iterator().next()));
        Assert.assertEquals(0, Bytes.toInt(hashConsumer.dequeue().iterator().next()));
        txContext.finish();
        // Dequeue again, now should get next entry
        txContext.start();
        Assert.assertEquals(1, Bytes.toInt(fifoConsumer.dequeue().iterator().next()));
        Assert.assertEquals(1, Bytes.toInt(hashConsumer.dequeue().iterator().next()));
        txContext.finish();
        // Dequeue a result and abort.
        txContext.start();
        DequeueResult<byte[]> fifoResult = fifoConsumer.dequeue();
        DequeueResult<byte[]> hashResult = hashConsumer.dequeue();
        Assert.assertEquals(2, Bytes.toInt(fifoResult.iterator().next()));
        Assert.assertEquals(2, Bytes.toInt(hashResult.iterator().next()));
        txContext.abort();
        // Now skip the result with a new transaction.
        txContext.start();
        fifoResult.reclaim();
        hashResult.reclaim();
        txContext.finish();
        // Dequeue again, it should see a new entry
        txContext.start();
        Assert.assertEquals(3, Bytes.toInt(fifoConsumer.dequeue().iterator().next()));
        Assert.assertEquals(3, Bytes.toInt(hashConsumer.dequeue().iterator().next()));
        txContext.finish();
        // Dequeue again, it should see a new entry
        txContext.start();
        Assert.assertEquals(4, Bytes.toInt(fifoConsumer.dequeue().iterator().next()));
        Assert.assertEquals(4, Bytes.toInt(hashConsumer.dequeue().iterator().next()));
        txContext.finish();
    }
    verifyQueueIsEmpty(queueName, consumerConfigs);
}
Also used : QueueConsumer(co.cask.cdap.data2.queue.QueueConsumer) TransactionContext(org.apache.tephra.TransactionContext) ConsumerConfig(co.cask.cdap.data2.queue.ConsumerConfig) QueueName(co.cask.cdap.common.queue.QueueName) ConsumerGroupConfig(co.cask.cdap.data2.queue.ConsumerGroupConfig) Test(org.junit.Test)

Example 7 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class QueueTest method testConcurrentEnqueue.

@Category(SlowTests.class)
@Test
public void testConcurrentEnqueue() throws Exception {
    // This test is for testing multiple producers that writes with a delay after a transaction started.
    // This is for verifying consumer advances the startKey correctly.
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "concurrent");
    configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0, 1, DequeueStrategy.FIFO, null)));
    final CyclicBarrier barrier = new CyclicBarrier(4);
    ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
    // Starts three producers to enqueue concurrently. For each entry, starts a TX, sleep, enqueue, commit.
    ExecutorService executor = Executors.newFixedThreadPool(3);
    final int entryCount = 50;
    for (int i = 0; i < 3; i++) {
        final QueueProducer producer = queueClientFactory.createProducer(queueName);
        final int producerId = i + 1;
        executor.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    barrier.await();
                    for (int i = 0; i < entryCount; i++) {
                        TransactionContext txContext = createTxContext(producer);
                        txContext.start();
                        // Sleeps at different rate to make the scan in consumer has higher change to see
                        // the transaction but not the entry (as not yet written)
                        TimeUnit.MILLISECONDS.sleep(producerId * 50);
                        producer.enqueue(new QueueEntry(Bytes.toBytes(i)));
                        txContext.finish();
                    }
                } catch (Exception e) {
                    LOG.error(e.getMessage(), e);
                } finally {
                    Closeables.closeQuietly(producer);
                }
            }
        });
    }
    // sum(0..entryCount) * 3
    int expectedSum = entryCount * (entryCount - 1) / 2 * 3;
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
        // Trigger starts of producer
        barrier.await();
        int dequeueSum = 0;
        int noProgress = 0;
        while (dequeueSum != expectedSum && noProgress < 200) {
            TransactionContext txContext = createTxContext(consumer);
            txContext.start();
            DequeueResult<byte[]> result = consumer.dequeue();
            if (!result.isEmpty()) {
                noProgress = 0;
                int value = Bytes.toInt(result.iterator().next());
                dequeueSum += value;
            } else {
                noProgress++;
                TimeUnit.MILLISECONDS.sleep(10);
            }
            txContext.finish();
        }
        Assert.assertEquals(expectedSum, dequeueSum);
    }
}
Also used : QueueEntry(co.cask.cdap.data2.queue.QueueEntry) TransactionFailureException(org.apache.tephra.TransactionFailureException) CyclicBarrier(java.util.concurrent.CyclicBarrier) QueueConsumer(co.cask.cdap.data2.queue.QueueConsumer) QueueProducer(co.cask.cdap.data2.queue.QueueProducer) TransactionContext(org.apache.tephra.TransactionContext) ExecutorService(java.util.concurrent.ExecutorService) ConsumerConfig(co.cask.cdap.data2.queue.ConsumerConfig) QueueName(co.cask.cdap.common.queue.QueueName) ConsumerGroupConfig(co.cask.cdap.data2.queue.ConsumerGroupConfig) Category(org.junit.experimental.categories.Category) Test(org.junit.Test)

Example 8 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class QueueTest method testDropAllForNamespace.

@Test
public void testDropAllForNamespace() throws Exception {
    // create 4 queues
    QueueName myQueue1 = QueueName.fromFlowlet(NAMESPACE_ID.getEntityName(), "myapp1", "myflow1", "myflowlet1", "myout1");
    QueueName myQueue2 = QueueName.fromFlowlet(NAMESPACE_ID.getEntityName(), "myapp2", "myflow2", "myflowlet2", "myout2");
    QueueName yourQueue1 = QueueName.fromFlowlet(NAMESPACE_ID1.getEntityName(), "yourapp1", "yourflow1", "yourflowlet1", "yourout1");
    QueueName yourQueue2 = QueueName.fromFlowlet(NAMESPACE_ID1.getEntityName(), "yourapp2", "yourflow2", "yourflowlet2", "yourout2");
    queueAdmin.create(myQueue1);
    queueAdmin.create(myQueue2);
    queueAdmin.create(yourQueue1);
    queueAdmin.create(yourQueue2);
    // verify that queues got created
    Assert.assertTrue(queueAdmin.exists(myQueue1) && queueAdmin.exists(myQueue2) && queueAdmin.exists(yourQueue1) && queueAdmin.exists(yourQueue2));
    // create some consumer configurations for all queues
    List<ConsumerGroupConfig> groupConfigs = ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null));
    configureGroups(myQueue1, groupConfigs);
    configureGroups(myQueue2, groupConfigs);
    configureGroups(yourQueue1, groupConfigs);
    configureGroups(yourQueue2, groupConfigs);
    // verify that the consumer config exists
    verifyConsumerConfigExists(myQueue1, myQueue2, yourQueue1, yourQueue2);
    // drop queues in NAMESPACE_ID
    queueAdmin.dropAllInNamespace(NAMESPACE_ID);
    // verify queues in NAMESPACE_ID are dropped
    Assert.assertFalse(queueAdmin.exists(myQueue1) || queueAdmin.exists(myQueue2));
    // also verify that consumer config of all queues in 'myspace' is deleted
    verifyConsumerConfigIsDeleted(myQueue1, myQueue2);
    // but the ones in NAMESPACE_ID1 still exist
    Assert.assertTrue(queueAdmin.exists(yourQueue1) && queueAdmin.exists(yourQueue2));
    // consumer config for queues in 'namespace1' should also still exist
    verifyConsumerConfigExists(yourQueue1, yourQueue2);
    // drop queues in NAMESPACE_ID1
    queueAdmin.dropAllInNamespace(NAMESPACE_ID1);
    // verify queues in NAMESPACE_ID are dropped
    Assert.assertFalse(queueAdmin.exists(yourQueue1) || queueAdmin.exists(yourQueue2));
    // verify that the consumer config of all queues in 'namespace1' is deleted
    verifyConsumerConfigIsDeleted(yourQueue1, yourQueue2);
}
Also used : QueueName(co.cask.cdap.common.queue.QueueName) ConsumerGroupConfig(co.cask.cdap.data2.queue.ConsumerGroupConfig) Test(org.junit.Test)

Example 9 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class QueueTest method testOneEnqueueDequeue.

private void testOneEnqueueDequeue(DequeueStrategy strategy) throws Exception {
    // since this is used by more than one test method, ensure uniqueness of the queue name by adding strategy
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queue1" + strategy.toString());
    configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, strategy, null), new ConsumerGroupConfig(1L, 1, strategy, null)));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0L, 0, 1, strategy, null), new ConsumerConfig(1L, 0, 1, strategy, null));
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        TransactionContext txContext = createTxContext(producer);
        txContext.start();
        producer.enqueue(new QueueEntry(Bytes.toBytes(55)));
        txContext.finish();
        try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2)) {
            txContext = createTxContext(consumer);
            txContext.start();
            Assert.assertEquals(55, Bytes.toInt(consumer.dequeue().iterator().next()));
            txContext.finish();
        }
    }
    forceEviction(queueName, 2);
    // verifying that consumer of the 2nd group can process items: they were not evicted
    try (QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
        TransactionContext txContext = createTxContext(consumer2);
        txContext.start();
        Assert.assertEquals(55, Bytes.toInt(consumer2.dequeue().iterator().next()));
        txContext.finish();
    }
    // now all should be evicted
    verifyQueueIsEmpty(queueName, consumerConfigs);
}
Also used : QueueConsumer(co.cask.cdap.data2.queue.QueueConsumer) QueueProducer(co.cask.cdap.data2.queue.QueueProducer) TransactionContext(org.apache.tephra.TransactionContext) ConsumerConfig(co.cask.cdap.data2.queue.ConsumerConfig) QueueName(co.cask.cdap.common.queue.QueueName) ConsumerGroupConfig(co.cask.cdap.data2.queue.ConsumerGroupConfig) QueueEntry(co.cask.cdap.data2.queue.QueueEntry)

Example 10 with ConsumerGroupConfig

use of co.cask.cdap.data2.queue.ConsumerGroupConfig in project cdap by caskdata.

the class HBaseQueueTest method testReconfigure.

@Test(timeout = 30000L)
public void testReconfigure() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "changeinstances");
    ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(0L, 2, DequeueStrategy.HASH, "key");
    configureGroups(queueName, ImmutableList.of(groupConfig));
    // Enqueue 10 items
    createEnqueueRunnable(queueName, 10, 1, null).run();
    // Map from instance id to items dequeued
    final Multimap<Integer, Integer> dequeued = ArrayListMultimap.create();
    // Consume 2 items for each consumer instances
    for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
        final ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
        try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
            Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    DequeueResult<byte[]> result = consumer.dequeue(2);
                    Assert.assertEquals(2, result.size());
                    for (byte[] data : result) {
                        dequeued.put(consumerConfig.getInstanceId(), Bytes.toInt(data));
                    }
                }
            });
        }
    }
    // Increase number of instances to 3
    changeInstances(queueName, 0L, 3);
    // Enqueue 10 more items
    createEnqueueRunnable(queueName, 10, 1, null).run();
    groupConfig = new ConsumerGroupConfig(0L, 3, DequeueStrategy.HASH, "key");
    // Dequeue everything
    while (dequeued.size() != 20) {
        for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
            final ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
            try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
                Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                    @Override
                    public void apply() throws Exception {
                        for (byte[] data : consumer.dequeue(20)) {
                            dequeued.put(consumerConfig.getInstanceId(), Bytes.toInt(data));
                        }
                    }
                });
            }
        }
    }
    // Instance 0 should see all evens before change instances
    Assert.assertEquals(ImmutableList.of(0, 2, 4, 6, 8, 0, 3, 6, 9), dequeued.get(0));
    // Instance 1 should see all odds before change instances
    Assert.assertEquals(ImmutableList.of(1, 3, 5, 7, 9, 1, 4, 7), dequeued.get(1));
    // Instance 2 should only see entries after change instances
    Assert.assertEquals(ImmutableList.of(2, 5, 8), dequeued.get(2));
    // All consumers should have empty dequeue now
    for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
        final ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
        try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
            Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    DequeueResult<byte[]> result = consumer.dequeue(20);
                    Assert.assertTrue(result.isEmpty());
                }
            });
        }
    }
    // Enqueue 6 more items for the 3 instances
    createEnqueueRunnable(queueName, 6, 1, null).run();
    // Reduce to 1 consumer
    changeInstances(queueName, 0L, 1);
    // The consumer 0 should be able to consume all 10 new items
    dequeued.clear();
    final ConsumerConfig consumerConfig = new ConsumerConfig(0L, 0, 1, DequeueStrategy.HASH, "key");
    try (final QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
        while (dequeued.size() != 6) {
            Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    for (byte[] data : consumer.dequeue(1)) {
                        dequeued.put(consumerConfig.getInstanceId(), Bytes.toInt(data));
                    }
                }
            });
        }
    }
    Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 4, 5), dequeued.get(0));
}
Also used : TransactionExecutor(org.apache.tephra.TransactionExecutor) IOException(java.io.IOException) TableNotFoundException(org.apache.hadoop.hbase.TableNotFoundException) QueueConsumer(co.cask.cdap.data2.queue.QueueConsumer) DequeueResult(co.cask.cdap.data2.queue.DequeueResult) TransactionAware(org.apache.tephra.TransactionAware) ConsumerConfig(co.cask.cdap.data2.queue.ConsumerConfig) QueueName(co.cask.cdap.common.queue.QueueName) ConsumerGroupConfig(co.cask.cdap.data2.queue.ConsumerGroupConfig) Test(org.junit.Test) QueueTest(co.cask.cdap.data2.transaction.queue.QueueTest)

Aggregations

ConsumerGroupConfig (co.cask.cdap.data2.queue.ConsumerGroupConfig)23 ConsumerConfig (co.cask.cdap.data2.queue.ConsumerConfig)12 QueueName (co.cask.cdap.common.queue.QueueName)11 QueueConsumer (co.cask.cdap.data2.queue.QueueConsumer)10 TransactionContext (org.apache.tephra.TransactionContext)8 Test (org.junit.Test)8 QueueEntry (co.cask.cdap.data2.queue.QueueEntry)6 QueueProducer (co.cask.cdap.data2.queue.QueueProducer)5 IOException (java.io.IOException)5 Map (java.util.Map)5 TransactionExecutor (org.apache.tephra.TransactionExecutor)5 TransactionFailureException (org.apache.tephra.TransactionFailureException)4 DequeueResult (co.cask.cdap.data2.queue.DequeueResult)3 QueueTest (co.cask.cdap.data2.transaction.queue.QueueTest)3 TableNotFoundException (org.apache.hadoop.hbase.TableNotFoundException)3 Put (co.cask.cdap.api.dataset.table.Put)2 Row (co.cask.cdap.api.dataset.table.Row)2 Scanner (co.cask.cdap.api.dataset.table.Scanner)2 FlowletDefinition (co.cask.cdap.api.flow.FlowletDefinition)2 QueueSpecification (co.cask.cdap.app.queue.QueueSpecification)2