Examples with QueueName co.cask.cdap.common.queue.QueueName used on opensource projects

Example 6 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class HBaseQueueTest method testReconfigure.

@Test(timeout = 30000L)
public void testReconfigure() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "changeinstances");
    ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(0L, 2, DequeueStrategy.HASH, "key");
    configureGroups(queueName, ImmutableList.of(groupConfig));
    // Enqueue 10 items
    createEnqueueRunnable(queueName, 10, 1, null).run();
    // Map from instance id to items dequeued
    final Multimap<Integer, Integer> dequeued = ArrayListMultimap.create();
    // Consume 2 items for each consumer instances
    for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
        final ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
        try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
            Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    DequeueResult<byte[]> result = consumer.dequeue(2);
                    Assert.assertEquals(2, result.size());
                    for (byte[] data : result) {
                        dequeued.put(consumerConfig.getInstanceId(), Bytes.toInt(data));
                    }
                }
            });
        }
    }
    // Increase number of instances to 3
    changeInstances(queueName, 0L, 3);
    // Enqueue 10 more items
    createEnqueueRunnable(queueName, 10, 1, null).run();
    groupConfig = new ConsumerGroupConfig(0L, 3, DequeueStrategy.HASH, "key");
    // Dequeue everything
    while (dequeued.size() != 20) {
        for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
            final ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
            try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
                Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                    @Override
                    public void apply() throws Exception {
                        for (byte[] data : consumer.dequeue(20)) {
                            dequeued.put(consumerConfig.getInstanceId(), Bytes.toInt(data));
                        }
                    }
                });
            }
        }
    }
    // Instance 0 should see all evens before change instances
    Assert.assertEquals(ImmutableList.of(0, 2, 4, 6, 8, 0, 3, 6, 9), dequeued.get(0));
    // Instance 1 should see all odds before change instances
    Assert.assertEquals(ImmutableList.of(1, 3, 5, 7, 9, 1, 4, 7), dequeued.get(1));
    // Instance 2 should only see entries after change instances
    Assert.assertEquals(ImmutableList.of(2, 5, 8), dequeued.get(2));
    // All consumers should have empty dequeue now
    for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
        final ConsumerConfig consumerConfig = new ConsumerConfig(groupConfig, instanceId);
        try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
            Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    DequeueResult<byte[]> result = consumer.dequeue(20);
                    Assert.assertTrue(result.isEmpty());
                }
            });
        }
    }
    // Enqueue 6 more items for the 3 instances
    createEnqueueRunnable(queueName, 6, 1, null).run();
    // Reduce to 1 consumer
    changeInstances(queueName, 0L, 1);
    // The consumer 0 should be able to consume all 10 new items
    dequeued.clear();
    final ConsumerConfig consumerConfig = new ConsumerConfig(0L, 0, 1, DequeueStrategy.HASH, "key");
    try (final QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
        while (dequeued.size() != 6) {
            Transactions.createTransactionExecutor(executorFactory, (TransactionAware) consumer).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    for (byte[] data : consumer.dequeue(1)) {
                        dequeued.put(consumerConfig.getInstanceId(), Bytes.toInt(data));
                    }
                }
            });
        }
    }
    Assert.assertEquals(ImmutableList.of(0, 1, 2, 3, 4, 5), dequeued.get(0));
}

Example 7 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class HBaseQueueTest method testQueueTableNameFormat.

// TODO: CDAP-1177 Should move to QueueTest after making getApplicationName() etc instance methods in a base class
@Test
public void testQueueTableNameFormat() throws Exception {
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "application1", "flow1", "flowlet1", "output1");
    HBaseQueueAdmin hbaseQueueAdmin = (HBaseQueueAdmin) queueAdmin;
    TableId tableId = hbaseQueueAdmin.getDataTableId(queueName);
    Assert.assertEquals(NamespaceId.DEFAULT.getEntityName(), tableId.getNamespace());
    Assert.assertEquals("system." + hbaseQueueAdmin.getType() + ".application1.flow1", tableId.getTableName());
    String tableName = tableUtil.buildHTableDescriptor(tableId).build().getNameAsString();
    Assert.assertEquals("application1", HBaseQueueAdmin.getApplicationName(tableName));
    Assert.assertEquals("flow1", HBaseQueueAdmin.getFlowName(tableName));
    queueName = QueueName.fromFlowlet("testNamespace", "application1", "flow1", "flowlet1", "output1");
    tableId = hbaseQueueAdmin.getDataTableId(queueName);
    Assert.assertEquals(String.format("%s_testNamespace", TABLE_PREFIX), tableId.getNamespace());
    Assert.assertEquals("system." + hbaseQueueAdmin.getType() + ".application1.flow1", tableId.getTableName());
    tableName = tableUtil.buildHTableDescriptor(tableId).build().getNameAsString();
    Assert.assertEquals("application1", HBaseQueueAdmin.getApplicationName(tableName));
    Assert.assertEquals("flow1", HBaseQueueAdmin.getFlowName(tableName));
}

Example 8 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class HBaseQueueTest method configTest.

@Test
public void configTest() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "configure");
    queueAdmin.create(queueName);
    final List<ConsumerGroupConfig> groupConfigs = ImmutableList.of(new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(2L, 2, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(3L, 3, DequeueStrategy.FIFO, null));
    try (HBaseConsumerStateStore stateStore = ((HBaseQueueAdmin) queueAdmin).getConsumerStateStore(queueName)) {
        TransactionExecutor txExecutor = Transactions.createTransactionExecutor(executorFactory, stateStore);
        // Intentionally set a row state for group 2, instance 0. It's for testing upgrade of config.
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                stateStore.updateState(2L, 0, QueueEntryRow.getQueueEntryRowKey(queueName, 10L, 0));
            }
        });
        // Set the group info
        configureGroups(queueName, groupConfigs);
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                for (ConsumerGroupConfig groupConfig : groupConfigs) {
                    long groupId = groupConfig.getGroupId();
                    List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(groupId);
                    Assert.assertEquals(1, queueBarriers.size());
                    for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
                        HBaseConsumerState state = stateStore.getState(groupId, instanceId);
                        if (groupId == 2L && instanceId == 0) {
                            // For group 2L instance 0, the start row shouldn't be changed.
                            // End row should be the same as the first barrier
                            Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), QueueEntryRow.getQueueEntryRowKey(queueName, 10L, 0)));
                            Assert.assertEquals(0, Bytes.compareTo(state.getNextBarrier(), queueBarriers.get(0).getStartRow()));
                        } else {
                            // For other group, they should have the start row the same as the first barrier info
                            Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(0).getStartRow()));
                        }
                    }
                }
            }
        });
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // Check consumers are all processed up to the barrier boundary
                for (long groupId = 1L; groupId <= 3L; groupId++) {
                    List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(groupId);
                    boolean allConsumed = stateStore.isAllConsumed(groupId, queueBarriers.get(0).getStartRow());
                    // For group 2, instance 0 is not consumed up to the boundary yet
                    Assert.assertTrue((groupId == 2L) != allConsumed);
                    if (groupId == 2L) {
                        // Mark group 2, instance 0 as completed the barrier.
                        stateStore.completed(groupId, 0);
                    }
                }
            }
        });
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // After group 2, instance 0 completed the current barrier, all consumers in group 2 should be able to
                // proceed
                List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(2L);
                byte[] startRow = stateStore.getState(2L, 0).getStartRow();
                Assert.assertEquals(0, Bytes.compareTo(startRow, queueBarriers.get(0).getStartRow()));
                Assert.assertTrue(stateStore.isAllConsumed(2L, startRow));
            }
        });
        // Add instance to group 2
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                stateStore.configureInstances(2L, 3);
            }
        });
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(2L);
                Assert.assertEquals(2, queueBarriers.size());
                // For existing instances, the start row shouldn't changed.
                for (int instanceId = 0; instanceId < 2; instanceId++) {
                    HBaseConsumerState state = stateStore.getState(2L, instanceId);
                    Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(0).getStartRow()));
                    Assert.assertEquals(0, Bytes.compareTo(state.getNextBarrier(), queueBarriers.get(1).getStartRow()));
                    // Complete the existing instance
                    stateStore.completed(2L, instanceId);
                }
                // For new instances, the start row should be the same as the new barrier
                HBaseConsumerState state = stateStore.getState(2L, 2);
                Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(1).getStartRow()));
                Assert.assertNull(state.getNextBarrier());
                // All instances should be consumed up to the beginning of the last barrier info
                Assert.assertTrue(stateStore.isAllConsumed(2L, queueBarriers.get(1).getStartRow()));
            }
        });
        // Reduce instances of group 2 through group reconfiguration, remove group 1 and 3, add group 4.
        configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(2L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(4L, 1, DequeueStrategy.FIFO, null)));
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // States and barrier info for removed groups should be gone
                try {
                    // There should be no barrier info for group 1
                    List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(1L);
                    Assert.assertTrue(queueBarriers.isEmpty());
                    stateStore.getState(1L, 0);
                    Assert.fail("Not expected to get state for group 1");
                } catch (Exception e) {
                // Expected
                }
                try {
                    // There should be no barrier info for group 3
                    List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(3L);
                    Assert.assertTrue(queueBarriers.isEmpty());
                    stateStore.getState(3L, 0);
                    Assert.fail("Not expected to get state for group 3");
                } catch (Exception e) {
                // Expected
                }
                // For group 2, there should be two barrier infos,
                // since all consumers passed the first barrier (groupSize = 2). Only the size = 3 and size = 1 left
                List<QueueBarrier> queueBarriers = stateStore.getAllBarriers(2L);
                Assert.assertEquals(2, queueBarriers.size());
                // Make all consumers (3 of them before reconfigure) in group 2 consumes everything
                for (int instanceId = 0; instanceId < 3; instanceId++) {
                    stateStore.completed(2L, instanceId);
                }
                // For the remaining consumer, it should start consuming from the latest barrier
                HBaseConsumerState state = stateStore.getState(2L, 0);
                Assert.assertEquals(0, Bytes.compareTo(state.getStartRow(), queueBarriers.get(1).getStartRow()));
                Assert.assertNull(state.getNextBarrier());
                // For removed instances, they should throw exception when retrieving their states
                for (int i = 1; i < 3; i++) {
                    try {
                        stateStore.getState(2L, i);
                        Assert.fail("Not expected to get state for group 2, instance " + i);
                    } catch (Exception e) {
                    // Expected
                    }
                }
            }
        });
    } finally {
        queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
    }
}

Example 9 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class LevelDBQueueTest method testQueueTableNameFormat.

// TODO: CDAP-1177 Should move to QueueTest after making getApplicationName() etc instance methods in a base class
@Test
public void testQueueTableNameFormat() throws Exception {
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getNamespace(), "application1", "flow1", "flowlet1", "output1");
    String tableName = ((LevelDBQueueAdmin) queueAdmin).getActualTableName(queueName);
    Assert.assertEquals("default.system.queue.application1.flow1", tableName);
    Assert.assertEquals("application1", LevelDBQueueAdmin.getApplicationName(tableName));
    Assert.assertEquals("flow1", LevelDBQueueAdmin.getFlowName(tableName));
    queueName = QueueName.fromFlowlet("testNamespace", "application1", "flow1", "flowlet1", "output1");
    tableName = ((LevelDBQueueAdmin) queueAdmin).getActualTableName(queueName);
    Assert.assertEquals("testNamespace.system.queue.application1.flow1", tableName);
    Assert.assertEquals("application1", LevelDBQueueAdmin.getApplicationName(tableName));
    Assert.assertEquals("flow1", LevelDBQueueAdmin.getFlowName(tableName));
}

Example 10 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class QueueTest method testDropAllForNamespace.

@Test
public void testDropAllForNamespace() throws Exception {
    // create 4 queues
    QueueName myQueue1 = QueueName.fromFlowlet(NAMESPACE_ID.getEntityName(), "myapp1", "myflow1", "myflowlet1", "myout1");
    QueueName myQueue2 = QueueName.fromFlowlet(NAMESPACE_ID.getEntityName(), "myapp2", "myflow2", "myflowlet2", "myout2");
    QueueName yourQueue1 = QueueName.fromFlowlet(NAMESPACE_ID1.getEntityName(), "yourapp1", "yourflow1", "yourflowlet1", "yourout1");
    QueueName yourQueue2 = QueueName.fromFlowlet(NAMESPACE_ID1.getEntityName(), "yourapp2", "yourflow2", "yourflowlet2", "yourout2");
    queueAdmin.create(myQueue1);
    queueAdmin.create(myQueue2);
    queueAdmin.create(yourQueue1);
    queueAdmin.create(yourQueue2);
    // verify that queues got created
    Assert.assertTrue(queueAdmin.exists(myQueue1) && queueAdmin.exists(myQueue2) && queueAdmin.exists(yourQueue1) && queueAdmin.exists(yourQueue2));
    // create some consumer configurations for all queues
    List<ConsumerGroupConfig> groupConfigs = ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null));
    configureGroups(myQueue1, groupConfigs);
    configureGroups(myQueue2, groupConfigs);
    configureGroups(yourQueue1, groupConfigs);
    configureGroups(yourQueue2, groupConfigs);
    // verify that the consumer config exists
    verifyConsumerConfigExists(myQueue1, myQueue2, yourQueue1, yourQueue2);
    // drop queues in NAMESPACE_ID
    queueAdmin.dropAllInNamespace(NAMESPACE_ID);
    // verify queues in NAMESPACE_ID are dropped
    Assert.assertFalse(queueAdmin.exists(myQueue1) || queueAdmin.exists(myQueue2));
    // also verify that consumer config of all queues in 'myspace' is deleted
    verifyConsumerConfigIsDeleted(myQueue1, myQueue2);
    // but the ones in NAMESPACE_ID1 still exist
    Assert.assertTrue(queueAdmin.exists(yourQueue1) && queueAdmin.exists(yourQueue2));
    // consumer config for queues in 'namespace1' should also still exist
    verifyConsumerConfigExists(yourQueue1, yourQueue2);
    // drop queues in NAMESPACE_ID1
    queueAdmin.dropAllInNamespace(NAMESPACE_ID1);
    // verify queues in NAMESPACE_ID are dropped
    Assert.assertFalse(queueAdmin.exists(yourQueue1) || queueAdmin.exists(yourQueue2));
    // verify that the consumer config of all queues in 'namespace1' is deleted
    verifyConsumerConfigIsDeleted(yourQueue1, yourQueue2);
}