Examples with QueueName co.cask.cdap.common.queue.QueueName used on opensource projects

Example 31 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class QueueTest method testReset.

@Test
public void testReset() throws Exception {
    // NOTE: using different name of the queue from other unit-tests because this test leaves entries
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queueReset");
    configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null), new ConsumerConfig(1, 0, 1, DequeueStrategy.FIFO, null));
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        TransactionContext txContext = createTxContext(producer);
        txContext.start();
        producer.enqueue(new QueueEntry(Bytes.toBytes(0)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(1)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(2)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(3)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(4)));
        txContext.finish();
        try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2)) {
            // Check that there's smth in the queue, but do not consume: abort tx after check
            txContext = createTxContext(consumer1);
            txContext.start();
            Assert.assertEquals(0, Bytes.toInt(consumer1.dequeue().iterator().next()));
            txContext.finish();
        }
        // Reset queues
        queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
        // we gonna need another one to check again to avoid caching side-affects
        configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
        try (QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
            txContext = createTxContext(consumer2);
            // Check again: should be nothing in the queue
            txContext.start();
            Assert.assertTrue(consumer2.dequeue().isEmpty());
            txContext.finish();
            // add another entry
            txContext = createTxContext(producer);
            txContext.start();
            producer.enqueue(new QueueEntry(Bytes.toBytes(5)));
            txContext.finish();
            txContext = createTxContext(consumer2);
            // Check again: consumer should see new entry
            txContext.start();
            Assert.assertEquals(5, Bytes.toInt(consumer2.dequeue().iterator().next()));
            txContext.finish();
        }
    }
}

Example 32 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class QueueTest method testMultiHash.

@Category(SlowTests.class)
@Test(timeout = TIMEOUT_MS)
public void testMultiHash() throws Exception {
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "multihash");
    enqueueDequeue(queueName, 2 * ROUNDS, ROUNDS, 1, 3, DequeueStrategy.HASH, 1);
}

Example 33 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class QueueTest method testDropAllQueues.

@Test
public void testDropAllQueues() throws Exception {
    // create a queue and a stream and enqueue one entry each
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "myApp", "myFlow", "myFlowlet", "tDAQ");
    ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
    configureGroups(queueName, ImmutableList.of(consumerConfig));
    try (final QueueProducer qProducer = queueClientFactory.createProducer(queueName)) {
        executorFactory.createExecutor(Lists.newArrayList((TransactionAware) qProducer)).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                qProducer.enqueue(new QueueEntry(Bytes.toBytes("q42")));
            }
        });
        // drop all queues
        queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
        // verify that queue is gone and stream is still there
        configureGroups(queueName, ImmutableList.of(consumerConfig));
        try (final QueueConsumer qConsumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
            executorFactory.createExecutor(Lists.newArrayList((TransactionAware) qConsumer)).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    DequeueResult<byte[]> dequeue = qConsumer.dequeue();
                    Assert.assertTrue(dequeue.isEmpty());
                }
            });
        }
    }
}

Example 34 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class ProgramLifecycleHttpHandlerTest method testDeleteQueues.

@Test
public void testDeleteQueues() throws Exception {
    QueueName queueName = QueueName.fromFlowlet(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME, "out");
    // enqueue some data
    enqueue(queueName, new QueueEntry("x".getBytes(Charsets.UTF_8)));
    // verify it exists
    Assert.assertTrue(dequeueOne(queueName));
    // clear queue in wrong namespace
    Assert.assertEquals(200, doDelete("/v3/namespaces/" + TEST_NAMESPACE2 + "/queues").getStatusLine().getStatusCode());
    // verify queue is still here
    Assert.assertTrue(dequeueOne(queueName));
    // clear queue in the right namespace
    Assert.assertEquals(200, doDelete("/v3/namespaces/" + TEST_NAMESPACE1 + "/queues").getStatusLine().getStatusCode());
    // verify queue is gone
    Assert.assertFalse(dequeueOne(queueName));
}

Example 35 with QueueName

use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.

the class HBaseQueueAdmin method deleteFlowConfigs.

private void deleteFlowConfigs(FlowId flowId) throws Exception {
    // It's a bit hacky here since we know how the HBaseConsumerStateStore works.
    // Maybe we need another Dataset set that works across all queues.
    final QueueName prefixName = QueueName.from(URI.create(QueueName.prefixForFlow(flowId)));
    DatasetId stateStoreId = getStateStoreId(flowId.getNamespace());
    Map<String, String> args = ImmutableMap.of(HBaseQueueDatasetModule.PROPERTY_QUEUE_NAME, prefixName.toString());
    HBaseConsumerStateStore stateStore = datasetFramework.getDataset(stateStoreId, args, null);
    if (stateStore == null) {
        // If the state store doesn't exists, meaning there is no queue, hence nothing to do.
        return;
    }
    try {
        final Table table = stateStore.getInternalTable();
        Transactions.createTransactionExecutor(txExecutorFactory, (TransactionAware) table).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // Prefix name is "/" terminated ("queue:///namespace/app/flow/"), hence the scan is unique for the flow
                byte[] startRow = Bytes.toBytes(prefixName.toString());
                try (Scanner scanner = table.scan(startRow, Bytes.stopKeyForPrefix(startRow))) {
                    Row row = scanner.next();
                    while (row != null) {
                        table.delete(row.getRow());
                        row = scanner.next();
                    }
                }
            }
        });
    } finally {
        stateStore.close();
    }
}