Examples with QueueEntry co.cask.cdap.data2.queue.QueueEntry used on opensource projects

Example 11 with QueueEntry

use of co.cask.cdap.data2.queue.QueueEntry in project cdap by caskdata.

the class AbstractQueueProducer method commitTx.

@Override
public boolean commitTx() throws Exception {
    Preconditions.checkState(transaction != null, "Commit without starting transaction.");
    Transaction tx = transaction;
    transaction = null;
    List<QueueEntry> entries = Lists.newArrayListWithCapacity(queue.size());
    queue.drainTo(entries);
    lastEnqueueCount = entries.size();
    lastEnqueueBytes = persist(entries, tx);
    return true;
}

Example 12 with QueueEntry

use of co.cask.cdap.data2.queue.QueueEntry in project cdap by caskdata.

the class QueueTest method testClearOrDropAllForFlow.

private void testClearOrDropAllForFlow(boolean doDrop) throws Exception {
    // this test is the same for clear and drop, except for two small places...
    // using a different app name for each case as this test leaves some entries
    String app = doDrop ? "tDAFF" : "tCAFF";
    QueueName queueName1 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow1", "flowlet1", "out1");
    QueueName queueName2 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow1", "flowlet2", "out2");
    QueueName queueName3 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow2", "flowlet1", "out");
    List<ConsumerGroupConfig> groupConfigs = ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null));
    configureGroups(queueName1, groupConfigs);
    configureGroups(queueName2, groupConfigs);
    configureGroups(queueName3, groupConfigs);
    try (QueueProducer producer1 = queueClientFactory.createProducer(queueName1);
        QueueProducer producer2 = queueClientFactory.createProducer(queueName2);
        QueueProducer producer3 = queueClientFactory.createProducer(queueName3)) {
        TransactionContext txContext = createTxContext(producer1, producer2, producer3);
        txContext.start();
        for (int i = 0; i < 10; i++) {
            for (QueueProducer producer : Arrays.asList(producer1, producer2, producer3)) {
                producer.enqueue(new QueueEntry(Bytes.toBytes(i)));
            }
        }
        txContext.finish();
    }
    // consume 1 element from each queue
    ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
    try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName1, consumerConfig, 1);
        QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName2, consumerConfig, 1);
        QueueConsumer consumer3 = queueClientFactory.createConsumer(queueName3, consumerConfig, 1)) {
        TransactionContext txContext = createTxContext(consumer1, consumer2, consumer3);
        txContext.start();
        for (QueueConsumer consumer : Arrays.asList(consumer1, consumer2, consumer3)) {
            DequeueResult<byte[]> result = consumer.dequeue(1);
            Assert.assertFalse(result.isEmpty());
            Assert.assertArrayEquals(Bytes.toBytes(0), result.iterator().next());
        }
        txContext.finish();
    }
    // verify the consumer config was deleted
    verifyConsumerConfigExists(queueName1, queueName2);
    // clear/drop all queues for flow1
    FlowId flow1Id = NamespaceId.DEFAULT.app(app).flow("flow1");
    if (doDrop) {
        queueAdmin.dropAllForFlow(flow1Id);
    } else {
        queueAdmin.clearAllForFlow(flow1Id);
    }
    if (doDrop) {
        // verify that only flow2's queues still exist
        Assert.assertFalse(queueAdmin.exists(queueName1));
        Assert.assertFalse(queueAdmin.exists(queueName2));
        Assert.assertTrue(queueAdmin.exists(queueName3));
    } else {
        // verify all queues still exist
        Assert.assertTrue(queueAdmin.exists(queueName1));
        Assert.assertTrue(queueAdmin.exists(queueName2));
        Assert.assertTrue(queueAdmin.exists(queueName3));
    }
    // verify the consumer config was deleted
    verifyConsumerConfigIsDeleted(queueName1, queueName2);
    // create new consumers because existing ones may have pre-fetched and cached some entries
    configureGroups(queueName1, groupConfigs);
    configureGroups(queueName2, groupConfigs);
    try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName1, consumerConfig, 1);
        QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName2, consumerConfig, 1);
        QueueConsumer consumer3 = queueClientFactory.createConsumer(queueName3, consumerConfig, 1)) {
        TransactionContext txContext = createTxContext(consumer1, consumer2, consumer3);
        txContext.start();
        // attempt to consume from flow1's queues, should be empty
        for (QueueConsumer consumer : Arrays.asList(consumer1, consumer2)) {
            DequeueResult<byte[]> result = consumer.dequeue(1);
            Assert.assertTrue(result.isEmpty());
        }
        // but flow2 was not deleted -> consumer 3 should get another entry
        DequeueResult<byte[]> result = consumer3.dequeue(1);
        Assert.assertFalse(result.isEmpty());
        Assert.assertArrayEquals(Bytes.toBytes(1), result.iterator().next());
        txContext.finish();
    }
}

Example 13 with QueueEntry

use of co.cask.cdap.data2.queue.QueueEntry in project cdap by caskdata.

the class QueueTest method testReset.

@Test
public void testReset() throws Exception {
    // NOTE: using different name of the queue from other unit-tests because this test leaves entries
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queueReset");
    configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null), new ConsumerConfig(1, 0, 1, DequeueStrategy.FIFO, null));
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        TransactionContext txContext = createTxContext(producer);
        txContext.start();
        producer.enqueue(new QueueEntry(Bytes.toBytes(0)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(1)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(2)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(3)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(4)));
        txContext.finish();
        try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2)) {
            // Check that there's smth in the queue, but do not consume: abort tx after check
            txContext = createTxContext(consumer1);
            txContext.start();
            Assert.assertEquals(0, Bytes.toInt(consumer1.dequeue().iterator().next()));
            txContext.finish();
        }
        // Reset queues
        queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
        // we gonna need another one to check again to avoid caching side-affects
        configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
        try (QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
            txContext = createTxContext(consumer2);
            // Check again: should be nothing in the queue
            txContext.start();
            Assert.assertTrue(consumer2.dequeue().isEmpty());
            txContext.finish();
            // add another entry
            txContext = createTxContext(producer);
            txContext.start();
            producer.enqueue(new QueueEntry(Bytes.toBytes(5)));
            txContext.finish();
            txContext = createTxContext(consumer2);
            // Check again: consumer should see new entry
            txContext.start();
            Assert.assertEquals(5, Bytes.toInt(consumer2.dequeue().iterator().next()));
            txContext.finish();
        }
    }
}

Example 14 with QueueEntry

use of co.cask.cdap.data2.queue.QueueEntry in project cdap by caskdata.

the class QueueTest method createEnqueueRunnable.

protected Runnable createEnqueueRunnable(final QueueName queueName, final int count, final int batchSize, final CyclicBarrier barrier) {
    return new Runnable() {

        @Override
        public void run() {
            try {
                if (barrier != null) {
                    barrier.await();
                }
                try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
                    TransactionContext txContext = createTxContext(producer);
                    LOG.info("Start enqueue {} entries.", count);
                    Stopwatch stopwatch = new Stopwatch();
                    stopwatch.start();
                    // Pre-Enqueue
                    int batches = count / batchSize;
                    List<QueueEntry> queueEntries = Lists.newArrayListWithCapacity(batchSize);
                    // include some negative hash values and some positive ones
                    int hashValueMultiplier = -1;
                    for (int i = 0; i < batches; i++) {
                        txContext.start();
                        try {
                            queueEntries.clear();
                            for (int j = 0; j < batchSize; j++) {
                                int val = i * batchSize + j;
                                byte[] queueData = Bytes.toBytes(val);
                                queueEntries.add(new QueueEntry("key", hashValueMultiplier * val, queueData));
                                hashValueMultiplier *= -1;
                            }
                            producer.enqueue(queueEntries);
                            txContext.finish();
                        } catch (TransactionFailureException e) {
                            LOG.error("Operation error", e);
                            txContext.abort();
                            throw Throwables.propagate(e);
                        }
                    }
                    long elapsed = stopwatch.elapsedTime(TimeUnit.MILLISECONDS);
                    LOG.info("Enqueue {} entries in {} ms for {}", count, elapsed, queueName.getSimpleName());
                    LOG.info("Enqueue avg {} entries per seconds for {}", (double) count * 1000 / elapsed, queueName.getSimpleName());
                    stopwatch.stop();
                }
            } catch (Exception e) {
                LOG.error(e.getMessage(), e);
            }
        }
    };
}

Example 15 with QueueEntry

use of co.cask.cdap.data2.queue.QueueEntry in project cdap by caskdata.

the class QueueTest method testDropAllQueues.

@Test
public void testDropAllQueues() throws Exception {
    // create a queue and a stream and enqueue one entry each
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "myApp", "myFlow", "myFlowlet", "tDAQ");
    ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
    configureGroups(queueName, ImmutableList.of(consumerConfig));
    try (final QueueProducer qProducer = queueClientFactory.createProducer(queueName)) {
        executorFactory.createExecutor(Lists.newArrayList((TransactionAware) qProducer)).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                qProducer.enqueue(new QueueEntry(Bytes.toBytes("q42")));
            }
        });
        // drop all queues
        queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
        // verify that queue is gone and stream is still there
        configureGroups(queueName, ImmutableList.of(consumerConfig));
        try (final QueueConsumer qConsumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
            executorFactory.createExecutor(Lists.newArrayList((TransactionAware) qConsumer)).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    DequeueResult<byte[]> dequeue = qConsumer.dequeue();
                    Assert.assertTrue(dequeue.isEmpty());
                }
            });
        }
    }
}