Examples with QueueEntry co.cask.cdap.data2.queue.QueueEntry used on opensource projects

Example 1 with QueueEntry

use of co.cask.cdap.data2.queue.QueueEntry in project cdap by caskdata.

the class LevelDBQueueProducer method persist.

@Override
protected int persist(Iterable<QueueEntry> entries, Transaction transaction) throws Exception {
    long writePointer = transaction.getWritePointer();
    byte[] rowKeyPrefix = Bytes.add(queueRowPrefix, Bytes.toBytes(writePointer));
    int count = 0;
    int bytes = 0;
    for (QueueEntry entry : entries) {
        // Row key = queue_name + writePointer + counter
        byte[] rowKey = Bytes.add(rowKeyPrefix, Bytes.toBytes(count++));
        NavigableMap<byte[], byte[]> row = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
        row.put(QueueEntryRow.DATA_COLUMN, entry.getData());
        row.put(QueueEntryRow.META_COLUMN, QueueEntry.serializeHashKeys(entry.getHashKeys()));
        changes.put(rowKey, row);
        bytes += entry.getData().length;
    }
    // TODO introduce a constant in the OcTableCore for the latest timestamp
    core.persist(changes, KeyValue.LATEST_TIMESTAMP);
    return bytes;
}

Example 2 with QueueEntry

use of co.cask.cdap.data2.queue.QueueEntry in project cdap by caskdata.

the class ShardedHBaseQueueStrategy method getRowKeys.

@Override
public void getRowKeys(Iterable<ConsumerGroupConfig> consumerGroupConfigs, QueueEntry queueEntry, byte[] rowKeyPrefix, long writePointer, int counter, Collection<byte[]> rowKeys) {
    byte[] rowKey = new byte[rowKeyPrefix.length + Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT];
    Bytes.putBytes(rowKey, 0, rowKeyPrefix, 0, rowKeyPrefix.length);
    Bytes.putLong(rowKey, rowKeyPrefix.length, writePointer);
    Bytes.putInt(rowKey, rowKey.length - Bytes.SIZEOF_INT, counter);
    // Generates all row keys, one per consumer group.
    for (ConsumerGroupConfig config : consumerGroupConfigs) {
        DequeueStrategy dequeueStrategy = config.getDequeueStrategy();
        // Default for FIFO
        int instanceId = -1;
        if (dequeueStrategy != DequeueStrategy.FIFO) {
            if (dequeueStrategy == DequeueStrategy.ROUND_ROBIN) {
                instanceId = QueueEntryRow.getRoundRobinConsumerInstance(writePointer, counter, config.getGroupSize());
            } else if (dequeueStrategy == DequeueStrategy.HASH) {
                instanceId = QueueEntryRow.getHashConsumerInstance(queueEntry.getHashKeys(), config.getHashKey(), config.getGroupSize());
            } else {
                throw new IllegalArgumentException("Unsupported consumer strategy: " + dequeueStrategy);
            }
        }
        rowKeys.add(rowKeyDistributor.getDistributedKey(getShardedKey(config, instanceId, rowKey)));
    }
}

Example 3 with QueueEntry

use of co.cask.cdap.data2.queue.QueueEntry in project cdap by caskdata.

the class InMemoryQueueProducer method persist.

@Override
protected int persist(Iterable<QueueEntry> entries, Transaction transaction) throws Exception {
    commitTransaction = transaction;
    int seqId = 0;
    int bytes = 0;
    InMemoryQueue queue = getQueue();
    for (QueueEntry entry : entries) {
        queue.enqueue(transaction.getWritePointer(), seqId++, entry);
        bytes += entry.getData().length;
    }
    lastEnqueueCount = seqId;
    return bytes;
}

Example 4 with QueueEntry

use of co.cask.cdap.data2.queue.QueueEntry in project cdap by caskdata.

the class QueueTest method testRollback.

@Test(timeout = TIMEOUT_MS)
public void testRollback() throws Exception {
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queuerollback");
    ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
    configureGroups(queueName, ImmutableList.of(consumerConfig));
    try (QueueProducer producer = queueClientFactory.createProducer(queueName);
        QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
        TransactionContext txContext = createTxContext(producer, consumer, new TransactionAware() {

            boolean canCommit = false;

            @Override
            public void startTx(Transaction tx) {
            }

            @Override
            public void updateTx(Transaction tx) {
            }

            @Override
            public Collection<byte[]> getTxChanges() {
                return ImmutableList.of();
            }

            @Override
            public boolean commitTx() throws Exception {
                // Flip-flop between commit success/failure.
                boolean res = canCommit;
                canCommit = !canCommit;
                return res;
            }

            @Override
            public void postTxCommit() {
            }

            @Override
            public boolean rollbackTx() throws Exception {
                return true;
            }

            @Override
            public String getTransactionAwareName() {
                return "test";
            }
        });
        // First, try to enqueue and commit would fail
        txContext.start();
        try {
            producer.enqueue(new QueueEntry(Bytes.toBytes(1)));
            txContext.finish();
            // If reaches here, it's wrong, as exception should be thrown.
            Assert.assertTrue(false);
        } catch (TransactionFailureException e) {
            txContext.abort();
        }
        // Try to enqueue again. Within the same transaction, dequeue should be empty.
        txContext.start();
        producer.enqueue(new QueueEntry(Bytes.toBytes(1)));
        Assert.assertTrue(consumer.dequeue().isEmpty());
        txContext.finish();
        // This time, enqueue has been committed, dequeue would see the item
        txContext.start();
        try {
            Assert.assertEquals(1, Bytes.toInt(consumer.dequeue().iterator().next()));
            txContext.finish();
            // If reaches here, it's wrong, as exception should be thrown.
            Assert.assertTrue(false);
        } catch (TransactionFailureException e) {
            txContext.abort();
        }
        // Dequeue again, since last tx was rollback, this dequeue should see the item again.
        txContext.start();
        Assert.assertEquals(1, Bytes.toInt(consumer.dequeue().iterator().next()));
        txContext.finish();
    }
}

Example 5 with QueueEntry

use of co.cask.cdap.data2.queue.QueueEntry in project cdap by caskdata.

the class QueueTest method testConcurrentEnqueue.

@Category(SlowTests.class)
@Test
public void testConcurrentEnqueue() throws Exception {
    // This test is for testing multiple producers that writes with a delay after a transaction started.
    // This is for verifying consumer advances the startKey correctly.
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "concurrent");
    configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0, 1, DequeueStrategy.FIFO, null)));
    final CyclicBarrier barrier = new CyclicBarrier(4);
    ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
    // Starts three producers to enqueue concurrently. For each entry, starts a TX, sleep, enqueue, commit.
    ExecutorService executor = Executors.newFixedThreadPool(3);
    final int entryCount = 50;
    for (int i = 0; i < 3; i++) {
        final QueueProducer producer = queueClientFactory.createProducer(queueName);
        final int producerId = i + 1;
        executor.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    barrier.await();
                    for (int i = 0; i < entryCount; i++) {
                        TransactionContext txContext = createTxContext(producer);
                        txContext.start();
                        // Sleeps at different rate to make the scan in consumer has higher change to see
                        // the transaction but not the entry (as not yet written)
                        TimeUnit.MILLISECONDS.sleep(producerId * 50);
                        producer.enqueue(new QueueEntry(Bytes.toBytes(i)));
                        txContext.finish();
                    }
                } catch (Exception e) {
                    LOG.error(e.getMessage(), e);
                } finally {
                    Closeables.closeQuietly(producer);
                }
            }
        });
    }
    // sum(0..entryCount) * 3
    int expectedSum = entryCount * (entryCount - 1) / 2 * 3;
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, 1)) {
        // Trigger starts of producer
        barrier.await();
        int dequeueSum = 0;
        int noProgress = 0;
        while (dequeueSum != expectedSum && noProgress < 200) {
            TransactionContext txContext = createTxContext(consumer);
            txContext.start();
            DequeueResult<byte[]> result = consumer.dequeue();
            if (!result.isEmpty()) {
                noProgress = 0;
                int value = Bytes.toInt(result.iterator().next());
                dequeueSum += value;
            } else {
                noProgress++;
                TimeUnit.MILLISECONDS.sleep(10);
            }
            txContext.finish();
        }
        Assert.assertEquals(expectedSum, dequeueSum);
    }
}