Examples with TransactionContext org.apache.tephra.TransactionContext used on opensource projects

Example 31 with TransactionContext

use of org.apache.tephra.TransactionContext in project cdap by caskdata.

the class QueueTest method testMultiStageConsumer.

@Test
public void testMultiStageConsumer() throws Exception {
    final QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "multistage");
    ConsumerGroupConfig groupConfig = new ConsumerGroupConfig(0L, 2, DequeueStrategy.HASH, "key");
    configureGroups(queueName, ImmutableList.of(groupConfig));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(groupConfig, 0), new ConsumerConfig(groupConfig, 1));
    // Enqueue 10 items
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        for (int i = 0; i < 10; i++) {
            TransactionContext txContext = createTxContext(producer);
            txContext.start();
            producer.enqueue(new QueueEntry("key", i, Bytes.toBytes(i)));
            txContext.finish();
        }
    }
    // Consumer all even entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 1)) {
        for (int i = 0; i < 5; i++) {
            TransactionContext txContext = createTxContext(consumer);
            txContext.start();
            DequeueResult<byte[]> result = consumer.dequeue();
            Assert.assertTrue(!result.isEmpty());
            Assert.assertEquals(i * 2, Bytes.toInt(result.iterator().next()));
            txContext.finish();
        }
    }
    // Consume 2 odd entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 1)) {
        TransactionContext txContext = createTxContext(consumer);
        txContext.start();
        DequeueResult<byte[]> result = consumer.dequeue(2);
        Assert.assertEquals(2, result.size());
        Iterator<byte[]> iter = result.iterator();
        for (int i = 0; i < 2; i++) {
            Assert.assertEquals(i * 2 + 1, Bytes.toInt(iter.next()));
        }
        txContext.finish();
    }
    // Consume the rest odd entries
    try (QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 1)) {
        for (int i = 2; i < 5; i++) {
            TransactionContext txContext = createTxContext(consumer);
            txContext.start();
            DequeueResult<byte[]> result = consumer.dequeue();
            Assert.assertTrue(!result.isEmpty());
            Assert.assertEquals(i * 2 + 1, Bytes.toInt(result.iterator().next()));
            txContext.finish();
        }
    }
}

Example 32 with TransactionContext

use of org.apache.tephra.TransactionContext in project cdap by caskdata.

the class QueueTest method createEnqueueRunnable.

protected Runnable createEnqueueRunnable(final QueueName queueName, final int count, final int batchSize, final CyclicBarrier barrier) {
    return new Runnable() {

        @Override
        public void run() {
            try {
                if (barrier != null) {
                    barrier.await();
                }
                try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
                    TransactionContext txContext = createTxContext(producer);
                    LOG.info("Start enqueue {} entries.", count);
                    Stopwatch stopwatch = new Stopwatch();
                    stopwatch.start();
                    // Pre-Enqueue
                    int batches = count / batchSize;
                    List<QueueEntry> queueEntries = Lists.newArrayListWithCapacity(batchSize);
                    // include some negative hash values and some positive ones
                    int hashValueMultiplier = -1;
                    for (int i = 0; i < batches; i++) {
                        txContext.start();
                        try {
                            queueEntries.clear();
                            for (int j = 0; j < batchSize; j++) {
                                int val = i * batchSize + j;
                                byte[] queueData = Bytes.toBytes(val);
                                queueEntries.add(new QueueEntry("key", hashValueMultiplier * val, queueData));
                                hashValueMultiplier *= -1;
                            }
                            producer.enqueue(queueEntries);
                            txContext.finish();
                        } catch (TransactionFailureException e) {
                            LOG.error("Operation error", e);
                            txContext.abort();
                            throw Throwables.propagate(e);
                        }
                    }
                    long elapsed = stopwatch.elapsedTime(TimeUnit.MILLISECONDS);
                    LOG.info("Enqueue {} entries in {} ms for {}", count, elapsed, queueName.getSimpleName());
                    LOG.info("Enqueue avg {} entries per seconds for {}", (double) count * 1000 / elapsed, queueName.getSimpleName());
                    stopwatch.stop();
                }
            } catch (Exception e) {
                LOG.error(e.getMessage(), e);
            }
        }
    };
}

Example 33 with TransactionContext

use of org.apache.tephra.TransactionContext in project cdap by caskdata.

the class QueueTest method testClearOrDropAllForFlow.

private void testClearOrDropAllForFlow(boolean doDrop) throws Exception {
    // this test is the same for clear and drop, except for two small places...
    // using a different app name for each case as this test leaves some entries
    String app = doDrop ? "tDAFF" : "tCAFF";
    QueueName queueName1 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow1", "flowlet1", "out1");
    QueueName queueName2 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow1", "flowlet2", "out2");
    QueueName queueName3 = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), app, "flow2", "flowlet1", "out");
    List<ConsumerGroupConfig> groupConfigs = ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null));
    configureGroups(queueName1, groupConfigs);
    configureGroups(queueName2, groupConfigs);
    configureGroups(queueName3, groupConfigs);
    try (QueueProducer producer1 = queueClientFactory.createProducer(queueName1);
        QueueProducer producer2 = queueClientFactory.createProducer(queueName2);
        QueueProducer producer3 = queueClientFactory.createProducer(queueName3)) {
        TransactionContext txContext = createTxContext(producer1, producer2, producer3);
        txContext.start();
        for (int i = 0; i < 10; i++) {
            for (QueueProducer producer : Arrays.asList(producer1, producer2, producer3)) {
                producer.enqueue(new QueueEntry(Bytes.toBytes(i)));
            }
        }
        txContext.finish();
    }
    // consume 1 element from each queue
    ConsumerConfig consumerConfig = new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null);
    try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName1, consumerConfig, 1);
        QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName2, consumerConfig, 1);
        QueueConsumer consumer3 = queueClientFactory.createConsumer(queueName3, consumerConfig, 1)) {
        TransactionContext txContext = createTxContext(consumer1, consumer2, consumer3);
        txContext.start();
        for (QueueConsumer consumer : Arrays.asList(consumer1, consumer2, consumer3)) {
            DequeueResult<byte[]> result = consumer.dequeue(1);
            Assert.assertFalse(result.isEmpty());
            Assert.assertArrayEquals(Bytes.toBytes(0), result.iterator().next());
        }
        txContext.finish();
    }
    // verify the consumer config was deleted
    verifyConsumerConfigExists(queueName1, queueName2);
    // clear/drop all queues for flow1
    FlowId flow1Id = NamespaceId.DEFAULT.app(app).flow("flow1");
    if (doDrop) {
        queueAdmin.dropAllForFlow(flow1Id);
    } else {
        queueAdmin.clearAllForFlow(flow1Id);
    }
    if (doDrop) {
        // verify that only flow2's queues still exist
        Assert.assertFalse(queueAdmin.exists(queueName1));
        Assert.assertFalse(queueAdmin.exists(queueName2));
        Assert.assertTrue(queueAdmin.exists(queueName3));
    } else {
        // verify all queues still exist
        Assert.assertTrue(queueAdmin.exists(queueName1));
        Assert.assertTrue(queueAdmin.exists(queueName2));
        Assert.assertTrue(queueAdmin.exists(queueName3));
    }
    // verify the consumer config was deleted
    verifyConsumerConfigIsDeleted(queueName1, queueName2);
    // create new consumers because existing ones may have pre-fetched and cached some entries
    configureGroups(queueName1, groupConfigs);
    configureGroups(queueName2, groupConfigs);
    try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName1, consumerConfig, 1);
        QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName2, consumerConfig, 1);
        QueueConsumer consumer3 = queueClientFactory.createConsumer(queueName3, consumerConfig, 1)) {
        TransactionContext txContext = createTxContext(consumer1, consumer2, consumer3);
        txContext.start();
        // attempt to consume from flow1's queues, should be empty
        for (QueueConsumer consumer : Arrays.asList(consumer1, consumer2)) {
            DequeueResult<byte[]> result = consumer.dequeue(1);
            Assert.assertTrue(result.isEmpty());
        }
        // but flow2 was not deleted -> consumer 3 should get another entry
        DequeueResult<byte[]> result = consumer3.dequeue(1);
        Assert.assertFalse(result.isEmpty());
        Assert.assertArrayEquals(Bytes.toBytes(1), result.iterator().next());
        txContext.finish();
    }
}

Example 34 with TransactionContext

use of org.apache.tephra.TransactionContext in project cdap by caskdata.

the class QueueTest method testReset.

@Test
public void testReset() throws Exception {
    // NOTE: using different name of the queue from other unit-tests because this test leaves entries
    QueueName queueName = QueueName.fromFlowlet(NamespaceId.DEFAULT.getEntityName(), "app", "flow", "flowlet", "queueReset");
    configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(0L, 1, DequeueStrategy.FIFO, null), new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
    List<ConsumerConfig> consumerConfigs = ImmutableList.of(new ConsumerConfig(0, 0, 1, DequeueStrategy.FIFO, null), new ConsumerConfig(1, 0, 1, DequeueStrategy.FIFO, null));
    try (QueueProducer producer = queueClientFactory.createProducer(queueName)) {
        TransactionContext txContext = createTxContext(producer);
        txContext.start();
        producer.enqueue(new QueueEntry(Bytes.toBytes(0)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(1)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(2)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(3)));
        producer.enqueue(new QueueEntry(Bytes.toBytes(4)));
        txContext.finish();
        try (QueueConsumer consumer1 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(0), 2)) {
            // Check that there's smth in the queue, but do not consume: abort tx after check
            txContext = createTxContext(consumer1);
            txContext.start();
            Assert.assertEquals(0, Bytes.toInt(consumer1.dequeue().iterator().next()));
            txContext.finish();
        }
        // Reset queues
        queueAdmin.dropAllInNamespace(NamespaceId.DEFAULT);
        // we gonna need another one to check again to avoid caching side-affects
        configureGroups(queueName, ImmutableList.of(new ConsumerGroupConfig(1L, 1, DequeueStrategy.FIFO, null)));
        try (QueueConsumer consumer2 = queueClientFactory.createConsumer(queueName, consumerConfigs.get(1), 2)) {
            txContext = createTxContext(consumer2);
            // Check again: should be nothing in the queue
            txContext.start();
            Assert.assertTrue(consumer2.dequeue().isEmpty());
            txContext.finish();
            // add another entry
            txContext = createTxContext(producer);
            txContext.start();
            producer.enqueue(new QueueEntry(Bytes.toBytes(5)));
            txContext.finish();
            txContext = createTxContext(consumer2);
            // Check again: consumer should see new entry
            txContext.start();
            Assert.assertEquals(5, Bytes.toInt(consumer2.dequeue().iterator().next()));
            txContext.finish();
        }
    }
}

Example 35 with TransactionContext

use of org.apache.tephra.TransactionContext in project cdap by caskdata.

the class TransactionContextTest method testPersistAndRollbackFailure.

@Test
public void testPersistAndRollbackFailure() throws TransactionFailureException, InterruptedException {
    ds1.failCommitTxOnce = InduceFailure.ThrowException;
    ds1.failRollbackTxOnce = InduceFailure.ThrowException;
    TransactionContext context = newTransactionContext(ds1, ds2);
    // start transaction
    context.start();
    // add a change to ds1 and ds2
    ds1.addChange(A);
    ds2.addChange(B);
    // commit transaction should fail and cause rollback
    try {
        context.finish();
        Assert.fail("Persist should have failed - exception should be thrown");
    } catch (TransactionFailureException e) {
        Assert.assertEquals("persist failure", e.getCause().getMessage());
    }
    // verify both are rolled back and tx is invalidated
    Assert.assertTrue(ds1.started);
    Assert.assertTrue(ds2.started);
    Assert.assertTrue(ds1.checked);
    Assert.assertTrue(ds2.checked);
    Assert.assertTrue(ds1.committed);
    Assert.assertFalse(ds2.committed);
    Assert.assertFalse(ds1.postCommitted);
    Assert.assertFalse(ds2.postCommitted);
    Assert.assertTrue(ds1.rolledBack);
    Assert.assertTrue(ds2.rolledBack);
    Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
}