use of org.apache.tephra.Transaction in project cdap by caskdata.
the class SparkTransactional method execute.
/**
* Executes the given runnable with transactionally. If there is an opened transaction that can be used, then
* the runnable will be executed with that existing transaction.
* Otherwise, a new long transaction will be created to execute the given runnable.
*
* @param runnable The {@link TxRunnable} to be executed inside a transaction
* @param transactionType The {@link TransactionType} of the Spark transaction.
*/
void execute(SparkTxRunnable runnable, TransactionType transactionType) throws TransactionFailureException {
TransactionalDatasetContext txDatasetContext = activeDatasetContext.get();
boolean needCommit = false;
// If there is an existing transaction
if (txDatasetContext != null) {
TransactionType currentTransactionType = txDatasetContext.getTransactionType();
// We don't support nested transaction
if (currentTransactionType == TransactionType.EXPLICIT && transactionType == TransactionType.EXPLICIT) {
throw new TransactionFailureException("Nested transaction not supported. Active transaction is " + txDatasetContext.getTransaction());
}
// If the current transaction is commit on job end, we need some special handling
if (currentTransactionType == TransactionType.IMPLICIT_COMMIT_ON_JOB_END) {
// associated with the transaction is completed (asynchronously).
if (txDatasetContext.isJobStarted()) {
try {
txDatasetContext.awaitCompletion();
txDatasetContext = null;
} catch (InterruptedException e) {
// Don't execute the runnable. Reset the interrupt flag and return
Thread.currentThread().interrupt();
return;
}
} else if (transactionType != TransactionType.IMPLICIT_COMMIT_ON_JOB_END) {
// If the job hasn't been started and the requested type is not commit on job end,
// we need to "upgrade" the transaction type based on the requested type
// E.g. if the requested type is EXPLICIT, then the current transaction will become an explicit one
txDatasetContext.setTransactionType(transactionType);
needCommit = true;
}
}
}
// If there is no active transaction, start a new long transaction
if (txDatasetContext == null) {
txDatasetContext = new TransactionalDatasetContext(datasetCache, transactionType);
activeDatasetContext.set(txDatasetContext);
needCommit = transactionType != TransactionType.IMPLICIT_COMMIT_ON_JOB_END;
}
Transaction transaction = txDatasetContext.getTransaction();
try {
// Call the runnable
runnable.run(txDatasetContext);
// Persist the changes
txDatasetContext.flush();
if (needCommit) {
txClient.commitOrThrow(transaction);
activeDatasetContext.remove();
txDatasetContext.postCommit();
txDatasetContext.discardDatasets();
}
} catch (Throwable t) {
// Only need to rollback and invalidate transaction if the current call needs to commit.
if (needCommit) {
// Any exception will cause invalidation of the transaction
activeDatasetContext.remove();
txDatasetContext.rollbackWithoutFailure();
Transactions.invalidateQuietly(txClient, transaction);
}
throw Transactions.asTransactionFailure(t);
}
}
use of org.apache.tephra.Transaction in project cdap by caskdata.
the class MessageTableTest method testSingleMessage.
@Test
public void testSingleMessage() throws Exception {
TopicId topicId = NamespaceId.DEFAULT.topic("singleMessage");
TopicMetadata metadata = new TopicMetadata(topicId, DEFAULT_PROPERTY);
String payload = "data";
long txWritePtr = 123L;
try (MessageTable table = getMessageTable();
MetadataTable metadataTable = getMetadataTable()) {
metadataTable.createTopic(metadata);
List<MessageTable.Entry> entryList = new ArrayList<>();
entryList.add(new TestMessageEntry(topicId, GENERATION, 0L, 0, txWritePtr, Bytes.toBytes(payload)));
table.store(entryList.iterator());
byte[] messageId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(0L, (short) 0, 0L, (short) 0, messageId, 0);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), false, 50, null)) {
// Fetch not including the first message, expect empty
Assert.assertFalse(iterator.hasNext());
}
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
// Fetch including the first message, should get the message
Assert.assertTrue(iterator.hasNext());
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, 0, 50, null)) {
// Fetch by time, should get the entry
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
RollbackDetail rollbackDetail = new TestRollbackDetail(123L, 0, (short) 0, 0L, (short) 0);
table.rollback(metadata, rollbackDetail);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, null)) {
// Fetching the message non-tx should provide a result even after deletion
MessageTable.Entry entry = iterator.next();
Assert.assertArrayEquals(Bytes.toBytes(payload), entry.getPayload());
Assert.assertFalse(iterator.hasNext());
}
Transaction tx = new Transaction(200, 200, new long[0], new long[0], -1);
try (CloseableIterator<MessageTable.Entry> iterator = table.fetch(metadata, new MessageId(messageId), true, 50, tx)) {
// Fetching messages transactionally should not return any entry
Assert.assertFalse(iterator.hasNext());
}
}
}
use of org.apache.tephra.Transaction in project cdap by caskdata.
the class MessagingHttpServiceTest method testBasicPubSub.
@Test
public void testBasicPubSub() throws Exception {
TopicId topicId = new NamespaceId("ns1").topic("testBasicPubSub");
// Publish to a non-existing topic should get not found exception
try {
client.publish(StoreRequestBuilder.of(topicId).addPayloads("a").build());
Assert.fail("Expected TopicNotFoundException");
} catch (TopicNotFoundException e) {
// Expected
}
// Consume from a non-existing topic should get not found exception
try {
client.prepareFetch(topicId).fetch();
Assert.fail("Expected TopicNotFoundException");
} catch (TopicNotFoundException e) {
// Expected
}
client.createTopic(new TopicMetadata(topicId));
// Publish a non-transactional message with empty payload should result in failure
try {
client.publish(StoreRequestBuilder.of(topicId).build());
Assert.fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
// Expected
}
// Publish a non-tx message, no RollbackDetail is returned
Assert.assertNull(client.publish(StoreRequestBuilder.of(topicId).addPayloads("m0", "m1").build()));
// Publish a transactional message, a RollbackDetail should be returned
RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topicId).addPayloads("m2").setTransaction(1L).build());
Assert.assertNotNull(rollbackDetail);
// Rollback the published message
client.rollback(topicId, rollbackDetail);
// Fetch messages non-transactionally (should be able to read all the messages since rolled back messages
// are still visible until ttl kicks in)
List<RawMessage> messages = new ArrayList<>();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) {
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(3, messages.size());
for (int i = 0; i < 3; i++) {
Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
}
// Consume transactionally. It should get only m0 and m1 since m2 has been rolled back
List<RawMessage> txMessages = new ArrayList<>();
Transaction transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(0).setTransaction(transaction).fetch()) {
Iterators.addAll(txMessages, iterator);
}
Assert.assertEquals(2, txMessages.size());
for (int i = 0; i < 2; i++) {
Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
}
// Fetch again from a given message offset exclusively.
// Expects one message to be fetched
byte[] startMessageId = messages.get(1).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
// It should have only one message (m2)
Assert.assertTrue(iterator.hasNext());
RawMessage msg = iterator.next();
Assert.assertEquals("m2", Bytes.toString(msg.getPayload()));
}
// Fetch again from the last message offset exclusively
// Expects no message to be fetched
startMessageId = messages.get(2).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
Assert.assertFalse(iterator.hasNext());
}
// Fetch with start time. It should get both m0 and m1 since they are published in the same request, hence
// having the same publish time
startMessageId = messages.get(1).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartTime(new MessageId(startMessageId).getPublishTimestamp()).setLimit(2).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(2, messages.size());
for (int i = 0; i < 2; i++) {
Assert.assertEquals("m" + i, Bytes.toString(messages.get(i).getPayload()));
}
// Publish 2 messages, one transactionally, one without transaction
client.publish(StoreRequestBuilder.of(topicId).addPayloads("m3").setTransaction(2L).build());
client.publish(StoreRequestBuilder.of(topicId).addPayloads("m4").build());
// Consume without transactional, it should see m2, m3 and m4
startMessageId = messages.get(1).getId();
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(3, messages.size());
for (int i = 0; i < 3; i++) {
Assert.assertEquals("m" + (i + 2), Bytes.toString(messages.get(i).getPayload()));
}
// Consume using a transaction that doesn't have tx = 2L visible. It should get no message as it should block on m3
transaction = new Transaction(3L, 3L, new long[0], new long[] { 2L }, 2L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
Assert.assertFalse(iterator.hasNext());
}
// Consume using a transaction that has tx = 2L in the invalid list. It should skip m3 and got m4
transaction = new Transaction(3L, 3L, new long[] { 2L }, new long[0], 0L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(1, messages.size());
Assert.assertEquals("m4", Bytes.toString(messages.get(0).getPayload()));
// Consume using a transaction that has tx = 2L committed. It should get m3 and m4
transaction = new Transaction(3L, 3L, new long[0], new long[0], 0L);
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setStartMessage(startMessageId, false).setTransaction(transaction).fetch()) {
messages.clear();
Iterators.addAll(messages, iterator);
}
Assert.assertEquals(2, messages.size());
for (int i = 0; i < 2; i++) {
Assert.assertEquals("m" + (i + 3), Bytes.toString(messages.get(i).getPayload()));
}
client.deleteTopic(topicId);
}
use of org.apache.tephra.Transaction in project cdap by caskdata.
the class CachingMessageTableTest method testCachePruning.
@Test
public void testCachePruning() throws Exception {
long txGracePeriod = 6;
CConfiguration cConf = CConfiguration.create();
cConf.setLong(CachingMessageTable.PRUNE_GRACE_PERIOD, txGracePeriod);
// Creates a CachingMessageTable with a controlled time provider
final AtomicLong currentTimeMillis = new AtomicLong(0);
MessageTable messageTable = new CachingMessageTable(cConf, super.getMessageTable(), cacheProvider, new TimeProvider() {
@Override
public long currentTimeMillis() {
return currentTimeMillis.get();
}
});
// Insert 10 entries, with different publish time
TopicMetadata metadata = new TopicMetadata(NamespaceId.DEFAULT.topic("test"), TopicMetadata.GENERATION_KEY, 1, TopicMetadata.TTL_KEY, 86400);
for (int i = 0; i < 10; i++) {
// Key is (topic, generation, publish time, sequence id)
byte[] key = Bytes.concat(MessagingUtils.toDataKeyPrefix(metadata.getTopicId(), metadata.getGeneration()), Bytes.toBytes((long) i), Bytes.toBytes((short) 0));
// Store a message with a write pointer
messageTable.store(Collections.singleton(new ImmutableMessageTableEntry(key, Bytes.toBytes("Payload " + i), Bytes.toBytes((long) i))).iterator());
}
// Update the current time to 11
currentTimeMillis.set(11);
// Fetch from the table without transaction, should get all entries.
try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, 0, 100, null)) {
List<MessageTable.Entry> entries = Lists.newArrayList(iter);
Assert.assertEquals(10, entries.size());
// All entries must be from the cache
for (MessageTable.Entry entry : entries) {
Assert.assertTrue(entry instanceof CachingMessageTable.CacheMessageTableEntry);
}
}
// Fetch with a transaction, with start time older than tx grace period / 2
Transaction tx = new Transaction(10, 11, new long[0], new long[0], 11);
long startTime = currentTimeMillis.get() - txGracePeriod / 2 - 1;
try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, startTime, 100, tx)) {
List<MessageTable.Entry> entries = Lists.newArrayList(iter);
// Should get three entries (7, 8, 9)
Assert.assertEquals(3, entries.size());
// The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
// from cache (current time = 11, grace period = 3)
Iterator<MessageTable.Entry> iterator = entries.iterator();
Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
}
// Fetch with a transaction, with start messageId publish time older than tx grace period / 2
byte[] rawId = new byte[MessageId.RAW_ID_SIZE];
MessageId.putRawId(startTime, (short) 0, 0L, (short) 0, rawId, 0);
MessageId messageId = new MessageId(rawId);
try (CloseableIterator<MessageTable.Entry> iter = messageTable.fetch(metadata, messageId, true, 100, tx)) {
List<MessageTable.Entry> entries = Lists.newArrayList(iter);
// Should get three entries (7, 8, 9)
Assert.assertEquals(3, entries.size());
// The first entry should be from the table, while the last two entries (timestamp 8 and 9) should be
// from cache (current time = 11, grace period = 3)
Iterator<MessageTable.Entry> iterator = entries.iterator();
Assert.assertFalse(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
Assert.assertTrue(iterator.next() instanceof CachingMessageTable.CacheMessageTableEntry);
}
}
use of org.apache.tephra.Transaction in project cdap by caskdata.
the class HBaseTableCoprocessorTestRun method testInvalidTx.
@Test
public void testInvalidTx() throws Exception {
try (MetadataTable metadataTable = getMetadataTable();
MessageTable messageTable = getMessageTable()) {
TopicId topicId = NamespaceId.DEFAULT.topic("invalidTx");
TopicMetadata topic = new TopicMetadata(topicId, TopicMetadata.TTL_KEY, "1000000", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION));
metadataTable.createTopic(topic);
List<MessageTable.Entry> entries = new ArrayList<>();
long invalidTxWritePtr = invalidList.toRawList().get(0);
entries.add(new TestMessageEntry(topicId, GENERATION, "data", invalidTxWritePtr, (short) 0));
messageTable.store(entries.iterator());
// Fetch the entries and make sure we are able to read it
try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
checkEntry(iterator, invalidTxWritePtr);
}
// Fetch the entries with tx and make sure we are able to read it
Transaction tx = new Transaction(V[8], V[8], new long[0], new long[0], -1);
try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, tx)) {
checkEntry(iterator, invalidTxWritePtr);
}
// Now run full compaction
forceFlushAndCompact(Table.MESSAGE);
// Try to fetch the entry non-transactionally and the entry should still be there
try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
checkEntry(iterator, invalidTxWritePtr);
}
// Fetch the entries transactionally and we should see no entries returned
try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, tx)) {
Assert.assertFalse(iterator.hasNext());
}
metadataTable.deleteTopic(topicId);
// Sleep so that the metadata cache expires
TimeUnit.SECONDS.sleep(3 * METADATA_CACHE_EXPIRY);
forceFlushAndCompact(Table.MESSAGE);
// Test deletion of messages from a deleted topic
try (CloseableIterator<MessageTable.Entry> iterator = messageTable.fetch(topic, 0, Integer.MAX_VALUE, null)) {
Assert.assertFalse(iterator.hasNext());
}
}
}
Aggregations