use of io.pravega.controller.store.task.TxnResource in project pravega by pravega.
the class StreamTransactionMetadataTasks method createTxnBody.
/**
* Creates txn on the specified stream.
*
* Post-condition:
* 1. If txn creation succeeds, then
* (a) txn node is created in the store,
* (b) txn segments are successfully created on respective segment stores,
* (c) txn is present in the host-txn index of current host,
* (d) txn's timeout is being tracked in timeout service.
*
* 2. If process fails after creating txn node, but before responding to the client, then since txn is
* present in the host-txn index, some other controller process shall abort the txn after maxLeaseValue
*
* 3. If timeout service tracks timeout of specified txn,
* then txn is also present in the host-txn index of current process.
*
* Invariant:
* The following invariants are maintained throughout the execution of createTxn, pingTxn and sealTxn methods.
* 1. If timeout service tracks timeout of a txn, then txn is also present in the host-txn index of current process.
* 2. If txn znode is updated, then txn is also present in the host-txn index of current process.
*
* @param scope scope name.
* @param stream stream name.
* @param lease txn lease.
* @param scaleGracePeriod amount of time for which txn may remain open after scale operation is initiated.
* @param ctx context.
* @return identifier of the created txn.
*/
CompletableFuture<Pair<VersionedTransactionData, List<Segment>>> createTxnBody(final String scope, final String stream, final long lease, final long scaleGracePeriod, final OperationContext ctx) {
// Step 1. Validate parameters.
CompletableFuture<Void> validate = validate(lease, scaleGracePeriod);
long maxExecutionPeriod = Math.min(MAX_EXECUTION_TIME_MULTIPLIER * lease, Duration.ofDays(1).toMillis());
UUID txnId = UUID.randomUUID();
TxnResource resource = new TxnResource(scope, stream, txnId);
// Step 2. Add txn to host-transaction index.
CompletableFuture<Void> addIndex = validate.thenComposeAsync(ignore -> streamMetadataStore.addTxnToIndex(hostId, resource, 0), executor).whenComplete((v, e) -> {
if (e != null) {
log.debug("Txn={}, failed adding txn to host-txn index of host={}", txnId, hostId);
} else {
log.debug("Txn={}, added txn to host-txn index of host={}", txnId, hostId);
}
});
// Step 3. Create txn node in the store.
CompletableFuture<VersionedTransactionData> txnFuture = addIndex.thenComposeAsync(ignore -> streamMetadataStore.createTransaction(scope, stream, txnId, lease, maxExecutionPeriod, scaleGracePeriod, ctx, executor), executor).whenComplete((v, e) -> {
if (e != null) {
log.debug("Txn={}, failed creating txn in store", txnId);
} else {
log.debug("Txn={}, created in store", txnId);
}
});
// Step 4. Notify segment stores about new txn.
CompletableFuture<List<Segment>> segmentsFuture = txnFuture.thenComposeAsync(txnData -> streamMetadataStore.getActiveSegments(scope, stream, txnData.getEpoch(), ctx, executor), executor);
CompletableFuture<Void> notify = segmentsFuture.thenComposeAsync(activeSegments -> notifyTxnCreation(scope, stream, activeSegments, txnId), executor).whenComplete((v, e) -> log.debug("Txn={}, notified segments stores", txnId));
// Step 5. Start tracking txn in timeout service
return notify.whenCompleteAsync((result, ex) -> {
int version = 0;
long executionExpiryTime = System.currentTimeMillis() + maxExecutionPeriod;
if (!txnFuture.isCompletedExceptionally()) {
version = txnFuture.join().getVersion();
executionExpiryTime = txnFuture.join().getMaxExecutionExpiryTime();
}
timeoutService.addTxn(scope, stream, txnId, version, lease, executionExpiryTime, scaleGracePeriod);
log.debug("Txn={}, added to timeout service on host={}", txnId, hostId);
}, executor).thenApplyAsync(v -> new ImmutablePair<>(txnFuture.join(), segmentsFuture.join()), executor);
}
use of io.pravega.controller.store.task.TxnResource in project pravega by pravega.
the class StreamTransactionMetadataTasks method sealTxnBody.
/**
* Seals a txn and transitions it to COMMITTING (resp. ABORTING) state if commit param is true (resp. false).
*
* Post-condition:
* 1. If seal completes successfully, then
* (a) txn state is COMMITTING/ABORTING,
* (b) CommitEvent/AbortEvent is present in the commit stream/abort stream,
* (c) txn is removed from host-txn index,
* (d) txn is removed from the timeout service.
*
* 2. If process fails after transitioning txn to COMMITTING/ABORTING state, but before responding to client, then
* since txn is present in the host-txn index, some other controller process shall put CommitEvent/AbortEvent to
* commit stream/abort stream.
*
* @param host host id. It is different from hostId iff invoked from TxnSweeper for aborting orphaned txn.
* @param scope scope name.
* @param stream stream name.
* @param commit boolean indicating whether to commit txn.
* @param txnId txn id.
* @param version expected version of txn node in store.
* @param ctx context.
* @return Txn status after sealing it.
*/
CompletableFuture<TxnStatus> sealTxnBody(final String host, final String scope, final String stream, final boolean commit, final UUID txnId, final Version version, final String writerId, final long timestamp, final OperationContext ctx) {
Preconditions.checkNotNull(ctx, "Operation context cannot be null");
long requestId = ctx.getRequestId();
TxnResource resource = new TxnResource(scope, stream, txnId);
Optional<Version> versionOpt = Optional.ofNullable(version);
// Step 1. Add txn to current host's index, if it is not already present
CompletableFuture<Void> addIndex = host.equals(hostId) && !timeoutService.containsTxn(scope, stream, txnId) ? // then txn would no longer be open.
streamMetadataStore.addTxnToIndex(hostId, resource, version) : CompletableFuture.completedFuture(null);
addIndex.whenComplete((v, e) -> {
if (e != null) {
log.debug(requestId, "Txn={}, already present/newly added to host-txn index of host={}", txnId, hostId);
} else {
log.debug(requestId, "Txn={}, added txn to host-txn index of host={}", txnId, hostId);
}
});
// Step 2. Seal txn
CompletableFuture<AbstractMap.SimpleEntry<TxnStatus, Integer>> sealFuture = addIndex.thenComposeAsync(x -> streamMetadataStore.sealTransaction(scope, stream, txnId, commit, versionOpt, writerId, timestamp, ctx, executor), executor).whenComplete((v, e) -> {
if (e != null) {
log.debug(requestId, "Txn={}, failed sealing txn", txnId);
} else {
log.debug(requestId, "Txn={}, sealed successfully, commit={}", txnId, commit);
}
});
// Step 3. write event to corresponding stream.
return sealFuture.thenComposeAsync(pair -> {
TxnStatus status = pair.getKey();
switch(status) {
case COMMITTING:
return writeCommitEvent(scope, stream, pair.getValue(), txnId, status, requestId);
case ABORTING:
return writeAbortEvent(scope, stream, pair.getValue(), txnId, status, requestId);
case ABORTED:
case COMMITTED:
return CompletableFuture.completedFuture(status);
case OPEN:
case UNKNOWN:
default:
// exception would be thrown.
return CompletableFuture.completedFuture(status);
}
}, executor).thenComposeAsync(status -> {
// Step 4. Remove txn from timeoutService, and from the index.
timeoutService.removeTxn(scope, stream, txnId);
log.debug(requestId, "Txn={}, removed from timeout service", txnId);
return streamMetadataStore.removeTxnFromIndex(host, resource, true).whenComplete((v, e) -> {
if (e != null) {
log.debug(requestId, "Txn={}, failed removing txn from host-txn index of host={}", txnId, hostId);
} else {
log.debug(requestId, "Txn={}, removed txn from host-txn index of host={}", txnId, hostId);
}
}).thenApply(x -> status);
}, executor);
}
use of io.pravega.controller.store.task.TxnResource in project pravega by pravega.
the class TxnSweeper method failOverTxn.
private CompletableFuture<Result> failOverTxn(String failedHost, TxnResource txn) {
String scope = txn.getScope();
String stream = txn.getStream();
UUID txnId = txn.getTxnId();
log.info("Host = {}, processing transaction {}/{}/{}", failedHost, scope, stream, txnId);
return streamMetadataStore.getTransactionData(scope, stream, txnId, null, executor).handle((r, e) -> {
if (e != null) {
if (Exceptions.unwrap(e) instanceof StoreException.DataNotFoundException) {
// transaction not found, which means it should already have completed. We will ignore such txns
return VersionedTransactionData.EMPTY;
} else {
throw new CompletionException(e);
}
}
return r;
}).thenComposeAsync(txData -> {
int epoch = txData.getEpoch();
switch(txData.getStatus()) {
case OPEN:
return failOverOpenTxn(failedHost, txn).handleAsync((v, e) -> new Result(txn, v, e), executor);
case ABORTING:
return failOverAbortingTxn(failedHost, epoch, txn).handleAsync((v, e) -> new Result(txn, v, e), executor);
case COMMITTING:
return failOverCommittingTxn(failedHost, epoch, txn).handleAsync((v, e) -> new Result(txn, v, e), executor);
case UNKNOWN:
default:
return streamMetadataStore.removeTxnFromIndex(failedHost, txn, true).thenApply(x -> new Result(txn, null, null));
}
}, executor).whenComplete((v, e) -> log.debug("Host = {}, processing transaction {}/{}/{} complete", failedHost, scope, stream, txnId));
}
use of io.pravega.controller.store.task.TxnResource in project pravega by pravega.
the class StreamMetadataStoreTest method txnHostIndexTest.
@Test(timeout = 30000)
public void txnHostIndexTest() {
String host1 = "host1";
String host2 = "host2";
TxnResource txn1 = new TxnResource(scope, stream1, UUID.randomUUID());
TxnResource txn2 = new TxnResource(scope, stream1, UUID.randomUUID());
addTxnToHost(host1, txn1, new Version.IntVersion(0));
Assert.assertEquals(1, store.listHostsOwningTxn().join().size());
Optional<TxnResource> txn = store.getRandomTxnFromIndex(host1).join();
Assert.assertTrue(txn.isPresent());
Assert.assertEquals(txn1.getTxnId().toString(), txn.get().getTxnId().toString());
// Adding a txn again should not fail.
addTxnToHost(host1, txn1, new Version.IntVersion(0));
addTxnToHost(host1, txn2, new Version.IntVersion(5));
Assert.assertEquals(1, store.listHostsOwningTxn().join().size());
// Fetching version of txn not existing in the index should return null.
Assert.assertNull(store.getTxnVersionFromIndex(host1, new TxnResource(scope, stream1, UUID.randomUUID())).join());
txn = store.getRandomTxnFromIndex(host1).join();
Assert.assertTrue(txn.isPresent());
UUID randomTxnId = txn.get().getTxnId();
Assert.assertTrue(randomTxnId.equals(txn1.getTxnId()) || randomTxnId.equals(txn2.getTxnId()));
Assert.assertEquals(scope, txn.get().getScope());
Assert.assertEquals(stream1, txn.get().getStream());
// Test remove txn from index.
store.removeTxnFromIndex(host1, txn1, true).join();
// Test remove is idempotent operation.
store.removeTxnFromIndex(host1, txn1, true).join();
// Test remove last txn from the index.
store.removeTxnFromIndex(host1, txn2, false).join();
Assert.assertEquals(1, store.listHostsOwningTxn().join().size());
// Test remove is idempotent operation.
store.removeTxnFromIndex(host1, txn2, true).join();
Assert.assertTrue(store.listHostsOwningTxn().join().size() <= 1);
// Test removal of txn that was never added.
store.removeTxnFromIndex(host1, new TxnResource(scope, stream1, UUID.randomUUID()), true).join();
// Test host removal.
store.removeHostFromIndex(host1).join();
Assert.assertEquals(0, store.listHostsOwningTxn().join().size());
// Test host removal is idempotent.
store.removeHostFromIndex(host1).join();
Assert.assertEquals(0, store.listHostsOwningTxn().join().size());
// Test removal of host that was never added.
store.removeHostFromIndex(host2).join();
Assert.assertEquals(0, store.listHostsOwningTxn().join().size());
}
use of io.pravega.controller.store.task.TxnResource in project pravega by pravega.
the class ZKStreamMetadataStoreTest method testError.
@Test(timeout = 5000)
public void testError() throws Exception {
String host = "host";
TxnResource txn = new TxnResource("SCOPE", "STREAM1", UUID.randomUUID());
Predicate<Throwable> checker = (Throwable ex) -> ex instanceof StoreException.UnknownException;
cli.close();
testFailure(host, txn, checker);
}
Aggregations