use of io.pravega.shared.controller.event.CommitEvent in project pravega by pravega.
the class CommitEventProcessor method process.
@Override
protected void process(CommitEvent event, Position position) {
String scope = event.getScope();
String stream = event.getStream();
int epoch = event.getEpoch();
UUID txnId = event.getTxid();
OperationContext context = streamMetadataStore.createContext(scope, stream);
log.debug("Committing transaction {} on stream {}/{}", event.getTxid(), event.getScope(), event.getStream());
streamMetadataStore.getActiveEpoch(scope, stream, context, false, executor).thenComposeAsync(pair -> {
// complete before transitioning the stream to new epoch.
if (epoch < pair.getKey()) {
return CompletableFuture.completedFuture(null);
} else if (epoch == pair.getKey()) {
// If the transaction's epoch is same as the stream's current epoch, commit it.
return completeCommit(scope, stream, epoch, txnId, context, this.streamMetadataTasks.retrieveDelegationToken());
} else {
// Otherwise, postpone commit operation until the stream transitions to next epoch.
return postponeCommitEvent(event);
}
}).whenCompleteAsync((result, error) -> {
if (error != null) {
log.error("Failed committing transaction {} on stream {}/{}", txnId, scope, stream);
} else {
log.debug("Successfully committed transaction {} on stream {}/{}", txnId, scope, stream);
if (processedEvents != null) {
processedEvents.offer(event);
}
}
}, executor).join();
}
use of io.pravega.shared.controller.event.CommitEvent in project pravega by pravega.
the class StreamTransactionMetadataTasksTest method failOverTests.
@Test
public void failOverTests() throws CheckpointStoreException, InterruptedException {
// Create mock writer objects.
EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
EventStreamReader<CommitEvent> commitReader = commitWriter.getReader();
EventStreamReader<AbortEvent> abortReader = abortWriter.getReader();
consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
// Create test scope and stream.
final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, System.currentTimeMillis()).join());
// Set up txn task for creating transactions from a failedHost.
StreamTransactionMetadataTasks failedTxnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "failedHost", connectionFactory, false, "");
failedTxnTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
// Create 3 transactions from failedHost.
VersionedTransactionData tx1 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
VersionedTransactionData tx2 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
VersionedTransactionData tx3 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
// Ping another txn from failedHost.
UUID txnId = UUID.randomUUID();
streamStore.createTransaction(SCOPE, STREAM, txnId, 10000, 30000, 30000, null, executor).join();
PingTxnStatus pingStatus = failedTxnTasks.pingTxn(SCOPE, STREAM, txnId, 10000, null).join();
VersionedTransactionData tx4 = streamStore.getTransactionData(SCOPE, STREAM, txnId, null, executor).join();
// Validate versions of all txn
Assert.assertEquals(0, tx1.getVersion());
Assert.assertEquals(0, tx2.getVersion());
Assert.assertEquals(0, tx3.getVersion());
Assert.assertEquals(1, tx4.getVersion());
Assert.assertEquals(PingTxnStatus.Status.OK, pingStatus.getStatus());
// Validate the txn index.
Assert.assertEquals(1, streamStore.listHostsOwningTxn().join().size());
// Change state of one txn to COMMITTING.
TxnStatus txnStatus2 = streamStore.sealTransaction(SCOPE, STREAM, tx2.getId(), true, Optional.empty(), null, executor).thenApply(AbstractMap.SimpleEntry::getKey).join();
Assert.assertEquals(TxnStatus.COMMITTING, txnStatus2);
// Change state of another txn to ABORTING.
TxnStatus txnStatus3 = streamStore.sealTransaction(SCOPE, STREAM, tx3.getId(), false, Optional.empty(), null, executor).thenApply(AbstractMap.SimpleEntry::getKey).join();
Assert.assertEquals(TxnStatus.ABORTING, txnStatus3);
// Create transaction tasks for sweeping txns from failedHost.
txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
TxnSweeper txnSweeper = new TxnSweeper(streamStore, txnTasks, 100, executor);
// Before initializing, txnSweeper.sweepFailedHosts would throw an error
AssertExtensions.assertThrows("IllegalStateException before initialization", txnSweeper.sweepFailedProcesses(() -> Collections.singleton("host")), ex -> ex instanceof IllegalStateException);
// Initialize stream writers.
txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
// Validate that txnTasks is ready.
assertTrue(txnTasks.isReady());
// Sweep txns that were being managed by failedHost.
txnSweeper.sweepFailedProcesses(() -> Collections.singleton("host")).join();
// Validate that sweeping completes correctly.
Assert.assertEquals(0, streamStore.listHostsOwningTxn().join().size());
Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx1.getId(), null, executor).join());
Assert.assertEquals(TxnStatus.COMMITTING, streamStore.transactionStatus(SCOPE, STREAM, tx2.getId(), null, executor).join());
Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx3.getId(), null, executor).join());
Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx4.getId(), null, executor).join());
// Create commit and abort event processors.
ConnectionFactory connectionFactory = Mockito.mock(ConnectionFactory.class);
BlockingQueue<CommitEvent> processedCommitEvents = new LinkedBlockingQueue<>();
BlockingQueue<AbortEvent> processedAbortEvents = new LinkedBlockingQueue<>();
createEventProcessor("commitRG", "commitStream", commitReader, commitWriter, () -> new CommitEventProcessor(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, connectionFactory, processedCommitEvents));
createEventProcessor("abortRG", "abortStream", abortReader, abortWriter, () -> new ConcurrentEventProcessor<>(new AbortRequestHandler(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, connectionFactory, processedAbortEvents), executor));
// Wait until the commit event is processed and ensure that the txn state is COMMITTED.
CommitEvent commitEvent = processedCommitEvents.take();
assertEquals(tx2.getId(), commitEvent.getTxid());
assertEquals(TxnStatus.COMMITTED, streamStore.transactionStatus(SCOPE, STREAM, tx2.getId(), null, executor).join());
// Wait until 3 abort events are processed and ensure that the txn state is ABORTED.
Predicate<AbortEvent> predicate = event -> event.getTxid().equals(tx1.getId()) || event.getTxid().equals(tx3.getId()) || event.getTxid().equals(tx4.getId());
AbortEvent abortEvent1 = processedAbortEvents.take();
assertTrue(predicate.test(abortEvent1));
AbortEvent abortEvent2 = processedAbortEvents.take();
assertTrue(predicate.test(abortEvent2));
AbortEvent abortEvent3 = processedAbortEvents.take();
assertTrue(predicate.test(abortEvent3));
assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx1.getId(), null, executor).join());
assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx3.getId(), null, executor).join());
assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx4.getId(), null, executor).join());
}
use of io.pravega.shared.controller.event.CommitEvent in project pravega by pravega.
the class ControllerEventProcessorsTest method testEventKey.
@Test(timeout = 10000)
public void testEventKey() {
UUID txid = UUID.randomUUID();
String scope = "test";
String stream = "test";
AbortEvent abortEvent = new AbortEvent(scope, stream, 0, txid);
CommitEvent commitEvent = new CommitEvent(scope, stream, 0, txid);
assertEquals(abortEvent.getKey(), "test/test");
assertEquals(commitEvent.getKey(), "test/test");
}
use of io.pravega.shared.controller.event.CommitEvent in project pravega by pravega.
the class StreamTransactionMetadataTasksTest method commitAbortTests.
@Test(timeout = 5000)
@SuppressWarnings("unchecked")
public void commitAbortTests() {
// Create mock writer objects.
final List<CompletableFuture<Void>> commitWriterResponses = getWriteResultSequence(5);
final List<CompletableFuture<Void>> abortWriterResponses = getWriteResultSequence(5);
EventStreamWriter<CommitEvent> commitWriter = Mockito.mock(EventStreamWriter.class);
Mockito.when(commitWriter.writeEvent(anyString(), any())).thenAnswer(new SequenceAnswer<>(commitWriterResponses));
EventStreamWriter<AbortEvent> abortWriter = Mockito.mock(EventStreamWriter.class);
Mockito.when(abortWriter.writeEvent(anyString(), any())).thenAnswer(new SequenceAnswer<>(abortWriterResponses));
// Create transaction tasks.
txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
// Create ControllerService.
consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
// Create stream and scope
Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, 0).join());
// Create 2 transactions
final long lease = 5000;
final long scaleGracePeriod = 10000;
VersionedTransactionData txData1 = txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null).join().getKey();
VersionedTransactionData txData2 = txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null).join().getKey();
// Commit the first one
TxnStatus status = txnTasks.commitTxn(SCOPE, STREAM, txData1.getId(), null).join();
Assert.assertEquals(TxnStatus.COMMITTING, status);
// Abort the second one
status = txnTasks.abortTxn(SCOPE, STREAM, txData2.getId(), txData2.getVersion(), null).join();
Assert.assertEquals(TxnStatus.ABORTING, status);
}
use of io.pravega.shared.controller.event.CommitEvent in project pravega by pravega.
the class StreamTransactionMetadataTasksTest method partialTxnCreationTest.
@Test(timeout = 10000)
public void partialTxnCreationTest() {
// Create mock writer objects.
EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
// Create transaction tasks.
txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, SegmentHelperMock.getFailingSegmentHelperMock(), executor, "host", connectionFactory, this.authEnabled, "secret");
txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
// Create ControllerService.
consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
// Create stream and scope
Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, 0).join());
// Create partial transaction
final long lease = 10000;
final long scaleGracePeriod = 10000;
AssertExtensions.assertThrows("Transaction creation fails, although a new txn id gets added to the store", txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null), e -> e instanceof RuntimeException);
// Ensure that exactly one transaction is active on the stream.
Set<UUID> txns = streamStore.getActiveTxns(SCOPE, STREAM, null, executor).join().keySet();
assertEquals(1, txns.size());
// Ensure that transaction state is OPEN.
UUID txn1 = txns.stream().findFirst().get();
assertEquals(TxnStatus.OPEN, streamStore.transactionStatus(SCOPE, STREAM, txn1, null, executor).join());
// Ensure that timeout service knows about the transaction.
assertTrue(txnTasks.getTimeoutService().containsTxn(SCOPE, STREAM, txn1));
}
Aggregations