Examples with StreamSegmentMergedException io.pravega.segmentstore.contracts.StreamSegmentMergedException used on opensource projects

Example 6 with StreamSegmentMergedException

use of io.pravega.segmentstore.contracts.StreamSegmentMergedException in project pravega by pravega.

the class StreamSegmentContainerTests method testTransactionOperations.

/**
 * Test the createTransaction, append-to-Transaction, mergeTransaction methods.
 */
@Test
public void testTransactionOperations() throws Exception {
    // Create Transaction and Append to Transaction were partially tested in the Delete test, so we will focus on merge Transaction here.
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    HashMap<String, ArrayList<String>> transactionsBySegment = createTransactions(segmentNames, context);
    activateAllSegments(segmentNames, context);
    transactionsBySegment.values().forEach(s -> activateAllSegments(s, context));
    // 2. Add some appends.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    appendToParentsAndTransactions(segmentNames, transactionsBySegment, lengths, segmentContents, context);
    // 3. Merge all the Transaction.
    Futures.allOf(mergeTransactions(transactionsBySegment, lengths, segmentContents, context, false)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Add more appends (to the parent segments)
    ArrayList<CompletableFuture<Long>> appendFutures = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        for (String segmentName : segmentNames) {
            RefCountByteArraySegment appendData = getAppendData(segmentName, APPENDS_PER_SEGMENT + i);
            appendFutures.add(context.container.append(segmentName, appendData, null, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.getLength());
            recordAppend(segmentName, appendData, segmentContents, null);
            // Verify that we can no longer append to Transaction.
            for (String transactionName : transactionsBySegment.get(segmentName)) {
                AssertExtensions.assertThrows("An append was allowed to a merged Transaction " + transactionName, context.container.append(transactionName, new ByteArraySegment("foo".getBytes()), null, TIMEOUT)::join, ex -> ex instanceof StreamSegmentMergedException || ex instanceof StreamSegmentNotExistsException);
            }
        }
    }
    Futures.allOf(appendFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 5. Verify their contents.
    checkReadIndex(segmentContents, lengths, context);
    // 6. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}

Example 7 with StreamSegmentMergedException

use of io.pravega.segmentstore.contracts.StreamSegmentMergedException in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessStreamSegmentAppend.

// region StreamSegmentAppendOperation
/**
 * Tests the preProcess method with StreamSegmentAppend operations.
 * Scenarios:
 * * Recovery Mode
 * * Non-recovery mode
 * * StreamSegment is Merged (both in-transaction and in-metadata)
 * * StreamSegment is Sealed (both in-transaction and in-metadata)
 */
@Test
public void testPreProcessStreamSegmentAppend() throws Exception {
    val metadata = createMetadata();
    StreamSegmentAppendOperation appendOp = createAppendNoOffset();
    // When everything is OK (in recovery mode) - nothing should change.
    metadata.enterRecoveryMode();
    val txn1 = createUpdateTransaction(metadata);
    txn1.preProcessOperation(appendOp);
    AssertExtensions.assertLessThan("Unexpected StreamSegmentOffset after call to preProcess in recovery mode.", 0, appendOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(appendOp, "call to preProcess in recovery mode");
    Assert.assertEquals("preProcess(Append) seems to have changed the Updater internal state in recovery mode.", SEGMENT_LENGTH, txn1.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("preProcess(Append) seems to have changed the metadata in recovery mode.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    // When everything is OK (no recovery mode).
    metadata.exitRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(appendOp);
    Assert.assertEquals("Unexpected StreamSegmentOffset after call to preProcess in non-recovery mode.", SEGMENT_LENGTH, appendOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(appendOp, "call to preProcess in non-recovery mode");
    Assert.assertEquals("preProcess(Append) seems to have changed the Updater internal state.", SEGMENT_LENGTH, txn2.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("preProcess(Append) seems to have changed the metadata.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    // When StreamSegment is merged (via transaction).
    StreamSegmentAppendOperation transactionAppendOp = new StreamSegmentAppendOperation(SEALED_SOURCE_ID, DEFAULT_APPEND_DATA, null);
    MergeSegmentOperation mergeOp = createMerge();
    txn2.preProcessOperation(mergeOp);
    txn2.acceptOperation(mergeOp);
    Assert.assertFalse("Transaction should not be merged in metadata (yet).", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is merged (in transaction).", () -> txn2.preProcessOperation(transactionAppendOp), ex -> ex instanceof StreamSegmentMergedException);
    // When StreamSegment is merged (via metadata).
    txn2.commit(metadata);
    Assert.assertTrue("Transaction should have been merged in metadata.", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is merged (in metadata).", () -> txn2.preProcessOperation(transactionAppendOp), ex -> ex instanceof StreamSegmentMergedException);
    // When StreamSegment is sealed (via transaction).
    StreamSegmentSealOperation sealOp = createSeal();
    txn2.preProcessOperation(sealOp);
    txn2.acceptOperation(sealOp);
    Assert.assertFalse("StreamSegment should not be sealed in metadata (yet).", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is sealed (in transaction).", () -> txn2.preProcessOperation(createAppendNoOffset()), ex -> ex instanceof StreamSegmentSealedException);
    // When StreamSegment is sealed (via metadata).
    txn2.commit(metadata);
    Assert.assertTrue("StreamSegment should have been sealed in metadata.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is sealed (in metadata).", () -> txn2.preProcessOperation(createAppendNoOffset()), ex -> ex instanceof StreamSegmentSealedException);
}

Example 8 with StreamSegmentMergedException

use of io.pravega.segmentstore.contracts.StreamSegmentMergedException in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessMergeTransaction.

// endregion
// region MergeTransactionOperation
/**
 * Tests the preProcess method with MergeTransactionOperations.
 * Scenarios:
 * * Recovery/non-recovery mode
 * * Target StreamSegment is sealed
 * * Target StreamSegment is a Transaction
 * * Transaction StreamSegment is already merged
 * * Transaction StreamSegment is not sealed
 */
@Test
public void testPreProcessMergeTransaction() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    // When everything is OK (recovery mode).
    MergeTransactionOperation recoveryMergeOp = createMerge();
    metadata.enterRecoveryMode();
    val txn1 = createUpdateTransaction(metadata);
    AssertExtensions.assertThrows("preProcess(Merge) handled an operation with no Transaction StreamSegment Length set.", () -> txn1.preProcessOperation(createMerge()), ex -> ex instanceof MetadataUpdateException);
    // In recovery mode, the updater does not set the length; it just validates that it has one.
    recoveryMergeOp.setLength(metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).getLength());
    txn1.preProcessOperation(recoveryMergeOp);
    AssertExtensions.assertLessThan("Unexpected Target StreamSegmentOffset after call to preProcess in recovery mode.", 0, recoveryMergeOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(recoveryMergeOp, "call to preProcess in recovery mode");
    Assert.assertFalse("preProcess(Merge) seems to have changed the Updater internal state in recovery mode.", txn1.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    Assert.assertFalse("preProcess(Merge) seems to have changed the metadata in recovery mode.", metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    // When everything is OK (non-recovery mode).
    MergeTransactionOperation mergeOp = createMerge();
    metadata.exitRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(mergeOp);
    Assert.assertEquals("Unexpected Transaction StreamSegmentLength after call to preProcess in non-recovery mode.", SEALED_TRANSACTION_LENGTH, mergeOp.getLength());
    Assert.assertEquals("Unexpected Target StreamSegmentOffset after call to preProcess in non-recovery mode.", SEGMENT_LENGTH, mergeOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(mergeOp, "call to preProcess in non-recovery mode");
    Assert.assertFalse("preProcess(Merge) seems to have changed the Updater internal state in non-recovery mode.", txn2.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    Assert.assertFalse("preProcess(Merge) seems to have changed the metadata in non-recovery mode.", metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID).isMerged());
    // When Target StreamSegment is sealed.
    StreamSegmentSealOperation sealTargetOp = createSeal();
    txn2.preProcessOperation(sealTargetOp);
    txn2.acceptOperation(sealTargetOp);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Target StreamSegment is sealed.", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentSealedException);
    // Rollback the seal
    txn2.clear();
    // When Target StreamSegment is a Transaction.
    MergeTransactionOperation mergeToTransactionOp = new MergeTransactionOperation(NOTSEALED_TRANSACTION_ID, SEALED_TRANSACTION_ID);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Target StreamSegment is a Transaction.", () -> txn2.preProcessOperation(mergeToTransactionOp), ex -> ex instanceof MetadataUpdateException);
    // When Transaction is not sealed.
    MergeTransactionOperation mergeNonSealed = new MergeTransactionOperation(NOTSEALED_TRANSACTION_ID, SEGMENT_ID);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is not sealed.", () -> txn2.preProcessOperation(mergeNonSealed), ex -> ex instanceof StreamSegmentNotSealedException);
    // When Transaction is already merged.
    txn2.preProcessOperation(mergeOp);
    txn2.acceptOperation(mergeOp);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is already merged (in transaction).", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentMergedException);
    txn2.commit(metadata);
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Merge) when Transaction StreamSegment is already merged (in metadata).", () -> txn2.preProcessOperation(createMerge()), ex -> ex instanceof StreamSegmentMergedException);
}

Example 9 with StreamSegmentMergedException

use of io.pravega.segmentstore.contracts.StreamSegmentMergedException in project pravega by pravega.

the class PravegaRequestProcessor method commitTransaction.

@Override
public void commitTransaction(CommitTransaction commitTx) {
    String transactionName = StreamSegmentNameUtils.getTransactionNameFromId(commitTx.getSegment(), commitTx.getTxid());
    long requestId = commitTx.getRequestId();
    log.debug("Committing transaction {} ", commitTx);
    if (!verifyToken(commitTx.getSegment(), commitTx.getRequestId(), commitTx.getDelegationToken(), READ_UPDATE, "Commit Transaction")) {
        return;
    }
    // Seal and Merge can execute concurrently, as long as they are invoked in the correct order (first Seal, then Merge).
    // If Seal fails for whatever reason (except already sealed), then Merge will also fail because the txn is not sealed,
    // but invoking them in parallel does provide benefits in terms of reduced latency.
    val seal = segmentStore.sealStreamSegment(transactionName, TIMEOUT).exceptionally(this::ignoreSegmentSealed).thenCompose(v -> recordStatForTransaction(transactionName, commitTx.getSegment()).exceptionally(e -> {
        // gobble up any errors from stat recording so we do not affect rest of the flow.
        log.error("exception while computing stats while merging txn {}", e);
        return null;
    }));
    val merge = segmentStore.mergeTransaction(transactionName, TIMEOUT).thenAccept(v -> connection.send(new TransactionCommitted(requestId, commitTx.getSegment(), commitTx.getTxid())));
    CompletableFuture.allOf(seal, merge).exceptionally(e -> {
        if (Exceptions.unwrap(e) instanceof StreamSegmentMergedException) {
            log.info("Stream segment is already merged '{}'.", transactionName);
            connection.send(new TransactionCommitted(requestId, commitTx.getSegment(), commitTx.getTxid()));
            return null;
        } else {
            return handleException(requestId, transactionName, "Commit transaction", e);
        }
    });
}

Example 10 with StreamSegmentMergedException

use of io.pravega.segmentstore.contracts.StreamSegmentMergedException in project pravega by pravega.

the class PravegaRequestProcessor method mergeSegmentsBatch.

@Override
public void mergeSegmentsBatch(WireCommands.MergeSegmentsBatch mergeSegments) {
    final String operation = "mergeSegmentsBatch";
    List<String> sources = mergeSegments.getSourceSegmentIds();
    if (!verifyTokenForUpdate(mergeSegments.getTargetSegmentId(), mergeSegments.getRequestId(), mergeSegments.getDelegationToken(), operation)) {
        return;
    }
    for (String s : sources) {
        if (!verifyToken(s, mergeSegments.getRequestId(), mergeSegments.getDelegationToken(), operation)) {
            return;
        }
    }
    log.info(mergeSegments.getRequestId(), "Merging Segments Batch in-order {} ", mergeSegments);
    Futures.allOfWithResults(sources.stream().map(source -> Futures.handleCompose(segmentStore.mergeStreamSegment(mergeSegments.getTargetSegmentId(), source, TIMEOUT), (r, e) -> {
        if (e != null) {
            Throwable unwrap = Exceptions.unwrap(e);
            if (unwrap instanceof StreamSegmentMergedException) {
                log.info(mergeSegments.getRequestId(), "Stream segment already merged '{}'.", source);
                return segmentStore.getStreamSegmentInfo(mergeSegments.getTargetSegmentId(), TIMEOUT).thenApply(SegmentProperties::getLength);
            }
            if (unwrap instanceof StreamSegmentNotExistsException) {
                StreamSegmentNotExistsException ex = (StreamSegmentNotExistsException) unwrap;
                if (ex.getStreamSegmentName().equals(source)) {
                    log.info(mergeSegments.getRequestId(), "Stream segment already merged '{}'.", source);
                    return segmentStore.getStreamSegmentInfo(mergeSegments.getTargetSegmentId(), TIMEOUT).thenApply(SegmentProperties::getLength);
                }
            }
            throw new CompletionException(e);
        } else {
            recordStatForTransaction(r, mergeSegments.getTargetSegmentId());
            return CompletableFuture.completedFuture(r.getTargetSegmentLength());
        }
    })).collect(Collectors.toUnmodifiableList())).thenAccept(mergeResults -> {
        connection.send(new WireCommands.SegmentsBatchMerged(mergeSegments.getRequestId(), mergeSegments.getTargetSegmentId(), sources, mergeResults));
    }).exceptionally(e -> {
        log.debug("error");
        return handleException(mergeSegments.getRequestId(), mergeSegments.getTargetSegmentId(), operation, e);
    });
}