Examples with StreamSegmentMergedException io.pravega.segmentstore.contracts.StreamSegmentMergedException used on opensource projects

Example 1 with StreamSegmentMergedException

use of io.pravega.segmentstore.contracts.StreamSegmentMergedException in project pravega by pravega.

the class PravegaRequestProcessor method mergeSegments.

@Override
public void mergeSegments(MergeSegments mergeSegments) {
    final String operation = "mergeSegments";
    if (!verifyToken(mergeSegments.getSource(), mergeSegments.getRequestId(), mergeSegments.getDelegationToken(), operation)) {
        return;
    }
    log.info(mergeSegments.getRequestId(), "Merging Segments {} ", mergeSegments);
    // Populate the AttributeUpdates for this mergeSegments operation, if any.
    AttributeUpdateCollection attributeUpdates = new AttributeUpdateCollection();
    if (mergeSegments.getAttributeUpdates() != null) {
        for (WireCommands.ConditionalAttributeUpdate update : mergeSegments.getAttributeUpdates()) {
            attributeUpdates.add(new AttributeUpdate(AttributeId.fromUUID(update.getAttributeId()), AttributeUpdateType.get(update.getAttributeUpdateType()), update.getNewValue(), update.getOldValue()));
        }
    }
    segmentStore.mergeStreamSegment(mergeSegments.getTarget(), mergeSegments.getSource(), attributeUpdates, TIMEOUT).thenAccept(mergeResult -> {
        recordStatForTransaction(mergeResult, mergeSegments.getTarget());
        connection.send(new WireCommands.SegmentsMerged(mergeSegments.getRequestId(), mergeSegments.getTarget(), mergeSegments.getSource(), mergeResult.getTargetSegmentLength()));
    }).exceptionally(e -> {
        if (Exceptions.unwrap(e) instanceof StreamSegmentMergedException) {
            log.info(mergeSegments.getRequestId(), "Stream segment is already merged '{}'.", mergeSegments.getSource());
            segmentStore.getStreamSegmentInfo(mergeSegments.getTarget(), TIMEOUT).thenAccept(properties -> {
                connection.send(new WireCommands.SegmentsMerged(mergeSegments.getRequestId(), mergeSegments.getTarget(), mergeSegments.getSource(), properties.getLength()));
            });
            return null;
        } else if (Exceptions.unwrap(e) instanceof BadAttributeUpdateException) {
            log.debug(mergeSegments.getRequestId(), "Conditional merge failed (Source segment={}, " + "Target segment={}): {}", mergeSegments.getSource(), mergeSegments.getTarget(), e.toString());
            connection.send(new SegmentAttributeUpdated(mergeSegments.getRequestId(), false));
            return null;
        } else {
            return handleException(mergeSegments.getRequestId(), mergeSegments.getSource(), operation, e);
        }
    });
}

Example 2 with StreamSegmentMergedException

use of io.pravega.segmentstore.contracts.StreamSegmentMergedException in project pravega by pravega.

the class StreamSegmentContainerTests method testTransactionOperations.

/**
 * Test the createTransaction, append-to-Transaction, mergeTransaction methods.
 */
@Test
public void testTransactionOperations() throws Exception {
    // Create Transaction and Append to Transaction were partially tested in the Delete test, so we will focus on merge Transaction here.
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    HashMap<String, ArrayList<String>> transactionsBySegment = createTransactions(segmentNames, context);
    activateAllSegments(segmentNames, context);
    transactionsBySegment.values().forEach(s -> activateAllSegments(s, context));
    // 2. Add some appends.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    appendToParentsAndTransactions(segmentNames, transactionsBySegment, lengths, segmentContents, context);
    // 3. Merge all the Transaction.
    Futures.allOf(mergeTransactions(transactionsBySegment, lengths, segmentContents, context, false)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Add more appends (to the parent segments)
    ArrayList<CompletableFuture<Long>> appendFutures = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        for (String segmentName : segmentNames) {
            RefCountByteArraySegment appendData = getAppendData(segmentName, APPENDS_PER_SEGMENT + i);
            appendFutures.add(context.container.append(segmentName, appendData, null, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.getLength());
            recordAppend(segmentName, appendData, segmentContents, null);
            // Verify that we can no longer append to Transaction.
            for (String transactionName : transactionsBySegment.get(segmentName)) {
                AssertExtensions.assertThrows("An append was allowed to a merged Transaction " + transactionName, context.container.append(transactionName, new ByteArraySegment("foo".getBytes()), null, TIMEOUT)::join, ex -> ex instanceof StreamSegmentMergedException || ex instanceof StreamSegmentNotExistsException);
            }
        }
    }
    Futures.allOf(appendFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 5. Verify their contents.
    checkReadIndex(segmentContents, lengths, context);
    // 6. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}

Example 3 with StreamSegmentMergedException

use of io.pravega.segmentstore.contracts.StreamSegmentMergedException in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessStreamSegmentAppend.

// region StreamSegmentAppendOperation
/**
 * Tests the preProcess method with StreamSegmentAppend operations.
 * Scenarios:
 * * Recovery Mode
 * * Non-recovery mode
 * * StreamSegment is Merged (both in-transaction and in-metadata)
 * * StreamSegment is Sealed (both in-transaction and in-metadata)
 */
@Test
public void testPreProcessStreamSegmentAppend() throws Exception {
    val metadata = createMetadata();
    StreamSegmentAppendOperation appendOp = createAppendNoOffset();
    // When everything is OK (in recovery mode) - nothing should change.
    metadata.enterRecoveryMode();
    val txn1 = createUpdateTransaction(metadata);
    txn1.preProcessOperation(appendOp);
    AssertExtensions.assertLessThan("Unexpected StreamSegmentOffset after call to preProcess in recovery mode.", 0, appendOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(appendOp, "call to preProcess in recovery mode");
    Assert.assertEquals("preProcess(Append) seems to have changed the Updater internal state in recovery mode.", SEGMENT_LENGTH, txn1.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("preProcess(Append) seems to have changed the metadata in recovery mode.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    // When everything is OK (no recovery mode).
    metadata.exitRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(appendOp);
    Assert.assertEquals("Unexpected StreamSegmentOffset after call to preProcess in non-recovery mode.", SEGMENT_LENGTH, appendOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(appendOp, "call to preProcess in non-recovery mode");
    Assert.assertEquals("preProcess(Append) seems to have changed the Updater internal state.", SEGMENT_LENGTH, txn2.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("preProcess(Append) seems to have changed the metadata.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    // When StreamSegment is merged (via transaction).
    StreamSegmentAppendOperation transactionAppendOp = new StreamSegmentAppendOperation(SEALED_SOURCE_ID, DEFAULT_APPEND_DATA, null);
    MergeSegmentOperation mergeOp = createMerge();
    txn2.preProcessOperation(mergeOp);
    txn2.acceptOperation(mergeOp);
    Assert.assertFalse("Transaction should not be merged in metadata (yet).", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is merged (in transaction).", () -> txn2.preProcessOperation(transactionAppendOp), ex -> ex instanceof StreamSegmentMergedException);
    // When StreamSegment is merged (via metadata).
    txn2.commit(metadata);
    Assert.assertTrue("Transaction should have been merged in metadata.", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is merged (in metadata).", () -> txn2.preProcessOperation(transactionAppendOp), ex -> ex instanceof StreamSegmentMergedException);
    // When StreamSegment is sealed (via transaction).
    StreamSegmentSealOperation sealOp = createSeal();
    txn2.preProcessOperation(sealOp);
    txn2.acceptOperation(sealOp);
    Assert.assertFalse("StreamSegment should not be sealed in metadata (yet).", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is sealed (in transaction).", () -> txn2.preProcessOperation(createAppendNoOffset()), ex -> ex instanceof StreamSegmentSealedException);
    // When StreamSegment is sealed (via metadata).
    txn2.commit(metadata);
    Assert.assertTrue("StreamSegment should have been sealed in metadata.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Append) when Segment is sealed (in metadata).", () -> txn2.preProcessOperation(createAppendNoOffset()), ex -> ex instanceof StreamSegmentSealedException);
}

Example 4 with StreamSegmentMergedException

use of io.pravega.segmentstore.contracts.StreamSegmentMergedException in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessStreamSegmentSeal.

// endregion
// region StreamSegmentSealOperation
/**
 * Tests the preProcess method with StreamSegmentSeal operations.
 * Scenarios:
 * * Recovery Mode
 * * Non-recovery mode
 * * StreamSegment is Merged (both in-transaction and in-metadata)
 * * StreamSegment is Sealed (both in-transaction and in-metadata)
 */
@Test
public void testPreProcessStreamSegmentSeal() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    StreamSegmentSealOperation sealOp = createSeal();
    // When everything is OK (in recovery mode) - nothing should change.
    metadata.enterRecoveryMode();
    val txn1 = createUpdateTransaction(metadata);
    txn1.preProcessOperation(sealOp);
    AssertExtensions.assertLessThan("Unexpected StreamSegmentLength after call to preProcess in recovery mode.", 0, sealOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(sealOp, "call to preProcess in recovery mode");
    Assert.assertFalse("preProcess(Seal) seems to have changed the Updater internal state in recovery mode.", txn1.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    Assert.assertFalse("preProcess(Seal) seems to have changed the metadata in recovery mode.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    // When everything is OK (no recovery mode).
    metadata.exitRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(sealOp);
    Assert.assertEquals("Unexpected StreamSegmentLength after call to preProcess in non-recovery mode.", SEGMENT_LENGTH, sealOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(sealOp, "call to preProcess in non-recovery mode");
    Assert.assertFalse("preProcess(Seal) seems to have changed the Updater internal state.", txn2.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    Assert.assertFalse("preProcess(Seal) seems to have changed the metadata.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    // When StreamSegment is merged (via transaction).
    StreamSegmentSealOperation transactionSealOp = new StreamSegmentSealOperation(SEALED_SOURCE_ID);
    MergeSegmentOperation mergeOp = createMerge();
    txn2.preProcessOperation(mergeOp);
    txn2.acceptOperation(mergeOp);
    Assert.assertFalse("Transaction should not be merged in metadata (yet).", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Seal) when Segment is merged (in transaction).", () -> txn2.preProcessOperation(transactionSealOp), ex -> ex instanceof StreamSegmentMergedException);
    // When StreamSegment is merged (via metadata).
    txn2.commit(metadata);
    Assert.assertTrue("Transaction should have been merged in metadata.", metadata.getStreamSegmentMetadata(SEALED_SOURCE_ID).isMerged());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Seal) when Segment is merged (in metadata).", () -> txn2.preProcessOperation(transactionSealOp), ex -> ex instanceof StreamSegmentMergedException);
    // When StreamSegment is sealed (via transaction).
    txn2.acceptOperation(sealOp);
    Assert.assertFalse("StreamSegment should not be sealed in metadata (yet).", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Seal) when Segment is sealed (in transaction).", () -> txn2.preProcessOperation(createSeal()), ex -> ex instanceof StreamSegmentSealedException);
    // When StreamSegment is sealed (via metadata).
    txn2.commit(metadata);
    Assert.assertTrue("StreamSegment should have been sealed in metadata.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    AssertExtensions.assertThrows("Unexpected behavior for preProcess(Seal) when Segment is sealed (in metadata).", () -> txn2.preProcessOperation(createSeal()), ex -> ex instanceof StreamSegmentSealedException);
}

Example 5 with StreamSegmentMergedException

use of io.pravega.segmentstore.contracts.StreamSegmentMergedException in project pravega by pravega.

the class PravegaRequestProcessorTest method testMetricsOnSegmentMerge.

@Test(timeout = 20000)
public void testMetricsOnSegmentMerge() throws Exception {
    String streamSegmentName = "scope/stream/txnSegment";
    UUID txnId = UUID.randomUUID();
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = spy(serviceBuilder.createStreamSegmentService());
    ServerConnection connection = mock(ServerConnection.class);
    doReturn(Futures.failedFuture(new StreamSegmentMergedException(streamSegmentName))).when(store).sealStreamSegment(anyString(), any());
    // test txn segment merge
    CompletableFuture<MergeStreamSegmentResult> txnFuture = CompletableFuture.completedFuture(createMergeStreamSegmentResult(streamSegmentName, txnId));
    doReturn(txnFuture).when(store).mergeStreamSegment(anyString(), anyString(), any(), any());
    SegmentStatsRecorder recorderMock = mock(SegmentStatsRecorder.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, mock(TableStore.class), new TrackedConnection(connection), recorderMock, TableSegmentStatsRecorder.noOp(), new PassingTokenVerifier(), false);
    processor.createSegment(new WireCommands.CreateSegment(0, streamSegmentName, WireCommands.CreateSegment.NO_SCALE, 0, "", 0));
    String transactionName = NameUtils.getTransactionNameFromId(streamSegmentName, txnId);
    processor.createSegment(new WireCommands.CreateSegment(1, transactionName, WireCommands.CreateSegment.NO_SCALE, 0, "", 0));
    processor.mergeSegments(new WireCommands.MergeSegments(2, streamSegmentName, transactionName, ""));
    verify(recorderMock).merge(streamSegmentName, 100L, 10, streamSegmentName.hashCode());
}