Examples with Operation io.pravega.segmentstore.server.logs.operations.Operation used on opensource projects

Example 26 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class OperationProcessorTests method testWithNoFailures.

/**
 * Tests the ability of the OperationProcessor to process Operations in a failure-free environment.
 */
@Test
public void testWithNoFailures() throws Exception {
    int streamSegmentCount = 50;
    int transactionsPerStreamSegment = 2;
    int appendsPerStreamSegment = 20;
    boolean mergeTransactions = true;
    boolean sealStreamSegments = true;
    @Cleanup TestContext context = new TestContext();
    // Generate some test data.
    HashSet<Long> streamSegmentIds = createStreamSegmentsInMetadata(streamSegmentCount, context.metadata);
    AbstractMap<Long, Long> transactions = createTransactionsInMetadata(streamSegmentIds, transactionsPerStreamSegment, context.metadata);
    List<Operation> operations = generateOperations(streamSegmentIds, transactions, appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, mergeTransactions, sealStreamSegments);
    // Setup an OperationProcessor and start it.
    @Cleanup TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, MAX_DATA_LOG_APPEND_SIZE, executorService());
    dataLog.initialize(TIMEOUT);
    @Cleanup OperationProcessor operationProcessor = new OperationProcessor(context.metadata, context.stateUpdater, dataLog, getNoOpCheckpointPolicy(), executorService());
    operationProcessor.startAsync().awaitRunning();
    // Process all generated operations.
    List<OperationWithCompletion> completionFutures = processOperations(operations, operationProcessor);
    // Wait for all such operations to complete. If any of them failed, this will fail too and report the exception.
    OperationWithCompletion.allOf(completionFutures).join();
    performLogOperationChecks(completionFutures, context.memoryLog, dataLog, context.metadata);
    performMetadataChecks(streamSegmentIds, new HashSet<>(), transactions, completionFutures, context.metadata, mergeTransactions, sealStreamSegments);
    performReadIndexChecks(completionFutures, context.readIndex);
    operationProcessor.stopAsync().awaitTerminated();
}

Example 27 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class OperationProcessorTests method performLogOperationChecks.

private void performLogOperationChecks(Collection<OperationWithCompletion> operations, SequencedItemList<Operation> memoryLog, DurableDataLog dataLog, TruncationMarkerRepository truncationMarkers, int maxCount) throws Exception {
    // Log Operation based checks
    val successfulOps = operations.stream().filter(oc -> !oc.completion.isCompletedExceptionally()).map(oc -> oc.operation).filter(Operation::canSerialize).limit(maxCount).collect(Collectors.toList());
    @Cleanup DataFrameReader<Operation> dataFrameReader = new DataFrameReader<>(dataLog, new OperationSerializer(), CONTAINER_ID);
    long lastSeqNo = -1;
    if (successfulOps.size() > 0) {
        // Writing to the memory log is asynchronous and we don't have any callbacks to know when it was written to.
        // We check periodically until the last item has been written.
        await(() -> memoryLog.read(successfulOps.get(successfulOps.size() - 1).getSequenceNumber() - 1, 1).hasNext(), 10);
    }
    Iterator<Operation> memoryLogIterator = memoryLog.read(-1, operations.size() + 1);
    OperationComparer memoryLogComparer = new OperationComparer(true);
    for (Operation expectedOp : successfulOps) {
        // Verify that the operations have been completed and assigned sequential Sequence Numbers.
        AssertExtensions.assertGreaterThan("Operations were not assigned sequential Sequence Numbers.", lastSeqNo, expectedOp.getSequenceNumber());
        lastSeqNo = expectedOp.getSequenceNumber();
        // MemoryLog: verify that the operations match that of the expected list.
        Assert.assertTrue("No more items left to read from MemoryLog. Expected: " + expectedOp, memoryLogIterator.hasNext());
        // Use memoryLogComparer: we are actually expecting the same object here.
        Operation actual = memoryLogIterator.next();
        memoryLogComparer.assertEquals("Unexpected Operation in MemoryLog.", expectedOp, actual);
        // DataLog: read back using DataFrameReader and verify the operations match that of the expected list.
        DataFrameRecord<Operation> dataFrameRecord = dataFrameReader.getNext();
        Assert.assertNotNull("No more items left to read from DataLog. Expected: " + expectedOp, dataFrameRecord);
        // We are reading the raw operation from the DataFrame, so expect different objects (but same contents).
        OperationComparer.DEFAULT.assertEquals(expectedOp, dataFrameRecord.getItem());
        // Check truncation markers if this is the last Operation to be written.
        LogAddress dataFrameAddress = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber());
        if (dataFrameRecord.getLastFullDataFrameAddress() != null && dataFrameRecord.getLastFullDataFrameAddress().getSequence() != dataFrameRecord.getLastUsedDataFrameAddress().getSequence()) {
            // This operation spans multiple DataFrames. The TruncationMarker should be set on the last DataFrame
            // that ends with a part of it.
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it spans multiple DataFrames.", dataFrameRecord.getLastFullDataFrameAddress(), dataFrameAddress);
        } else if (dataFrameRecord.isLastFrameEntry()) {
            // The operation was the last one in the frame. This is a Truncation Marker.
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is the last entry in a DataFrame.", dataFrameRecord.getLastUsedDataFrameAddress(), dataFrameAddress);
        } else {
            // The operation is not the last in the frame, and it doesn't span multiple frames either.
            // There could be data after it that is not safe to truncate. The correct Truncation Marker is the
            // same as the one for the previous operation.
            LogAddress expectedTruncationMarker = truncationMarkers.getClosestTruncationMarker(expectedOp.getSequenceNumber() - 1);
            Assert.assertEquals("Unexpected truncation marker for Operation SeqNo " + expectedOp.getSequenceNumber() + " when it is in the middle of a DataFrame.", expectedTruncationMarker, dataFrameAddress);
        }
    }
}

Example 28 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class OperationProcessorTests method testWithDataLogFailures.

/**
 * Tests the ability of the OperationProcessor to process Operations when there are DataLog write failures. The expected
 * outcome is that the OperationProcessor will auto-shutdown when such errors are encountered.
 */
@Test
public void testWithDataLogFailures() throws Exception {
    int streamSegmentCount = 10;
    int appendsPerStreamSegment = 80;
    // Fail (asynchronously) after X DataFrame commits (to DataLog).
    int failAfterCommits = 5;
    @Cleanup TestContext context = new TestContext();
    // Generate some test data (no need to complicate ourselves with Transactions here; that is tested in the no-failure test).
    HashSet<Long> streamSegmentIds = createStreamSegmentsInMetadata(streamSegmentCount, context.metadata);
    List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
    // Setup an OperationProcessor and start it.
    @Cleanup TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, MAX_DATA_LOG_APPEND_SIZE, executorService());
    dataLog.initialize(TIMEOUT);
    @Cleanup OperationProcessor operationProcessor = new OperationProcessor(context.metadata, context.stateUpdater, dataLog, getNoOpCheckpointPolicy(), executorService());
    operationProcessor.startAsync().awaitRunning();
    ErrorInjector<Exception> aSyncErrorInjector = new ErrorInjector<>(count -> count >= failAfterCommits, () -> new DurableDataLogException("intentional"));
    dataLog.setAppendErrorInjectors(null, aSyncErrorInjector);
    // Process all generated operations.
    List<OperationWithCompletion> completionFutures = processOperations(operations, operationProcessor);
    // Wait for all such operations to complete. We are expecting exceptions, so verify that we do.
    AssertExtensions.assertThrows("No operations failed.", OperationWithCompletion.allOf(completionFutures)::join, super::isExpectedExceptionForNonDataCorruption);
    // Wait for the OperationProcessor to shutdown with failure.
    ServiceListeners.awaitShutdown(operationProcessor, TIMEOUT, false);
    Assert.assertEquals("Expected the OperationProcessor to fail after DurableDataLogException encountered.", Service.State.FAILED, operationProcessor.state());
    performLogOperationChecks(completionFutures, context.memoryLog, dataLog, context.metadata);
    performMetadataChecks(streamSegmentIds, new HashSet<>(), new HashMap<>(), completionFutures, context.metadata, false, false);
    performReadIndexChecks(completionFutures, context.readIndex);
}

Example 29 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class OperationProcessorTests method testWithDataCorruptionFailures.

/**
 * Tests the ability of the OperationProcessor to process Operations when a simulated DataCorruptionException
 * is generated.
 */
@Test
@SuppressWarnings("checkstyle:CyclomaticComplexity")
public void testWithDataCorruptionFailures() throws Exception {
    // If a DataCorruptionException is thrown for a particular Operation, the OperationQueueProcessor should
    // immediately shut down and stop accepting other ops.
    int streamSegmentCount = 10;
    int appendsPerStreamSegment = 80;
    // Fail Operation at index X.
    int failAtOperationIndex = 123;
    @Cleanup TestContext context = new TestContext();
    // Create a different state updater and Memory log - and use these throughout this test.
    CorruptedMemoryOperationLog corruptedMemoryLog = new CorruptedMemoryOperationLog(failAtOperationIndex);
    MemoryStateUpdater stateUpdater = new MemoryStateUpdater(corruptedMemoryLog, context.readIndex, Runnables.doNothing());
    // Generate some test data (no need to complicate ourselves with Transactions here; that is tested in the no-failure test).
    HashSet<Long> streamSegmentIds = createStreamSegmentsInMetadata(streamSegmentCount, context.metadata);
    List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
    // Setup an OperationProcessor and start it.
    @Cleanup TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, MAX_DATA_LOG_APPEND_SIZE, executorService());
    dataLog.initialize(TIMEOUT);
    @Cleanup OperationProcessor operationProcessor = new OperationProcessor(context.metadata, stateUpdater, dataLog, getNoOpCheckpointPolicy(), executorService());
    operationProcessor.startAsync().awaitRunning();
    // Process all generated operations.
    List<OperationWithCompletion> completionFutures = processOperations(operations, operationProcessor);
    // Wait for the store to fail (and make sure it failed).
    AssertExtensions.assertThrows("Operation Processor did not shut down with failure.", () -> ServiceListeners.awaitShutdown(operationProcessor, true), ex -> ex instanceof IllegalStateException);
    Assert.assertEquals("Unexpected service state after encountering DataCorruptionException.", Service.State.FAILED, operationProcessor.state());
    // Verify that the "right" operations failed, while the others succeeded.
    int successCount = 0;
    boolean encounteredFirstFailure = false;
    for (int i = 0; i < completionFutures.size(); i++) {
        OperationWithCompletion oc = completionFutures.get(i);
        if (!oc.operation.canSerialize()) {
            // Non-serializable operations (i.e., ProbeOperations always complete normally).
            continue;
        }
        // Once an operation failed (in our scenario), no other operation can succeed.
        if (encounteredFirstFailure) {
            Assert.assertTrue("Encountered successful operation after a failed operation: " + oc.operation, oc.completion.isCompletedExceptionally());
        }
        // with it, which is why it's hard to determine precisely what the first expected failed operation is.
        if (oc.completion.isCompletedExceptionally()) {
            // If we do find a failed one in this area, make sure it is failed with DataCorruptionException.
            AssertExtensions.assertThrows("Unexpected exception for failed Operation in the same DataFrame as intentionally failed operation.", oc.completion::join, super::isExpectedExceptionForDataCorruption);
            encounteredFirstFailure = true;
        } else {
            successCount++;
        }
    }
    AssertExtensions.assertGreaterThan("No operation succeeded.", 0, successCount);
    performLogOperationChecks(completionFutures, corruptedMemoryLog, dataLog, context.metadata, failAtOperationIndex - 1);
// There is no point in performing metadata checks. A DataCorruptionException means the Metadata (and the general
// state of the Container) is in an undefined state.
}

Example 30 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testWithBadAttributes.

private void testWithBadAttributes(Function<Collection<AttributeUpdate>, Operation> createOperation) throws Exception {
    final UUID attributeNoUpdate = UUID.randomUUID();
    final UUID attributeReplaceIfGreater = UUID.randomUUID();
    final UUID attributeReplaceIfEquals = UUID.randomUUID();
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    // Append #1.
    Collection<AttributeUpdate> attributeUpdates = new ArrayList<>();
    // Initial add, so it's ok.
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    // Initial Add.
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.Replace, 2));
    val expectedValues = attributeUpdates.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue));
    Operation op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    // Append #2: Try to update attribute that cannot be updated.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 3));
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an unmodifiable attribute.", () -> txn.preProcessOperation(createOperation.apply(attributeUpdates)), ex -> ex instanceof BadAttributeUpdateException);
    // Append #3: Try to update attribute with bad value for ReplaceIfGreater attribute.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 1));
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an attribute with the wrong value for ReplaceIfGreater.", () -> txn.preProcessOperation(createOperation.apply(attributeUpdates)), ex -> ex instanceof BadAttributeUpdateException);
    // Append #4: Try to update attribute with bad value for ReplaceIfEquals attribute.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, 3, 3));
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an attribute with the wrong comparison value for ReplaceIfGreater.", () -> txn.preProcessOperation(createOperation.apply(attributeUpdates)), ex -> ex instanceof BadAttributeUpdateException);
    // Reset the attribute update list to its original state so we can do the final verification.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfGreater, 2, 2));
    verifyAttributeUpdates("after rejected operations", txn, attributeUpdates, expectedValues);
    txn.commit(metadata);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in segment metadata after commit.", expectedValues, metadata.getStreamSegmentMetadata(SEGMENT_ID));
}