Examples with StorageOperation io.pravega.segmentstore.server.logs.operations.StorageOperation used on opensource projects

Example 16 with StorageOperation

use of io.pravega.segmentstore.server.logs.operations.StorageOperation in project pravega by pravega.

the class MemoryStateUpdaterTests method testProcess.

/**
 * Tests the functionality of the process() method.
 */
@Test
public void testProcess() throws Exception {
    int segmentCount = 10;
    int operationCountPerType = 5;
    // Add to MTL + Add to ReadIndex (append; beginMerge).
    SequencedItemList<Operation> opLog = new SequencedItemList<>();
    ArrayList<TestReadIndex.MethodInvocation> methodInvocations = new ArrayList<>();
    TestReadIndex readIndex = new TestReadIndex(methodInvocations::add);
    AtomicInteger flushCallbackCallCount = new AtomicInteger();
    MemoryStateUpdater updater = new MemoryStateUpdater(opLog, readIndex, flushCallbackCallCount::incrementAndGet);
    ArrayList<Operation> operations = populate(updater, segmentCount, operationCountPerType);
    // Verify they were properly processed.
    int triggerFutureCount = (int) methodInvocations.stream().filter(mi -> mi.methodName.equals(TestReadIndex.TRIGGER_FUTURE_READS)).count();
    int addCount = methodInvocations.size() - triggerFutureCount;
    Assert.assertEquals("Unexpected number of items added to ReadIndex.", operations.size() - segmentCount * operationCountPerType, addCount);
    Assert.assertEquals("Unexpected number of calls to the ReadIndex triggerFutureReads method.", 1, triggerFutureCount);
    Assert.assertEquals("Unexpected number of calls to the flushCallback provided in the constructor.", 1, flushCallbackCallCount.get());
    // Verify add calls.
    Iterator<Operation> logIterator = opLog.read(-1, operations.size());
    int currentIndex = -1;
    int currentReadIndex = -1;
    while (logIterator.hasNext()) {
        currentIndex++;
        Operation expected = operations.get(currentIndex);
        Operation actual = logIterator.next();
        if (expected instanceof StorageOperation) {
            currentReadIndex++;
            TestReadIndex.MethodInvocation invokedMethod = methodInvocations.get(currentReadIndex);
            if (expected instanceof StreamSegmentAppendOperation) {
                Assert.assertTrue("StreamSegmentAppendOperation was not added as a CachedStreamSegmentAppendOperation to the Memory Log.", actual instanceof CachedStreamSegmentAppendOperation);
                StreamSegmentAppendOperation appendOp = (StreamSegmentAppendOperation) expected;
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was not added to the ReadIndex.", TestReadIndex.APPEND, invokedMethod.methodName);
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getStreamSegmentId(), invokedMethod.args.get("streamSegmentId"));
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getStreamSegmentOffset(), invokedMethod.args.get("offset"));
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getData(), invokedMethod.args.get("data"));
            } else if (expected instanceof MergeTransactionOperation) {
                MergeTransactionOperation mergeOp = (MergeTransactionOperation) expected;
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was not added to the ReadIndex.", TestReadIndex.BEGIN_MERGE, invokedMethod.methodName);
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getStreamSegmentId(), invokedMethod.args.get("targetStreamSegmentId"));
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getStreamSegmentOffset(), invokedMethod.args.get("offset"));
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getTransactionSegmentId(), invokedMethod.args.get("sourceStreamSegmentId"));
            }
        }
    }
    // Verify triggerFutureReads args.
    @SuppressWarnings("unchecked") Collection<Long> triggerSegmentIds = (Collection<Long>) methodInvocations.stream().filter(mi -> mi.methodName.equals(TestReadIndex.TRIGGER_FUTURE_READS)).findFirst().get().args.get("streamSegmentIds");
    val expectedSegmentIds = operations.stream().filter(op -> op instanceof SegmentOperation).map(op -> ((SegmentOperation) op).getStreamSegmentId()).collect(Collectors.toSet());
    AssertExtensions.assertContainsSameElements("ReadIndex.triggerFutureReads() was called with the wrong set of StreamSegmentIds.", expectedSegmentIds, triggerSegmentIds);
    // Test DataCorruptionException.
    AssertExtensions.assertThrows("MemoryStateUpdater accepted an operation that was out of order.", // This does not have a SequenceNumber set, so it should trigger a DCE.
    () -> updater.process(new MergeTransactionOperation(1, 2)), ex -> ex instanceof DataCorruptionException);
}

Example 17 with StorageOperation

use of io.pravega.segmentstore.server.logs.operations.StorageOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testRollback.

/**
 * Tests the ability of the ContainerMetadataUpdateTransaction to rollback all outstanding changes.
 */
@Test
public void testRollback() throws Exception {
    // Create a couple of operations, commit them, and then create a few more appends, merge a Transaction and seal the parent stream.
    // Then call rollback(); verify no changes have been applied after the call to commit().
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    StreamSegmentAppendOperation committedAppend = createAppendNoOffset();
    txn.preProcessOperation(committedAppend);
    txn.acceptOperation(committedAppend);
    // This is the last extent of the modifications to the metadata.
    txn.commit(metadata);
    int appendCount = 500;
    ArrayList<StorageOperation> operations = new ArrayList<>();
    for (int i = 0; i < appendCount; i++) {
        operations.add(createAppendNoOffset());
    }
    operations.add(createMerge());
    operations.add(createSeal());
    long seqNo = 0;
    for (StorageOperation op : operations) {
        txn.preProcessOperation(op);
        op.setSequenceNumber(++seqNo);
        txn.acceptOperation(op);
    }
    txn.clear();
    long expectedLength = SEGMENT_LENGTH + DEFAULT_APPEND_DATA.length;
    // Verify metadata is untouched and that the updater has truly rolled back.
    SegmentMetadata parentMetadata = metadata.getStreamSegmentMetadata(SEGMENT_ID);
    Assert.assertEquals("Unexpected Length in metadata after rollback.", expectedLength, parentMetadata.getLength());
    Assert.assertFalse("Unexpected value for isSealed in metadata after rollback.", parentMetadata.isSealed());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Parent after rollback.", 0, parentMetadata);
    SegmentMetadata transactionMetadata = metadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID);
    Assert.assertFalse("Unexpected value for isMerged in transaction segment metadata after rollback.", transactionMetadata.isMerged());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Transaction segment after rollback.", 0, transactionMetadata);
    // Now the updater
    parentMetadata = txn.getStreamSegmentMetadata(SEGMENT_ID);
    Assert.assertEquals("Unexpected Length in transaction after rollback.", expectedLength, parentMetadata.getLength());
    Assert.assertFalse("Unexpected value for isSealed in transaction after rollback.", parentMetadata.isSealed());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Parent (txn) after rollback.", 0, parentMetadata);
    transactionMetadata = txn.getStreamSegmentMetadata(SEALED_TRANSACTION_ID);
    Assert.assertFalse("Unexpected value for isMerged in transaction segment in update transaction after rollback.", transactionMetadata.isMerged());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Transaction segment in update transaction after rollback.", 0, transactionMetadata);
}

Example 18 with StorageOperation

use of io.pravega.segmentstore.server.logs.operations.StorageOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessAndAcceptWithInvalidSegmentId.

// endregion
// region Other tests
/**
 * Tests the behavior of preProcessOperation and acceptOperation when encountering an invalid StreamSegmentId, or
 * when encountering a StreamSegment Id for a deleted StreamSegment.
 */
@Test
public void testPreProcessAndAcceptWithInvalidSegmentId() throws Exception {
    UpdateableContainerMetadata metadata = createBlankMetadata();
    val txn = createUpdateTransaction(metadata);
    ArrayList<StorageOperation> testOperations = new ArrayList<>();
    testOperations.add(createAppendNoOffset());
    testOperations.add(createSeal());
    testOperations.add(createMerge());
    for (StorageOperation op : testOperations) {
        AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when processing an operation for a non-existent Segment: " + op, () -> txn.preProcessOperation(op), ex -> ex instanceof MetadataUpdateException);
        AssertExtensions.assertThrows("Unexpected behavior from acceptOperation when processing an operation for a non-existent Segment: " + op, () -> txn.acceptOperation(op), ex -> ex instanceof MetadataUpdateException);
    }
    // If the StreamSegment was previously marked as deleted.
    UpdateableSegmentMetadata segmentMetadata = metadata.mapStreamSegmentId("foo", SEGMENT_ID);
    segmentMetadata.markDeleted();
    for (StorageOperation op : testOperations) {
        AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when processing an operation for deleted Segment: " + op, () -> txn.preProcessOperation(op), ex -> ex instanceof StreamSegmentNotExistsException);
    }
}

Example 19 with StorageOperation

use of io.pravega.segmentstore.server.logs.operations.StorageOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testCommit.

private void testCommit(UpdateableContainerMetadata baseMetadata, UpdateableContainerMetadata targetMetadata) throws Exception {
    // Create a few appends, merge a Transaction and seal the parent stream. Verify all changes have been applied after
    // a call to commit().
    int appendCount = 500;
    ArrayList<StorageOperation> operations = new ArrayList<>();
    for (int i = 0; i < appendCount; i++) {
        operations.add(createAppendNoOffset());
    }
    operations.add(createMerge());
    operations.add(createSeal());
    baseMetadata.getStreamSegmentMetadata(SEGMENT_ID).updateAttributes(Collections.singletonMap(Attributes.CREATION_TIME, 0L));
    val txn = createUpdateTransaction(baseMetadata);
    long expectedLastUsedParent = -1;
    long expectedLastUsedTransaction = -1;
    long seqNo = 0;
    for (StorageOperation op : operations) {
        txn.preProcessOperation(op);
        op.setSequenceNumber(++seqNo);
        txn.acceptOperation(op);
        if (op.getStreamSegmentId() == SEGMENT_ID) {
            expectedLastUsedParent = op.getSequenceNumber();
        }
        if (op instanceof MergeTransactionOperation) {
            expectedLastUsedParent = op.getSequenceNumber();
            expectedLastUsedTransaction = op.getSequenceNumber();
        }
    }
    txn.commit(targetMetadata);
    Assert.assertEquals("commit() seems to have modified the metadata sequence number while not in recovery mode.", ContainerMetadata.INITIAL_OPERATION_SEQUENCE_NUMBER, targetMetadata.nextOperationSequenceNumber() - 1);
    long expectedLength = SEGMENT_LENGTH + appendCount * DEFAULT_APPEND_DATA.length + SEALED_TRANSACTION_LENGTH;
    SegmentMetadata parentMetadata = targetMetadata.getStreamSegmentMetadata(SEGMENT_ID);
    Assert.assertEquals("Unexpected Length in metadata after commit.", expectedLength, parentMetadata.getLength());
    Assert.assertTrue("Unexpected value for isSealed in metadata after commit.", parentMetadata.isSealed());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Parent after commit.", expectedLastUsedParent, parentMetadata);
    SegmentMetadata transactionMetadata = targetMetadata.getStreamSegmentMetadata(SEALED_TRANSACTION_ID);
    Assert.assertTrue("Unexpected value for isSealed in Transaction metadata after commit.", transactionMetadata.isSealed());
    Assert.assertTrue("Unexpected value for isMerged in Transaction metadata after commit.", transactionMetadata.isMerged());
    Assert.assertEquals("Unexpected number of attributes for parent segment.", 1, parentMetadata.getAttributes().size());
    Assert.assertEquals("Unexpected number of attributes for transaction.", 0, transactionMetadata.getAttributes().size());
    checkLastKnownSequenceNumber("Unexpected lastUsed for Transaction after commit.", expectedLastUsedTransaction, transactionMetadata);
}

Example 20 with StorageOperation

use of io.pravega.segmentstore.server.logs.operations.StorageOperation in project pravega by pravega.

the class SegmentAggregatorTests method testReconcileAppends.

// endregion
// region Unknown outcome operation reconciliation
/**
 * Tests the ability of the SegmentAggregator to reconcile AppendOperations (Cached/NonCached).
 */
@Test
public void testReconcileAppends() throws Exception {
    final WriterConfig config = DEFAULT_CONFIG;
    final int appendCount = 1000;
    final int failEvery = 3;
    @Cleanup TestContext context = new TestContext(config);
    context.storage.create(context.segmentAggregator.getMetadata().getName(), TIMEOUT).join();
    context.segmentAggregator.initialize(TIMEOUT).join();
    // The writes always succeed, but every few times we return some random error, indicating that they didn't.
    AtomicInteger writeCount = new AtomicInteger();
    AtomicReference<Exception> setException = new AtomicReference<>();
    context.storage.setWriteInterceptor((segmentName, offset, data, length, storage) -> {
        if (writeCount.incrementAndGet() % failEvery == 0) {
            // Time to wreak some havoc.
            return storage.write(writeHandle(segmentName), offset, data, length, TIMEOUT).thenAccept(v -> {
                IntentionalException ex = new IntentionalException(String.format("S=%s,O=%d,L=%d", segmentName, offset, length));
                setException.set(ex);
                throw ex;
            });
        } else {
            setException.set(null);
            return null;
        }
    });
    @Cleanup ByteArrayOutputStream writtenData = new ByteArrayOutputStream();
    for (int i = 0; i < appendCount; i++) {
        // Add another operation and record its length.
        StorageOperation appendOp = generateAppendAndUpdateMetadata(i, SEGMENT_ID, context);
        context.segmentAggregator.add(appendOp);
        getAppendData(appendOp, writtenData, context);
    }
    // Force a flush by incrementing the time by a lot.
    context.increaseTime(config.getFlushThresholdTime().toMillis() + 1);
    while (context.segmentAggregator.mustFlush()) {
        // Call flush() and inspect the result.
        FlushResult flushResult = null;
        try {
            flushResult = context.segmentAggregator.flush(TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            Assert.assertNull("An exception was expected, but none was thrown.", setException.get());
            Assert.assertNotNull("No FlushResult provided.", flushResult);
        } catch (Exception ex) {
            if (setException.get() != null) {
                Assert.assertEquals("Unexpected exception thrown.", setException.get(), Exceptions.unwrap(ex));
            } else {
                // Only expecting a BadOffsetException after our own injected exception.
                Throwable realEx = Exceptions.unwrap(ex);
                Assert.assertTrue("Unexpected exception thrown: " + realEx, realEx instanceof BadOffsetException);
            }
        }
        // Check flush result.
        if (flushResult != null) {
            AssertExtensions.assertGreaterThan("Not enough bytes were flushed (time threshold).", 0, flushResult.getFlushedBytes());
            Assert.assertEquals("Not expecting any merged bytes in this test.", 0, flushResult.getMergedBytes());
        }
        // Force a flush by incrementing the time by a lot.
        context.increaseTime(config.getFlushThresholdTime().toMillis() + 1);
    }
    // Verify data.
    byte[] expectedData = writtenData.toByteArray();
    byte[] actualData = new byte[expectedData.length];
    long storageLength = context.storage.getStreamSegmentInfo(context.segmentAggregator.getMetadata().getName(), TIMEOUT).join().getLength();
    Assert.assertEquals("Unexpected number of bytes flushed to Storage.", expectedData.length, storageLength);
    context.storage.read(readHandle(context.segmentAggregator.getMetadata().getName()), 0, actualData, 0, actualData.length, TIMEOUT).join();
    Assert.assertArrayEquals("Unexpected data written to storage.", expectedData, actualData);
}