Examples with SegmentOperation io.pravega.segmentstore.server.logs.operations.SegmentOperation used on opensource projects

Example 1 with SegmentOperation

use of io.pravega.segmentstore.server.logs.operations.SegmentOperation in project pravega by pravega.

the class MemoryStateUpdater method process.

/**
 * Processes the given operations and applies them to the ReadIndex and InMemory OperationLog.
 *
 * @param operations An Iterator iterating over the operations to process (in sequence).
 * @throws DataCorruptionException If a serious, non-recoverable, data corruption was detected, such as trying to
 *                                 append operations out of order.
 */
void process(Iterator<Operation> operations) throws DataCorruptionException {
    HashSet<Long> segmentIds = new HashSet<>();
    while (operations.hasNext()) {
        Operation op = operations.next();
        process(op);
        if (op instanceof SegmentOperation) {
            // Record recent activity on stream segment, if applicable. This should be recorded for any kind
            // of Operation that touches a Segment, since when we issue 'triggerFutureReads' on the readIndex,
            // it should include 'sealed' StreamSegments too - any Future Reads waiting on that Offset will be cancelled.
            segmentIds.add(((SegmentOperation) op).getStreamSegmentId());
        }
    }
    if (!this.recoveryMode.get()) {
        // Trigger Future Reads on those segments which were touched by Appends or Seals.
        this.readIndex.triggerFutureReads(segmentIds);
        if (this.commitSuccess != null) {
            this.commitSuccess.run();
        }
    }
}

Example 2 with SegmentOperation

use of io.pravega.segmentstore.server.logs.operations.SegmentOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method preProcessOperation.

/**
 * Pre-processes the given Operation. See OperationMetadataUpdater.preProcessOperation for more details on behavior.
 *
 * @param operation The operation to pre-process.
 * @throws ContainerException     If the given operation was rejected given the current state of the container metadata.
 * @throws StreamSegmentException If the given operation was incompatible with the current state of the Segment.
 *                                For example: StreamSegmentNotExistsException, StreamSegmentSealedException or
 *                                StreamSegmentMergedException.
 */
void preProcessOperation(Operation operation) throws ContainerException, StreamSegmentException {
    checkNotSealed();
    SegmentMetadataUpdateTransaction segmentMetadata = null;
    if (operation instanceof SegmentOperation) {
        segmentMetadata = getSegmentUpdateTransaction(((SegmentOperation) operation).getStreamSegmentId());
        if (segmentMetadata.isDeleted()) {
            throw new StreamSegmentNotExistsException(segmentMetadata.getName());
        }
    }
    if (operation instanceof StreamSegmentAppendOperation) {
        segmentMetadata.preProcessOperation((StreamSegmentAppendOperation) operation);
    } else if (operation instanceof StreamSegmentSealOperation) {
        segmentMetadata.preProcessOperation((StreamSegmentSealOperation) operation);
    } else if (operation instanceof MergeTransactionOperation) {
        MergeTransactionOperation mbe = (MergeTransactionOperation) operation;
        SegmentMetadataUpdateTransaction transactionMetadata = getSegmentUpdateTransaction(mbe.getTransactionSegmentId());
        transactionMetadata.preProcessAsTransactionSegment(mbe);
        segmentMetadata.preProcessAsParentSegment(mbe, transactionMetadata);
    } else if (operation instanceof StreamSegmentMapOperation) {
        preProcessMetadataOperation((StreamSegmentMapOperation) operation);
    } else if (operation instanceof UpdateAttributesOperation) {
        segmentMetadata.preProcessOperation((UpdateAttributesOperation) operation);
    } else if (operation instanceof MetadataCheckpointOperation) {
        // MetadataCheckpointOperations do not require preProcess and accept; they can be handled in a single stage.
        processMetadataOperation((MetadataCheckpointOperation) operation);
    } else if (operation instanceof StorageMetadataCheckpointOperation) {
        // StorageMetadataCheckpointOperation do not require preProcess and accept; they can be handled in a single stage.
        processMetadataOperation((StorageMetadataCheckpointOperation) operation);
    } else if (operation instanceof StreamSegmentTruncateOperation) {
        segmentMetadata.preProcessOperation((StreamSegmentTruncateOperation) operation);
    }
}

Example 3 with SegmentOperation

use of io.pravega.segmentstore.server.logs.operations.SegmentOperation in project pravega by pravega.

the class MemoryStateUpdaterTests method testProcess.

/**
 * Tests the functionality of the process() method.
 */
@Test
public void testProcess() throws Exception {
    int segmentCount = 10;
    int operationCountPerType = 5;
    // Add to MTL + Add to ReadIndex (append; beginMerge).
    SequencedItemList<Operation> opLog = new SequencedItemList<>();
    ArrayList<TestReadIndex.MethodInvocation> methodInvocations = new ArrayList<>();
    TestReadIndex readIndex = new TestReadIndex(methodInvocations::add);
    AtomicInteger flushCallbackCallCount = new AtomicInteger();
    MemoryStateUpdater updater = new MemoryStateUpdater(opLog, readIndex, flushCallbackCallCount::incrementAndGet);
    ArrayList<Operation> operations = populate(updater, segmentCount, operationCountPerType);
    // Verify they were properly processed.
    int triggerFutureCount = (int) methodInvocations.stream().filter(mi -> mi.methodName.equals(TestReadIndex.TRIGGER_FUTURE_READS)).count();
    int addCount = methodInvocations.size() - triggerFutureCount;
    Assert.assertEquals("Unexpected number of items added to ReadIndex.", operations.size() - segmentCount * operationCountPerType, addCount);
    Assert.assertEquals("Unexpected number of calls to the ReadIndex triggerFutureReads method.", 1, triggerFutureCount);
    Assert.assertEquals("Unexpected number of calls to the flushCallback provided in the constructor.", 1, flushCallbackCallCount.get());
    // Verify add calls.
    Iterator<Operation> logIterator = opLog.read(-1, operations.size());
    int currentIndex = -1;
    int currentReadIndex = -1;
    while (logIterator.hasNext()) {
        currentIndex++;
        Operation expected = operations.get(currentIndex);
        Operation actual = logIterator.next();
        if (expected instanceof StorageOperation) {
            currentReadIndex++;
            TestReadIndex.MethodInvocation invokedMethod = methodInvocations.get(currentReadIndex);
            if (expected instanceof StreamSegmentAppendOperation) {
                Assert.assertTrue("StreamSegmentAppendOperation was not added as a CachedStreamSegmentAppendOperation to the Memory Log.", actual instanceof CachedStreamSegmentAppendOperation);
                StreamSegmentAppendOperation appendOp = (StreamSegmentAppendOperation) expected;
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was not added to the ReadIndex.", TestReadIndex.APPEND, invokedMethod.methodName);
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getStreamSegmentId(), invokedMethod.args.get("streamSegmentId"));
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getStreamSegmentOffset(), invokedMethod.args.get("offset"));
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getData(), invokedMethod.args.get("data"));
            } else if (expected instanceof MergeTransactionOperation) {
                MergeTransactionOperation mergeOp = (MergeTransactionOperation) expected;
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was not added to the ReadIndex.", TestReadIndex.BEGIN_MERGE, invokedMethod.methodName);
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getStreamSegmentId(), invokedMethod.args.get("targetStreamSegmentId"));
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getStreamSegmentOffset(), invokedMethod.args.get("offset"));
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getTransactionSegmentId(), invokedMethod.args.get("sourceStreamSegmentId"));
            }
        }
    }
    // Verify triggerFutureReads args.
    @SuppressWarnings("unchecked") Collection<Long> triggerSegmentIds = (Collection<Long>) methodInvocations.stream().filter(mi -> mi.methodName.equals(TestReadIndex.TRIGGER_FUTURE_READS)).findFirst().get().args.get("streamSegmentIds");
    val expectedSegmentIds = operations.stream().filter(op -> op instanceof SegmentOperation).map(op -> ((SegmentOperation) op).getStreamSegmentId()).collect(Collectors.toSet());
    AssertExtensions.assertContainsSameElements("ReadIndex.triggerFutureReads() was called with the wrong set of StreamSegmentIds.", expectedSegmentIds, triggerSegmentIds);
    // Test DataCorruptionException.
    AssertExtensions.assertThrows("MemoryStateUpdater accepted an operation that was out of order.", // This does not have a SequenceNumber set, so it should trigger a DCE.
    () -> updater.process(new MergeTransactionOperation(1, 2)), ex -> ex instanceof DataCorruptionException);
}

Example 4 with SegmentOperation

use of io.pravega.segmentstore.server.logs.operations.SegmentOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method acceptOperation.

/**
 * Accepts the given Operation. The Operation's effects are reflected in the pending transaction.
 * This method has no effect on Metadata Operations.
 * See OperationMetadataUpdater.acceptOperation for more details on behavior.
 *
 * @param operation The operation to accept.
 * @throws MetadataUpdateException If the given operation was rejected given the current state of the metadata.
 * @throws NullPointerException    If the operation is null.
 */
void acceptOperation(Operation operation) throws MetadataUpdateException {
    checkNotSealed();
    SegmentMetadataUpdateTransaction segmentMetadata = null;
    if (operation instanceof SegmentOperation) {
        segmentMetadata = getSegmentUpdateTransaction(((SegmentOperation) operation).getStreamSegmentId());
        segmentMetadata.setLastUsed(operation.getSequenceNumber());
    }
    if (operation instanceof StreamSegmentAppendOperation) {
        segmentMetadata.acceptOperation((StreamSegmentAppendOperation) operation);
    } else if (operation instanceof StreamSegmentSealOperation) {
        segmentMetadata.acceptOperation((StreamSegmentSealOperation) operation);
    } else if (operation instanceof MergeTransactionOperation) {
        MergeTransactionOperation mto = (MergeTransactionOperation) operation;
        SegmentMetadataUpdateTransaction transactionMetadata = getSegmentUpdateTransaction(mto.getTransactionSegmentId());
        transactionMetadata.acceptAsTransactionSegment(mto);
        transactionMetadata.setLastUsed(operation.getSequenceNumber());
        segmentMetadata.acceptAsParentSegment(mto, transactionMetadata);
    } else if (operation instanceof MetadataCheckpointOperation) {
        // A MetadataCheckpointOperation represents a valid truncation point. Record it as such.
        this.newTruncationPoints.add(operation.getSequenceNumber());
    } else if (operation instanceof StreamSegmentMapOperation) {
        acceptMetadataOperation((StreamSegmentMapOperation) operation);
    } else if (operation instanceof UpdateAttributesOperation) {
        segmentMetadata.acceptOperation((UpdateAttributesOperation) operation);
    } else if (operation instanceof StreamSegmentTruncateOperation) {
        segmentMetadata.acceptOperation((StreamSegmentTruncateOperation) operation);
    }
}