Examples with StreamSegmentAppendOperation io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation used on opensource projects

Example 6 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testProcessMetadataCheckpoint.

/**
 * Tests the processMetadataOperation method with MetadataCheckpoint operations.
 */
@Test
public void testProcessMetadataCheckpoint() throws Exception {
    // When encountering MetadataCheckpoint in non-Recovery Mode, the ContainerMetadataUpdateTransaction serializes a snapshot
    // of the current metadata inside the Operation.
    // When encountering MetadataCheckpoint in Recovery Mode, the ContainerMetadataUpdateTransaction deserializes the snapshot-ted
    // metadata in it and applies it to the container metadata (inside the transaction). All existing metadata updates
    // are cleared.
    String newSegmentName = "NewSegmentId";
    AtomicLong seqNo = new AtomicLong();
    // Create a non-empty metadata, and in addition, seal a segment and truncate it.
    this.timeProvider.setElapsedMillis(1234);
    UpdateableContainerMetadata metadata = createMetadata();
    metadata.getStreamSegmentMetadata(SEGMENT_ID).markSealed();
    metadata.getStreamSegmentMetadata(SEGMENT_ID).setStartOffset(SEGMENT_LENGTH / 2);
    val txn = createUpdateTransaction(metadata);
    // Checkpoint 1: original metadata.
    // Checkpoint 2: Checkpoint 1 + 1 StreamSegment and 1 Transaction + 1 Append
    MetadataCheckpointOperation checkpoint1 = createMetadataCheckpoint();
    MetadataCheckpointOperation checkpoint2 = createMetadataCheckpoint();
    // Checkpoint 1 Should have original metadata.
    processOperation(checkpoint1, txn, seqNo::incrementAndGet);
    UpdateableContainerMetadata checkpointedMetadata = getCheckpointedMetadata(checkpoint1);
    assertMetadataSame("Unexpected metadata before any operation.", metadata, checkpointedMetadata);
    // Map another StreamSegment, and add an append
    StreamSegmentMapOperation mapOp = new StreamSegmentMapOperation(StreamSegmentInformation.builder().name(newSegmentName).length(SEGMENT_LENGTH).build());
    processOperation(mapOp, txn, seqNo::incrementAndGet);
    processOperation(new StreamSegmentAppendOperation(mapOp.getStreamSegmentId(), DEFAULT_APPEND_DATA, createAttributeUpdates()), txn, seqNo::incrementAndGet);
    processOperation(checkpoint2, txn, seqNo::incrementAndGet);
    // Checkpoint 2 should have Checkpoint 1 + New StreamSegment + Append.
    txn.commit(metadata);
    checkpointedMetadata = getCheckpointedMetadata(checkpoint2);
    assertMetadataSame("Unexpected metadata after deserializing checkpoint.", metadata, checkpointedMetadata);
}

Example 7 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testStreamSegmentAppendWithOffset.

/**
 * Tests the ability of the ContainerMetadataUpdateTransaction to process (and accept) StreamSegmentAppendOperations with
 * predefined offsets.
 */
@Test
public void testStreamSegmentAppendWithOffset() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    // Append #1 (at offset 0).
    long offset = metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength();
    StreamSegmentAppendOperation appendOp = createAppendWithOffset(offset);
    txn.preProcessOperation(appendOp);
    Assert.assertEquals("Unexpected StreamSegmentOffset after call to preProcess in non-recovery mode.", offset, appendOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(appendOp, "call to preProcess in non-recovery mode");
    Assert.assertEquals("preProcess(Append) seems to have changed the Updater internal state.", offset, txn.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("preProcess(Append) seems to have changed the metadata.", offset, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    txn.acceptOperation(appendOp);
    // Append #2 (after Append #1)
    offset = appendOp.getStreamSegmentOffset() + appendOp.getLength();
    appendOp = createAppendWithOffset(offset);
    txn.preProcessOperation(appendOp);
    Assert.assertEquals("Unexpected StreamSegmentOffset after call to preProcess in non-recovery mode.", offset, appendOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(appendOp, "call to preProcess in non-recovery mode");
    Assert.assertEquals("preProcess(Append) seems to have changed the Updater internal state.", offset, txn.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("preProcess(Append) seems to have changed the metadata.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    txn.acceptOperation(appendOp);
    // Append #3 (wrong offset)
    offset = appendOp.getStreamSegmentOffset() + appendOp.getLength() - 1;
    StreamSegmentAppendOperation badAppendOp = createAppendWithOffset(offset);
    AssertExtensions.assertThrows("preProcessOperations accepted an append with the wrong offset.", () -> txn.preProcessOperation(badAppendOp), ex -> ex instanceof BadOffsetException);
    AssertExtensions.assertThrows("acceptOperation accepted an append that was rejected during preProcessing.", () -> txn.acceptOperation(badAppendOp), ex -> ex instanceof MetadataUpdateException);
}

Example 8 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class MemoryStateUpdater method addToReadIndex.

/**
 * Registers the given operation in the ReadIndex.
 *
 * @param operation The operation to register.
 */
private void addToReadIndex(StorageOperation operation) {
    try {
        if (operation instanceof StreamSegmentAppendOperation) {
            // Record a StreamSegmentAppendOperation. Just in case, we also support this type of operation, but we need to
            // log a warning indicating so. This means we do not optimize memory properly, and we end up storing data
            // in two different places.
            StreamSegmentAppendOperation appendOperation = (StreamSegmentAppendOperation) operation;
            this.readIndex.append(appendOperation.getStreamSegmentId(), appendOperation.getStreamSegmentOffset(), appendOperation.getData());
        } else if (operation instanceof MergeTransactionOperation) {
            // Record a MergeTransactionOperation. We call beginMerge here, and the StorageWriter will call completeMerge.
            MergeTransactionOperation mergeOperation = (MergeTransactionOperation) operation;
            this.readIndex.beginMerge(mergeOperation.getStreamSegmentId(), mergeOperation.getStreamSegmentOffset(), mergeOperation.getTransactionSegmentId());
        } else {
            assert !(operation instanceof CachedStreamSegmentAppendOperation) : "attempted to add a CachedStreamSegmentAppendOperation to the ReadIndex";
        }
    } catch (ObjectClosedException | StreamSegmentNotExistsException ex) {
        // The Segment is in the process of being deleted. We usually end up in here because a concurrent delete
        // request has updated the metadata while we were executing.
        log.warn("Not adding operation '{}' to ReadIndex because it refers to a deleted StreamSegment.", operation);
    }
}

Example 9 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class SegmentAggregator method reconcileAppendOperation.

/**
 * Attempts to reconcile the given Append Operation. Since Append Operations can be partially flushed, reconciliation
 * may be for the full operation or for a part of it.
 *
 * @param op          The Operation (StreamSegmentAppendOperation or CachedStreamSegmentAppendOperation) to reconcile.
 * @param storageInfo The current state of the Segment in Storage.
 * @param timer       Timer for the operation.
 * @return A CompletableFuture containing a FlushResult with the number of bytes reconciled, or failed with a ReconciliationFailureException,
 * if the operation cannot be reconciled, based on the in-memory metadata or the current state of the Segment in Storage.
 */
private CompletableFuture<FlushResult> reconcileAppendOperation(StorageOperation op, SegmentProperties storageInfo, TimeoutTimer timer) {
    Preconditions.checkArgument(op instanceof AggregatedAppendOperation, "Not given an append operation.");
    // Read data from Storage, and compare byte-by-byte.
    InputStream appendStream = this.dataSource.getAppendData(op.getStreamSegmentId(), op.getStreamSegmentOffset(), (int) op.getLength());
    if (appendStream == null) {
        return Futures.failedFuture(new ReconciliationFailureException(String.format("Unable to reconcile operation '%s' because no append data is associated with it.", op), this.metadata, storageInfo));
    }
    // Only read as much data as we need.
    long readLength = Math.min(op.getLastStreamSegmentOffset(), storageInfo.getLength()) - op.getStreamSegmentOffset();
    assert readLength > 0 : "Append Operation to be reconciled is beyond the Segment's StorageLength " + op;
    AtomicInteger bytesReadSoFar = new AtomicInteger();
    // Read all data from storage.
    byte[] storageData = new byte[(int) readLength];
    return Futures.loop(() -> bytesReadSoFar.get() < readLength, () -> this.storage.read(this.handle.get(), op.getStreamSegmentOffset() + bytesReadSoFar.get(), storageData, bytesReadSoFar.get(), (int) readLength - bytesReadSoFar.get(), timer.getRemaining()), bytesRead -> {
        assert bytesRead > 0 : String.format("Unable to make any read progress when reconciling operation '%s' after reading %s bytes.", op, bytesReadSoFar);
        bytesReadSoFar.addAndGet(bytesRead);
    }, this.executor).thenApplyAsync(v -> {
        // Compare, byte-by-byte, the contents of the append.
        verifySame(appendStream, storageData, op, storageInfo);
        if (readLength >= op.getLength() && op.getLastStreamSegmentOffset() <= storageInfo.getLength()) {
            // Operation has been completely validated; pop it off the list.
            StorageOperation removedOp = this.operations.removeFirst();
            assert op == removedOp : "Reconciled operation is not the same as removed operation";
        }
        return new FlushResult().withFlushedBytes(readLength);
    }, this.executor);
}

Example 10 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class StreamSegmentContainer method append.

// endregion
// region StreamSegmentStore Implementation
@Override
public CompletableFuture<Void> append(String streamSegmentName, byte[] data, Collection<AttributeUpdate> attributeUpdates, Duration timeout) {
    ensureRunning();
    TimeoutTimer timer = new TimeoutTimer(timeout);
    logRequest("append", streamSegmentName, data.length);
    this.metrics.append();
    return this.segmentMapper.getOrAssignStreamSegmentId(streamSegmentName, timer.getRemaining(), streamSegmentId -> {
        StreamSegmentAppendOperation operation = new StreamSegmentAppendOperation(streamSegmentId, data, attributeUpdates);
        return this.durableLog.add(operation, timer.getRemaining());
    });
}