Search in sources :

Example 6 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class DataRecoveryTest method testRepairLogEditOperationUserInput.

@Test
public void testRepairLogEditOperationUserInput() throws IOException {
    // Setup command object.
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    STATE.get().getConfigBuilder().include(pravegaProperties);
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    // Case 1: Input a Delete Edit Operation with wrong initial/final ids. Then retry with correct ids.
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).doReturn(1L).doReturn(2L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("delete").when(command).getStringUserInput(Mockito.any());
    Assert.assertEquals(List.of(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.DELETE_OPERATION, 1, 2, null)), command.getDurableLogEditsFromUser());
    // Case 2: Input an Add Edit Operation with a wrong operation type. Then retry with correct operation type.
    Mockito.doReturn(true).doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("add").doReturn("wrong").doReturn("add").doReturn("DeleteSegmentOperation").when(command).getStringUserInput(Mockito.any());
    DeleteSegmentOperation deleteOperationAdded = new DeleteSegmentOperation(1);
    List<DurableDataLogRepairCommand.LogEditOperation> editOps = new ArrayList<>();
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, deleteOperationAdded));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, deleteOperationAdded));
    Assert.assertEquals(editOps, command.getDurableLogEditsFromUser());
    // Case 3: Create rest of operation types without payload (MergeSegmentOperation, StreamSegmentMapOperation, StreamSegmentTruncateOperation, UpdateAttributesOperation)
    long timestamp = System.currentTimeMillis();
    UUID uuid = UUID.randomUUID();
    editOps.clear();
    Mockito.doReturn(true).doReturn(false).doReturn(false).doReturn(true).doReturn(true).doReturn(false).doReturn(false).doReturn(true).doReturn(false).doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(1L).doReturn(1L).doReturn(2L).doReturn(1L).doReturn(2L).doReturn(123L).doReturn(2L).doReturn(2L).doReturn(3L).doReturn(1L).doReturn(10L).doReturn(timestamp).doReturn(3L).doReturn(3L).doReturn(4L).doReturn(4L).doReturn(3L).doReturn(1L).doReturn(2L).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("add").doReturn("MergeSegmentOperation").doReturn(uuid.toString()).doReturn("add").doReturn("StreamSegmentMapOperation").doReturn("test").doReturn(uuid.toString()).doReturn("add").doReturn("StreamSegmentTruncateOperation").doReturn("add").doReturn("UpdateAttributesOperation").doReturn(uuid.toString()).when(command).getStringUserInput(Mockito.any());
    Mockito.doReturn((int) AttributeUpdateType.Replace.getTypeId()).when(command).getIntUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).getBooleanUserInput(Mockito.any());
    AttributeUpdateCollection attributeUpdates = new AttributeUpdateCollection();
    attributeUpdates.add(new AttributeUpdate(AttributeId.fromUUID(uuid), AttributeUpdateType.Replace, 1, 2));
    MergeSegmentOperation mergeSegmentOperation = new MergeSegmentOperation(1, 2, attributeUpdates);
    mergeSegmentOperation.setStreamSegmentOffset(123);
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 1, 1, mergeSegmentOperation));
    Map<AttributeId, Long> attributes = new HashMap<>();
    attributes.put(AttributeId.fromUUID(uuid), 10L);
    SegmentProperties segmentProperties = StreamSegmentInformation.builder().name("test").startOffset(2).length(3).storageLength(1).sealed(true).deleted(false).sealedInStorage(true).deletedInStorage(false).attributes(attributes).lastModified(new ImmutableDate(timestamp)).build();
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 2, 2, new StreamSegmentMapOperation(segmentProperties)));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 3, 3, new StreamSegmentTruncateOperation(3, 3)));
    editOps.add(new DurableDataLogRepairCommand.LogEditOperation(DurableDataLogRepairCommand.LogEditType.ADD_OPERATION, 4, 4, new UpdateAttributesOperation(4, attributeUpdates)));
    Assert.assertEquals(editOps, command.getDurableLogEditsFromUser());
    // Case 4: Add wrong inputs.
    Mockito.doReturn(true).doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doThrow(NumberFormatException.class).doThrow(NullPointerException.class).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("wrong").doReturn("replace").doReturn("replace").when(command).getStringUserInput(Mockito.any());
    command.getDurableLogEditsFromUser();
}
Also used : CommandArgs(io.pravega.cli.admin.CommandArgs) AttributeUpdateCollection(io.pravega.segmentstore.contracts.AttributeUpdateCollection) AttributeUpdate(io.pravega.segmentstore.contracts.AttributeUpdate) UpdateAttributesOperation(io.pravega.segmentstore.server.logs.operations.UpdateAttributesOperation) HashMap(java.util.HashMap) AttributeId(io.pravega.segmentstore.contracts.AttributeId) ArrayList(java.util.ArrayList) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) Properties(java.util.Properties) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) ImmutableDate(io.pravega.common.util.ImmutableDate) StreamSegmentTruncateOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) UUID(java.util.UUID) AdminCommandState(io.pravega.cli.admin.AdminCommandState) Test(org.junit.Test)

Example 7 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class StreamSegmentContainer method mergeStreamSegment.

private CompletableFuture<MergeStreamSegmentResult> mergeStreamSegment(long targetSegmentId, long sourceSegmentId, AttributeUpdateCollection attributeUpdates, TimeoutTimer timer) {
    // Get a reference to the source segment's metadata now, before the merge. It may not be accessible afterwards.
    SegmentMetadata sourceMetadata = this.metadata.getStreamSegmentMetadata(sourceSegmentId);
    CompletableFuture<Void> sealResult = trySealStreamSegment(sourceMetadata, timer.getRemaining());
    if (sourceMetadata.getLength() == 0) {
        // writes in the pipeline. As such, we cannot pipeline the two operations, and must wait for the seal to finish first.
        return sealResult.thenComposeAsync(v -> {
            // to and including the seal, so if there were any writes outstanding before, they should now be reflected in it.
            if (sourceMetadata.getLength() == 0) {
                // Source is still empty after sealing - OK to delete.
                log.debug("{}: Updating attributes (if any) and deleting empty source segment instead of merging {}.", this.traceObjectId, sourceMetadata.getName());
                // Execute the attribute update on the target segment only if needed.
                Supplier<CompletableFuture<Void>> updateAttributesIfNeeded = () -> attributeUpdates == null ? CompletableFuture.completedFuture(null) : updateAttributesForSegment(targetSegmentId, attributeUpdates, timer.getRemaining());
                return updateAttributesIfNeeded.get().thenCompose(v2 -> deleteStreamSegment(sourceMetadata.getName(), timer.getRemaining()).thenApply(v3 -> new MergeStreamSegmentResult(this.metadata.getStreamSegmentMetadata(targetSegmentId).getLength(), sourceMetadata.getLength(), sourceMetadata.getAttributes())));
            } else {
                // Source now has some data - we must merge the two.
                MergeSegmentOperation operation = new MergeSegmentOperation(targetSegmentId, sourceSegmentId, attributeUpdates);
                return processAttributeUpdaterOperation(operation, timer).thenApply(v2 -> new MergeStreamSegmentResult(operation.getStreamSegmentOffset() + operation.getLength(), operation.getLength(), sourceMetadata.getAttributes()));
            }
        }, this.executor);
    } else {
        // Source is not empty, so we cannot delete. Make use of the DurableLog's pipelining abilities by queueing up
        // the Merge right after the Seal.
        MergeSegmentOperation operation = new MergeSegmentOperation(targetSegmentId, sourceSegmentId, attributeUpdates);
        return CompletableFuture.allOf(sealResult, processAttributeUpdaterOperation(operation, timer)).thenApply(v2 -> new MergeStreamSegmentResult(operation.getStreamSegmentOffset() + operation.getLength(), operation.getLength(), sourceMetadata.getAttributes()));
    }
}
Also used : UpdateableSegmentMetadata(io.pravega.segmentstore.server.UpdateableSegmentMetadata) SegmentMetadata(io.pravega.segmentstore.server.SegmentMetadata) ATTRIBUTE_SLTS_LATEST_SNAPSHOT_ID(io.pravega.segmentstore.contracts.Attributes.ATTRIBUTE_SLTS_LATEST_SNAPSHOT_ID) Storage(io.pravega.segmentstore.storage.Storage) StreamSegmentNotExistsException(io.pravega.segmentstore.contracts.StreamSegmentNotExistsException) ContainerEventProcessor(io.pravega.segmentstore.server.ContainerEventProcessor) OperationPriority(io.pravega.segmentstore.server.logs.operations.OperationPriority) SneakyThrows(lombok.SneakyThrows) Retry(io.pravega.common.util.Retry) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) RequiredArgsConstructor(lombok.RequiredArgsConstructor) ContainerMetadata(io.pravega.segmentstore.server.ContainerMetadata) SnapshotInfoStore(io.pravega.segmentstore.storage.chunklayer.SnapshotInfoStore) UpdateableSegmentMetadata(io.pravega.segmentstore.server.UpdateableSegmentMetadata) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) ReadIndexFactory(io.pravega.segmentstore.server.ReadIndexFactory) AttributeUpdate(io.pravega.segmentstore.contracts.AttributeUpdate) StreamSegmentSealedException(io.pravega.segmentstore.contracts.StreamSegmentSealedException) AttributeIndexFactory(io.pravega.segmentstore.server.attributes.AttributeIndexFactory) BufferView(io.pravega.common.util.BufferView) Duration(java.time.Duration) Map(java.util.Map) AbstractService(com.google.common.util.concurrent.AbstractService) Operation(io.pravega.segmentstore.server.logs.operations.Operation) StreamSegmentTruncateOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation) RetryAndThrowConditionally(io.pravega.common.util.Retry.RetryAndThrowConditionally) Services(io.pravega.common.concurrent.Services) ContainerOfflineException(io.pravega.segmentstore.server.ContainerOfflineException) Attributes(io.pravega.segmentstore.contracts.Attributes) SegmentStoreMetrics(io.pravega.segmentstore.server.SegmentStoreMetrics) Collection(java.util.Collection) CompletionException(java.util.concurrent.CompletionException) Writer(io.pravega.segmentstore.server.Writer) Collectors(java.util.stream.Collectors) SimpleStorageFactory(io.pravega.segmentstore.storage.SimpleStorageFactory) Objects(java.util.Objects) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) SegmentContainerFactory(io.pravega.segmentstore.server.SegmentContainerFactory) ContainerTableExtension(io.pravega.segmentstore.server.tables.ContainerTableExtension) ATTRIBUTE_SLTS_LATEST_SNAPSHOT_EPOCH(io.pravega.segmentstore.contracts.Attributes.ATTRIBUTE_SLTS_LATEST_SNAPSHOT_EPOCH) WriterSegmentProcessor(io.pravega.segmentstore.server.WriterSegmentProcessor) Futures(io.pravega.common.concurrent.Futures) ReadResult(io.pravega.segmentstore.contracts.ReadResult) Getter(lombok.Getter) IllegalContainerStateException(io.pravega.segmentstore.server.IllegalContainerStateException) ExtendedChunkInfo(io.pravega.segmentstore.contracts.ExtendedChunkInfo) Exceptions(io.pravega.common.Exceptions) StorageFactory(io.pravega.segmentstore.storage.StorageFactory) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) PriorityCalculator(io.pravega.segmentstore.server.logs.PriorityCalculator) BadAttributeUpdateException(io.pravega.segmentstore.contracts.BadAttributeUpdateException) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) SegmentMetadata(io.pravega.segmentstore.server.SegmentMetadata) SegmentType(io.pravega.segmentstore.contracts.SegmentType) ImmutableList(com.google.common.collect.ImmutableList) AttributeUpdaterOperation(io.pravega.segmentstore.server.logs.operations.AttributeUpdaterOperation) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) TableBasedMetadataStore(io.pravega.segmentstore.storage.metadata.TableBasedMetadataStore) AttributeIterator(io.pravega.segmentstore.server.AttributeIterator) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) Nullable(javax.annotation.Nullable) ChunkedSegmentStorage(io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage) LoggerHelpers(io.pravega.common.LoggerHelpers) SegmentContainerExtension(io.pravega.segmentstore.server.SegmentContainerExtension) WriterFactory(io.pravega.segmentstore.server.WriterFactory) NameUtils(io.pravega.shared.NameUtils) AttributeIndex(io.pravega.segmentstore.server.AttributeIndex) TimeoutTimer(io.pravega.common.TimeoutTimer) UpdateAttributesOperation(io.pravega.segmentstore.server.logs.operations.UpdateAttributesOperation) AttributeId(io.pravega.segmentstore.contracts.AttributeId) lombok.val(lombok.val) OperationLog(io.pravega.segmentstore.server.OperationLog) MergeStreamSegmentResult(io.pravega.segmentstore.contracts.MergeStreamSegmentResult) StreamSegmentMergedException(io.pravega.segmentstore.contracts.StreamSegmentMergedException) UtilsWrapper(io.pravega.segmentstore.storage.chunklayer.UtilsWrapper) Service(com.google.common.util.concurrent.Service) SnapshotInfo(io.pravega.segmentstore.storage.chunklayer.SnapshotInfo) Consumer(java.util.function.Consumer) DirectSegmentAccess(io.pravega.segmentstore.server.DirectSegmentAccess) ContainerAttributeIndex(io.pravega.segmentstore.server.attributes.ContainerAttributeIndex) AttributeUpdateCollection(io.pravega.segmentstore.contracts.AttributeUpdateCollection) SegmentOperation(io.pravega.segmentstore.server.SegmentOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) OperationLogFactory(io.pravega.segmentstore.server.OperationLogFactory) SegmentContainer(io.pravega.segmentstore.server.SegmentContainer) Preconditions(com.google.common.base.Preconditions) AttributeUpdateType(io.pravega.segmentstore.contracts.AttributeUpdateType) ReadIndex(io.pravega.segmentstore.server.ReadIndex) Collections(java.util.Collections) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) StreamSegmentSealOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation) CompletableFuture(java.util.concurrent.CompletableFuture) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) MergeStreamSegmentResult(io.pravega.segmentstore.contracts.MergeStreamSegmentResult)

Example 8 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class MemoryStateUpdater method addToReadIndex.

/**
 * Registers the given operation in the ReadIndex.
 *
 * @param operation The operation to register.
 * @throws CacheFullException If the operation could not be added to the {@link ReadIndex} due to the cache being
 * full and unable to evict anything to make room for more.
 * @throws ServiceHaltException If any unexpected exception occurred that prevented the operation from being
 * added to the {@link ReadIndex}. Unexpected exceptions are all exceptions other than those declared in this
 * method or that indicate we are shutting down or that the segment has been deleted.
 */
private void addToReadIndex(StorageOperation operation) throws ServiceHaltException, CacheFullException {
    try {
        if (operation instanceof StreamSegmentAppendOperation) {
            // Record a StreamSegmentAppendOperation. Just in case, we also support this type of operation, but we need to
            // log a warning indicating so. This means we do not optimize memory properly, and we end up storing data
            // in two different places.
            StreamSegmentAppendOperation appendOperation = (StreamSegmentAppendOperation) operation;
            this.readIndex.append(appendOperation.getStreamSegmentId(), appendOperation.getStreamSegmentOffset(), appendOperation.getData());
        } else if (operation instanceof MergeSegmentOperation) {
            // Record a MergeSegmentOperation. We call beginMerge here, and the StorageWriter will call completeMerge.
            MergeSegmentOperation mergeOperation = (MergeSegmentOperation) operation;
            this.readIndex.beginMerge(mergeOperation.getStreamSegmentId(), mergeOperation.getStreamSegmentOffset(), mergeOperation.getSourceSegmentId());
        } else {
            assert !(operation instanceof CachedStreamSegmentAppendOperation) : "attempted to add a CachedStreamSegmentAppendOperation to the ReadIndex";
        }
    } catch (ObjectClosedException | StreamSegmentNotExistsException ex) {
        // The Segment is in the process of being deleted. We usually end up in here because a concurrent delete
        // request has updated the metadata while we were executing.
        log.warn("Not adding operation '{}' to ReadIndex because it refers to a deleted StreamSegment.", operation);
    } catch (CacheFullException ex) {
        // Record the operation that we couldn't add and re-throw the exception as we cannot do anything about it here.
        log.warn("Not adding operation '{}' to ReadIndex because the Cache is full.", operation);
        throw ex;
    } catch (Exception ex) {
        throw new ServiceHaltException(String.format("Unable to add operation '%s' to ReadIndex.", operation), ex);
    }
}
Also used : ObjectClosedException(io.pravega.common.ObjectClosedException) ServiceHaltException(io.pravega.segmentstore.server.ServiceHaltException) CacheFullException(io.pravega.segmentstore.storage.cache.CacheFullException) CachedStreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.CachedStreamSegmentAppendOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) StreamSegmentNotExistsException(io.pravega.segmentstore.contracts.StreamSegmentNotExistsException) ObjectClosedException(io.pravega.common.ObjectClosedException) StreamSegmentNotExistsException(io.pravega.segmentstore.contracts.StreamSegmentNotExistsException) ServiceHaltException(io.pravega.segmentstore.server.ServiceHaltException) CacheFullException(io.pravega.segmentstore.storage.cache.CacheFullException) DataCorruptionException(io.pravega.segmentstore.server.DataCorruptionException) CachedStreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.CachedStreamSegmentAppendOperation)

Example 9 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method preProcessOperation.

/**
 * Pre-processes the given Operation. See OperationMetadataUpdater.preProcessOperation for more details on behavior.
 *
 * @param operation The operation to pre-process.
 * @throws ContainerException     If the given operation was rejected given the current state of the container metadata.
 * @throws StreamSegmentException If the given operation was incompatible with the current state of the Segment.
 *                                For example: StreamSegmentNotExistsException, StreamSegmentSealedException or
 *                                StreamSegmentMergedException.
 */
void preProcessOperation(Operation operation) throws ContainerException, StreamSegmentException {
    checkNotSealed();
    if (operation instanceof SegmentOperation) {
        val segmentMetadata = getSegmentUpdateTransaction(((SegmentOperation) operation).getStreamSegmentId());
        if (segmentMetadata.isDeleted()) {
            throw new StreamSegmentNotExistsException(segmentMetadata.getName());
        }
        if (operation instanceof StreamSegmentAppendOperation) {
            segmentMetadata.preProcessOperation((StreamSegmentAppendOperation) operation);
        } else if (operation instanceof StreamSegmentSealOperation) {
            segmentMetadata.preProcessOperation((StreamSegmentSealOperation) operation);
        } else if (operation instanceof MergeSegmentOperation) {
            MergeSegmentOperation mbe = (MergeSegmentOperation) operation;
            SegmentMetadataUpdateTransaction sourceMetadata = getSegmentUpdateTransaction(mbe.getSourceSegmentId());
            sourceMetadata.preProcessAsSourceSegment(mbe);
            segmentMetadata.preProcessAsTargetSegment(mbe, sourceMetadata);
        } else if (operation instanceof UpdateAttributesOperation) {
            segmentMetadata.preProcessOperation((UpdateAttributesOperation) operation);
        } else if (operation instanceof StreamSegmentTruncateOperation) {
            segmentMetadata.preProcessOperation((StreamSegmentTruncateOperation) operation);
        } else if (operation instanceof DeleteSegmentOperation) {
            segmentMetadata.preProcessOperation((DeleteSegmentOperation) operation);
        }
    }
    if (operation instanceof MetadataCheckpointOperation) {
        // MetadataCheckpointOperations do not require preProcess and accept; they can be handled in a single stage.
        processMetadataOperation((MetadataCheckpointOperation) operation);
    } else if (operation instanceof StorageMetadataCheckpointOperation) {
        // StorageMetadataCheckpointOperation do not require preProcess and accept; they can be handled in a single stage.
        processMetadataOperation((StorageMetadataCheckpointOperation) operation);
    } else if (operation instanceof StreamSegmentMapOperation) {
        preProcessMetadataOperation((StreamSegmentMapOperation) operation);
    }
}
Also used : lombok.val(lombok.val) StreamSegmentSealOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) MetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.MetadataCheckpointOperation) UpdateAttributesOperation(io.pravega.segmentstore.server.logs.operations.UpdateAttributesOperation) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) SegmentOperation(io.pravega.segmentstore.server.SegmentOperation) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) StorageMetadataCheckpointOperation(io.pravega.segmentstore.server.logs.operations.StorageMetadataCheckpointOperation) StreamSegmentMapOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation) StreamSegmentAppendOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation) StreamSegmentNotExistsException(io.pravega.segmentstore.contracts.StreamSegmentNotExistsException) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation) DeleteSegmentOperation(io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation) StreamSegmentTruncateOperation(io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation)

Example 10 with MergeSegmentOperation

use of io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation in project pravega by pravega.

the class SegmentAggregator method mergeCompleted.

/**
 * Executes post-Storage merge tasks, including state and metadata updates.
 */
private void mergeCompleted(SegmentProperties segmentProperties, UpdateableSegmentMetadata transactionMetadata, MergeSegmentOperation mergeOp) {
    // We have processed a MergeSegmentOperation, pop the first operation off and decrement the counter.
    StorageOperation processedOperation = this.operations.removeFirst();
    assert processedOperation != null && processedOperation instanceof MergeSegmentOperation : "First outstanding operation was not a MergeSegmentOperation";
    MergeSegmentOperation mop = (MergeSegmentOperation) processedOperation;
    assert mop.getSourceSegmentId() == transactionMetadata.getId() : "First outstanding operation was a MergeSegmentOperation for the wrong Transaction id.";
    int newCount = this.mergeTransactionCount.decrementAndGet();
    assert newCount >= 0 : "Negative value for mergeTransactionCount";
    // Post-merger validation. Verify we are still in agreement with the storage.
    long expectedNewLength = this.metadata.getStorageLength() + mergeOp.getLength();
    if (segmentProperties.getLength() != expectedNewLength) {
        throw new CompletionException(new DataCorruptionException(String.format("Transaction Segment '%s' was merged into parent '%s' but the parent segment has an unexpected StorageLength after the merger. Previous=%d, MergeLength=%d, Expected=%d, Actual=%d", transactionMetadata.getName(), this.metadata.getName(), segmentProperties.getLength(), mergeOp.getLength(), expectedNewLength, segmentProperties.getLength())));
    }
    updateMetadata(segmentProperties);
    updateMetadataForTransactionPostMerger(transactionMetadata, mop.getStreamSegmentId());
}
Also used : CompletionException(java.util.concurrent.CompletionException) StorageOperation(io.pravega.segmentstore.server.logs.operations.StorageOperation) DataCorruptionException(io.pravega.segmentstore.server.DataCorruptionException) MergeSegmentOperation(io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation)

Aggregations

MergeSegmentOperation (io.pravega.segmentstore.server.logs.operations.MergeSegmentOperation)29 StreamSegmentAppendOperation (io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation)19 lombok.val (lombok.val)16 StreamSegmentSealOperation (io.pravega.segmentstore.server.logs.operations.StreamSegmentSealOperation)15 UpdateableSegmentMetadata (io.pravega.segmentstore.server.UpdateableSegmentMetadata)13 StorageOperation (io.pravega.segmentstore.server.logs.operations.StorageOperation)12 DeleteSegmentOperation (io.pravega.segmentstore.server.logs.operations.DeleteSegmentOperation)11 Operation (io.pravega.segmentstore.server.logs.operations.Operation)11 StreamSegmentTruncateOperation (io.pravega.segmentstore.server.logs.operations.StreamSegmentTruncateOperation)11 DataCorruptionException (io.pravega.segmentstore.server.DataCorruptionException)10 CachedStreamSegmentAppendOperation (io.pravega.segmentstore.server.logs.operations.CachedStreamSegmentAppendOperation)10 ArrayList (java.util.ArrayList)10 StreamSegmentNotExistsException (io.pravega.segmentstore.contracts.StreamSegmentNotExistsException)9 SegmentMetadata (io.pravega.segmentstore.server.SegmentMetadata)9 SegmentProperties (io.pravega.segmentstore.contracts.SegmentProperties)8 StreamSegmentSealedException (io.pravega.segmentstore.contracts.StreamSegmentSealedException)8 SegmentOperation (io.pravega.segmentstore.server.SegmentOperation)8 Collectors (java.util.stream.Collectors)8 Exceptions (io.pravega.common.Exceptions)7 Futures (io.pravega.common.concurrent.Futures)7