Examples with MergeTransactionOperation io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation used on opensource projects

Example 16 with MergeTransactionOperation

use of io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method acceptOperation.

/**
 * Accepts the given Operation. The Operation's effects are reflected in the pending transaction.
 * This method has no effect on Metadata Operations.
 * See OperationMetadataUpdater.acceptOperation for more details on behavior.
 *
 * @param operation The operation to accept.
 * @throws MetadataUpdateException If the given operation was rejected given the current state of the metadata.
 * @throws NullPointerException    If the operation is null.
 */
void acceptOperation(Operation operation) throws MetadataUpdateException {
    checkNotSealed();
    SegmentMetadataUpdateTransaction segmentMetadata = null;
    if (operation instanceof SegmentOperation) {
        segmentMetadata = getSegmentUpdateTransaction(((SegmentOperation) operation).getStreamSegmentId());
        segmentMetadata.setLastUsed(operation.getSequenceNumber());
    }
    if (operation instanceof StreamSegmentAppendOperation) {
        segmentMetadata.acceptOperation((StreamSegmentAppendOperation) operation);
    } else if (operation instanceof StreamSegmentSealOperation) {
        segmentMetadata.acceptOperation((StreamSegmentSealOperation) operation);
    } else if (operation instanceof MergeTransactionOperation) {
        MergeTransactionOperation mto = (MergeTransactionOperation) operation;
        SegmentMetadataUpdateTransaction transactionMetadata = getSegmentUpdateTransaction(mto.getTransactionSegmentId());
        transactionMetadata.acceptAsTransactionSegment(mto);
        transactionMetadata.setLastUsed(operation.getSequenceNumber());
        segmentMetadata.acceptAsParentSegment(mto, transactionMetadata);
    } else if (operation instanceof MetadataCheckpointOperation) {
        // A MetadataCheckpointOperation represents a valid truncation point. Record it as such.
        this.newTruncationPoints.add(operation.getSequenceNumber());
    } else if (operation instanceof StreamSegmentMapOperation) {
        acceptMetadataOperation((StreamSegmentMapOperation) operation);
    } else if (operation instanceof UpdateAttributesOperation) {
        segmentMetadata.acceptOperation((UpdateAttributesOperation) operation);
    } else if (operation instanceof StreamSegmentTruncateOperation) {
        segmentMetadata.acceptOperation((StreamSegmentTruncateOperation) operation);
    }
}

Example 17 with MergeTransactionOperation

use of io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation in project pravega by pravega.

the class OperationLogTestBase method getExpectedContents.

/**
 * Given a list of Log Operations, generates an InputStream for each encountered StreamSegment that contains the final
 * contents of that StreamSegment. Only considers operations of type StreamSegmentAppendOperation and MergeTransactionOperation.
 */
private AbstractMap<Long, InputStream> getExpectedContents(Collection<OperationWithCompletion> operations) {
    HashMap<Long, List<ByteArrayInputStream>> partialContents = new HashMap<>();
    for (OperationWithCompletion o : operations) {
        Assert.assertTrue("Operation is not completed.", o.completion.isDone());
        if (o.completion.isCompletedExceptionally()) {
            // This is failed operation; ignore it.
            continue;
        }
        if (o.operation instanceof StreamSegmentAppendOperation) {
            StreamSegmentAppendOperation appendOperation = (StreamSegmentAppendOperation) o.operation;
            List<ByteArrayInputStream> segmentContents = partialContents.get(appendOperation.getStreamSegmentId());
            if (segmentContents == null) {
                segmentContents = new ArrayList<>();
                partialContents.put(appendOperation.getStreamSegmentId(), segmentContents);
            }
            segmentContents.add(new ByteArrayInputStream(appendOperation.getData()));
        } else if (o.operation instanceof MergeTransactionOperation) {
            MergeTransactionOperation mergeOperation = (MergeTransactionOperation) o.operation;
            List<ByteArrayInputStream> targetSegmentContents = partialContents.get(mergeOperation.getStreamSegmentId());
            if (targetSegmentContents == null) {
                targetSegmentContents = new ArrayList<>();
                partialContents.put(mergeOperation.getStreamSegmentId(), targetSegmentContents);
            }
            List<ByteArrayInputStream> sourceSegmentContents = partialContents.get(mergeOperation.getTransactionSegmentId());
            targetSegmentContents.addAll(sourceSegmentContents);
            partialContents.remove(mergeOperation.getTransactionSegmentId());
        }
    }
    // Construct final result.
    HashMap<Long, InputStream> result = new HashMap<>();
    for (Map.Entry<Long, List<ByteArrayInputStream>> e : partialContents.entrySet()) {
        result.put(e.getKey(), new SequenceInputStream(Iterators.asEnumeration(e.getValue().iterator())));
    }
    return result;
}

Example 18 with MergeTransactionOperation

use of io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation in project pravega by pravega.

the class OperationLogTestBase method getExpectedLengths.

/**
 * Given a list of LogOperations, calculates the final lengths of the StreamSegments that are encountered, by inspecting
 * every StreamSegmentAppendOperation and MergeTransactionOperation. All other types of Log Operations are ignored.
 */
private AbstractMap<Long, Integer> getExpectedLengths(Collection<OperationWithCompletion> operations) {
    HashMap<Long, Integer> result = new HashMap<>();
    for (OperationWithCompletion o : operations) {
        Assert.assertTrue("Operation is not completed.", o.completion.isDone());
        if (o.completion.isCompletedExceptionally()) {
            // This is a failed operation; ignore it.
            continue;
        }
        if (o.operation instanceof StreamSegmentAppendOperation) {
            StreamSegmentAppendOperation appendOperation = (StreamSegmentAppendOperation) o.operation;
            result.put(appendOperation.getStreamSegmentId(), result.getOrDefault(appendOperation.getStreamSegmentId(), 0) + appendOperation.getData().length);
        } else if (o.operation instanceof MergeTransactionOperation) {
            MergeTransactionOperation mergeOperation = (MergeTransactionOperation) o.operation;
            result.put(mergeOperation.getStreamSegmentId(), result.getOrDefault(mergeOperation.getStreamSegmentId(), 0) + result.getOrDefault(mergeOperation.getTransactionSegmentId(), 0));
            result.remove(mergeOperation.getTransactionSegmentId());
        }
    }
    return result;
}

Example 19 with MergeTransactionOperation

use of io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation in project pravega by pravega.

the class OperationMetadataUpdaterTests method mergeTransaction.

private void mergeTransaction(long transactionId, OperationMetadataUpdater updater, UpdateableContainerMetadata referenceMetadata) throws Exception {
    long parentSegmentId = updater.getStreamSegmentMetadata(transactionId).getParentId();
    val op = new MergeTransactionOperation(parentSegmentId, transactionId);
    process(op, updater);
    if (referenceMetadata != null) {
        referenceMetadata.getStreamSegmentMetadata(transactionId).markMerged();
        val rsm = referenceMetadata.getStreamSegmentMetadata(parentSegmentId);
        rsm.setLength(rsm.getLength() + op.getLength());
    }
}

Example 20 with MergeTransactionOperation

use of io.pravega.segmentstore.server.logs.operations.MergeTransactionOperation in project pravega by pravega.

the class SegmentAggregator method reconcileMergeOperation.

/**
 * Attempts to reconcile the given MergeTransactionOperation.
 *
 * @param op          The Operation to reconcile.
 * @param storageInfo The current state of the Segment in Storage.
 * @param timer       Timer for the operation
 * @return A CompletableFuture containing a FlushResult with the number of bytes reconciled, or failed with a ReconciliationFailureException,
 * if the operation cannot be reconciled, based on the in-memory metadata or the current state of the Segment in Storage.
 */
private CompletableFuture<FlushResult> reconcileMergeOperation(MergeTransactionOperation op, SegmentProperties storageInfo, TimeoutTimer timer) {
    // Verify that the transaction segment is still registered in metadata.
    UpdateableSegmentMetadata transactionMeta = this.dataSource.getStreamSegmentMetadata(op.getTransactionSegmentId());
    if (transactionMeta == null || transactionMeta.isDeleted()) {
        return Futures.failedFuture(new ReconciliationFailureException(String.format("Cannot reconcile operation '%s' because the transaction segment is deleted or missing from the metadata.", op), this.metadata, storageInfo));
    }
    // Verify that the operation fits fully within this segment (mergers are atomic - they either merge all or nothing).
    if (op.getLastStreamSegmentOffset() > storageInfo.getLength()) {
        return Futures.failedFuture(new ReconciliationFailureException(String.format("Cannot reconcile operation '%s' because the transaction segment is not fully merged into the parent.", op), this.metadata, storageInfo));
    }
    // Verify that the transaction segment does not exist in Storage anymore.
    return this.storage.exists(transactionMeta.getName(), timer.getRemaining()).thenApplyAsync(exists -> {
        if (exists) {
            throw new CompletionException(new ReconciliationFailureException(String.format("Cannot reconcile operation '%s' because the transaction segment still exists in Storage.", op), this.metadata, storageInfo));
        }
        // Pop the first operation off the list and update the metadata for the transaction segment.
        StorageOperation processedOperation = this.operations.removeFirst();
        assert processedOperation != null && processedOperation instanceof MergeTransactionOperation : "First outstanding operation was not a MergeTransactionOperation";
        int newCount = this.mergeTransactionCount.decrementAndGet();
        assert newCount >= 0 : "Negative value for mergeTransactionCount";
        updateMetadataForTransactionPostMerger(transactionMeta);
        return new FlushResult().withMergedBytes(op.getLength());
    }, this.executor);
}