Examples with StreamSegmentAppendOperation io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation used on opensource projects

Example 21 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class StorageWriterTests method appendData.

private void appendData(UpdateableSegmentMetadata segmentMetadata, int appendId, int writeId, HashMap<Long, ByteArrayOutputStream> segmentContents, TestContext context) {
    byte[] data = getAppendData(segmentMetadata.getName(), segmentMetadata.getId(), appendId, writeId);
    // Make sure we increase the Length prior to appending; the Writer checks for this.
    long offset = segmentMetadata.getLength();
    segmentMetadata.setLength(offset + data.length);
    StreamSegmentAppendOperation op = new StreamSegmentAppendOperation(segmentMetadata.getId(), data, null);
    op.setStreamSegmentOffset(offset);
    context.dataSource.recordAppend(op);
    context.dataSource.add(new CachedStreamSegmentAppendOperation(op));
    recordAppend(segmentMetadata.getId(), data, segmentContents);
}

Example 22 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method acceptOperation.

/**
 * Accepts the given Operation. The Operation's effects are reflected in the pending transaction.
 * This method has no effect on Metadata Operations.
 * See OperationMetadataUpdater.acceptOperation for more details on behavior.
 *
 * @param operation The operation to accept.
 * @throws MetadataUpdateException If the given operation was rejected given the current state of the metadata.
 * @throws NullPointerException    If the operation is null.
 */
void acceptOperation(Operation operation) throws MetadataUpdateException {
    checkNotSealed();
    SegmentMetadataUpdateTransaction segmentMetadata = null;
    if (operation instanceof SegmentOperation) {
        segmentMetadata = getSegmentUpdateTransaction(((SegmentOperation) operation).getStreamSegmentId());
        segmentMetadata.setLastUsed(operation.getSequenceNumber());
    }
    if (operation instanceof StreamSegmentAppendOperation) {
        segmentMetadata.acceptOperation((StreamSegmentAppendOperation) operation);
    } else if (operation instanceof StreamSegmentSealOperation) {
        segmentMetadata.acceptOperation((StreamSegmentSealOperation) operation);
    } else if (operation instanceof MergeTransactionOperation) {
        MergeTransactionOperation mto = (MergeTransactionOperation) operation;
        SegmentMetadataUpdateTransaction transactionMetadata = getSegmentUpdateTransaction(mto.getTransactionSegmentId());
        transactionMetadata.acceptAsTransactionSegment(mto);
        transactionMetadata.setLastUsed(operation.getSequenceNumber());
        segmentMetadata.acceptAsParentSegment(mto, transactionMetadata);
    } else if (operation instanceof MetadataCheckpointOperation) {
        // A MetadataCheckpointOperation represents a valid truncation point. Record it as such.
        this.newTruncationPoints.add(operation.getSequenceNumber());
    } else if (operation instanceof StreamSegmentMapOperation) {
        acceptMetadataOperation((StreamSegmentMapOperation) operation);
    } else if (operation instanceof UpdateAttributesOperation) {
        segmentMetadata.acceptOperation((UpdateAttributesOperation) operation);
    } else if (operation instanceof StreamSegmentTruncateOperation) {
        segmentMetadata.acceptOperation((StreamSegmentTruncateOperation) operation);
    }
}

Example 23 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class OperationLogTestBase method getExpectedContents.

/**
 * Given a list of Log Operations, generates an InputStream for each encountered StreamSegment that contains the final
 * contents of that StreamSegment. Only considers operations of type StreamSegmentAppendOperation and MergeTransactionOperation.
 */
private AbstractMap<Long, InputStream> getExpectedContents(Collection<OperationWithCompletion> operations) {
    HashMap<Long, List<ByteArrayInputStream>> partialContents = new HashMap<>();
    for (OperationWithCompletion o : operations) {
        Assert.assertTrue("Operation is not completed.", o.completion.isDone());
        if (o.completion.isCompletedExceptionally()) {
            // This is failed operation; ignore it.
            continue;
        }
        if (o.operation instanceof StreamSegmentAppendOperation) {
            StreamSegmentAppendOperation appendOperation = (StreamSegmentAppendOperation) o.operation;
            List<ByteArrayInputStream> segmentContents = partialContents.get(appendOperation.getStreamSegmentId());
            if (segmentContents == null) {
                segmentContents = new ArrayList<>();
                partialContents.put(appendOperation.getStreamSegmentId(), segmentContents);
            }
            segmentContents.add(new ByteArrayInputStream(appendOperation.getData()));
        } else if (o.operation instanceof MergeTransactionOperation) {
            MergeTransactionOperation mergeOperation = (MergeTransactionOperation) o.operation;
            List<ByteArrayInputStream> targetSegmentContents = partialContents.get(mergeOperation.getStreamSegmentId());
            if (targetSegmentContents == null) {
                targetSegmentContents = new ArrayList<>();
                partialContents.put(mergeOperation.getStreamSegmentId(), targetSegmentContents);
            }
            List<ByteArrayInputStream> sourceSegmentContents = partialContents.get(mergeOperation.getTransactionSegmentId());
            targetSegmentContents.addAll(sourceSegmentContents);
            partialContents.remove(mergeOperation.getTransactionSegmentId());
        }
    }
    // Construct final result.
    HashMap<Long, InputStream> result = new HashMap<>();
    for (Map.Entry<Long, List<ByteArrayInputStream>> e : partialContents.entrySet()) {
        result.put(e.getKey(), new SequenceInputStream(Iterators.asEnumeration(e.getValue().iterator())));
    }
    return result;
}

Example 24 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class OperationLogTestBase method getExpectedLengths.

/**
 * Given a list of LogOperations, calculates the final lengths of the StreamSegments that are encountered, by inspecting
 * every StreamSegmentAppendOperation and MergeTransactionOperation. All other types of Log Operations are ignored.
 */
private AbstractMap<Long, Integer> getExpectedLengths(Collection<OperationWithCompletion> operations) {
    HashMap<Long, Integer> result = new HashMap<>();
    for (OperationWithCompletion o : operations) {
        Assert.assertTrue("Operation is not completed.", o.completion.isDone());
        if (o.completion.isCompletedExceptionally()) {
            // This is a failed operation; ignore it.
            continue;
        }
        if (o.operation instanceof StreamSegmentAppendOperation) {
            StreamSegmentAppendOperation appendOperation = (StreamSegmentAppendOperation) o.operation;
            result.put(appendOperation.getStreamSegmentId(), result.getOrDefault(appendOperation.getStreamSegmentId(), 0) + appendOperation.getData().length);
        } else if (o.operation instanceof MergeTransactionOperation) {
            MergeTransactionOperation mergeOperation = (MergeTransactionOperation) o.operation;
            result.put(mergeOperation.getStreamSegmentId(), result.getOrDefault(mergeOperation.getStreamSegmentId(), 0) + result.getOrDefault(mergeOperation.getTransactionSegmentId(), 0));
            result.remove(mergeOperation.getTransactionSegmentId());
        }
    }
    return result;
}

Example 25 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class OperationProcessorTests method testWithOperationSerializationFailures.

/**
 * Tests the ability of the OperationProcessor to process Operations when Serialization errors happen.
 */
@Test
public void testWithOperationSerializationFailures() throws Exception {
    int streamSegmentCount = 10;
    int appendsPerStreamSegment = 80;
    // Fail every X appends encountered.
    int failAppendFrequency = 7;
    @Cleanup TestContext context = new TestContext();
    // Generate some test data (no need to complicate ourselves with Transactions here; that is tested in the no-failure test).
    HashSet<Long> streamSegmentIds = createStreamSegmentsInMetadata(streamSegmentCount, context.metadata);
    List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
    // Replace some of the Append Operations with a FailedAppendOperation. Some operations fail at the beginning,
    // some at the end of the serialization.
    int appendCount = 0;
    HashSet<Integer> failedOperationIndices = new HashSet<>();
    for (int i = 0; i < operations.size(); i++) {
        if (operations.get(i) instanceof StreamSegmentAppendOperation) {
            if ((appendCount++) % failAppendFrequency == 0) {
                operations.set(i, new FailedStreamSegmentAppendOperation((StreamSegmentAppendOperation) operations.get(i)));
                failedOperationIndices.add(i);
            }
        }
    }
    // Setup an OperationProcessor and start it.
    @Cleanup TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, MAX_DATA_LOG_APPEND_SIZE, executorService());
    dataLog.initialize(TIMEOUT);
    @Cleanup OperationProcessor operationProcessor = new OperationProcessor(context.metadata, context.stateUpdater, dataLog, getNoOpCheckpointPolicy(), executorService());
    operationProcessor.startAsync().awaitRunning();
    // Process all generated operations.
    List<OperationWithCompletion> completionFutures = processOperations(operations, operationProcessor);
    // Wait for all such operations to complete. We are expecting exceptions, so verify that we do.
    AssertExtensions.assertThrows("No operations failed.", OperationWithCompletion.allOf(completionFutures)::join, ex -> ex instanceof IntentionalException);
    // Verify that the "right" operations failed, while the others succeeded.
    for (int i = 0; i < completionFutures.size(); i++) {
        OperationWithCompletion oc = completionFutures.get(i);
        if (failedOperationIndices.contains(i)) {
            AssertExtensions.assertThrows("Unexpected exception for failed Operation.", oc.completion::join, ex -> ex instanceof IntentionalException);
        } else {
            // Verify no exception was thrown.
            oc.completion.join();
        }
    }
    performLogOperationChecks(completionFutures, context.memoryLog, dataLog, context.metadata);
    performMetadataChecks(streamSegmentIds, new HashSet<>(), new HashMap<>(), completionFutures, context.metadata, false, false);
    performReadIndexChecks(completionFutures, context.readIndex);
    operationProcessor.stopAsync().awaitTerminated();
}