Examples with StreamSegmentAppendOperation io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation used on opensource projects

Example 11 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class StreamSegmentContainer method append.

@Override
public CompletableFuture<Void> append(String streamSegmentName, long offset, byte[] data, Collection<AttributeUpdate> attributeUpdates, Duration timeout) {
    ensureRunning();
    TimeoutTimer timer = new TimeoutTimer(timeout);
    logRequest("appendWithOffset", streamSegmentName, data.length);
    this.metrics.appendWithOffset();
    return this.segmentMapper.getOrAssignStreamSegmentId(streamSegmentName, timer.getRemaining(), streamSegmentId -> {
        StreamSegmentAppendOperation operation = new StreamSegmentAppendOperation(streamSegmentId, offset, data, attributeUpdates);
        return this.durableLog.add(operation, timer.getRemaining());
    });
}

Example 12 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method preProcessOperation.

/**
 * Pre-processes the given Operation. See OperationMetadataUpdater.preProcessOperation for more details on behavior.
 *
 * @param operation The operation to pre-process.
 * @throws ContainerException     If the given operation was rejected given the current state of the container metadata.
 * @throws StreamSegmentException If the given operation was incompatible with the current state of the Segment.
 *                                For example: StreamSegmentNotExistsException, StreamSegmentSealedException or
 *                                StreamSegmentMergedException.
 */
void preProcessOperation(Operation operation) throws ContainerException, StreamSegmentException {
    checkNotSealed();
    SegmentMetadataUpdateTransaction segmentMetadata = null;
    if (operation instanceof SegmentOperation) {
        segmentMetadata = getSegmentUpdateTransaction(((SegmentOperation) operation).getStreamSegmentId());
        if (segmentMetadata.isDeleted()) {
            throw new StreamSegmentNotExistsException(segmentMetadata.getName());
        }
    }
    if (operation instanceof StreamSegmentAppendOperation) {
        segmentMetadata.preProcessOperation((StreamSegmentAppendOperation) operation);
    } else if (operation instanceof StreamSegmentSealOperation) {
        segmentMetadata.preProcessOperation((StreamSegmentSealOperation) operation);
    } else if (operation instanceof MergeTransactionOperation) {
        MergeTransactionOperation mbe = (MergeTransactionOperation) operation;
        SegmentMetadataUpdateTransaction transactionMetadata = getSegmentUpdateTransaction(mbe.getTransactionSegmentId());
        transactionMetadata.preProcessAsTransactionSegment(mbe);
        segmentMetadata.preProcessAsParentSegment(mbe, transactionMetadata);
    } else if (operation instanceof StreamSegmentMapOperation) {
        preProcessMetadataOperation((StreamSegmentMapOperation) operation);
    } else if (operation instanceof UpdateAttributesOperation) {
        segmentMetadata.preProcessOperation((UpdateAttributesOperation) operation);
    } else if (operation instanceof MetadataCheckpointOperation) {
        // MetadataCheckpointOperations do not require preProcess and accept; they can be handled in a single stage.
        processMetadataOperation((MetadataCheckpointOperation) operation);
    } else if (operation instanceof StorageMetadataCheckpointOperation) {
        // StorageMetadataCheckpointOperation do not require preProcess and accept; they can be handled in a single stage.
        processMetadataOperation((StorageMetadataCheckpointOperation) operation);
    } else if (operation instanceof StreamSegmentTruncateOperation) {
        segmentMetadata.preProcessOperation((StreamSegmentTruncateOperation) operation);
    }
}

Example 13 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class DurableLogTests method testTailReads.

/**
 * Tests the ability to block reads if the read is at the tail and no more data is available (for now).
 */
@Test
public void testTailReads() throws Exception {
    final int operationCount = 10;
    final long segmentId = 1;
    final String segmentName = Long.toString(segmentId);
    // Setup a DurableLog and start it.
    @Cleanup ContainerSetup setup = new ContainerSetup(executorService());
    @Cleanup DurableLog durableLog = setup.createDurableLog();
    durableLog.startAsync().awaitRunning();
    // Create a segment, which will be used for testing later.
    UpdateableSegmentMetadata segmentMetadata = setup.metadata.mapStreamSegmentId(segmentName, segmentId);
    segmentMetadata.setLength(0);
    segmentMetadata.setStorageLength(0);
    // Setup a bunch of read operations, and make sure they are blocked (since there is no data).
    ArrayList<CompletableFuture<Iterator<Operation>>> readFutures = new ArrayList<>();
    for (int i = 0; i < operationCount; i++) {
        long afterSeqNo = i + 1;
        CompletableFuture<Iterator<Operation>> readFuture = durableLog.read(afterSeqNo, operationCount, TIMEOUT);
        Assert.assertFalse("read() returned a completed future when there is no data available (afterSeqNo = " + afterSeqNo + ").", readFuture.isDone());
        readFutures.add(readFuture);
    }
    // Add one operation at at time, and each time, verify that the correct Read got activated.
    OperationComparer operationComparer = new OperationComparer(true);
    for (int appendId = 0; appendId < operationCount; appendId++) {
        Operation operation = new StreamSegmentAppendOperation(segmentId, ("foo" + Integer.toString(appendId)).getBytes(), null);
        durableLog.add(operation, TIMEOUT).join();
        for (int readId = 0; readId < readFutures.size(); readId++) {
            val readFuture = readFutures.get(readId);
            boolean expectedComplete = readId <= appendId;
            if (expectedComplete) {
                // The internal callback happens asynchronously, so wait for this future to complete in a bit.
                readFuture.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            }
            Assert.assertEquals(String.format("Unexpected read completion status for read after seqNo %d after adding op with seqNo %d", readId + 1, operation.getSequenceNumber()), expectedComplete, readFutures.get(readId).isDone());
            if (appendId == readId) {
                // Verify that the read result matches the operation.
                Iterator<Operation> readResult = readFuture.join();
                // Verify that we actually have a non-empty read result.
                Assert.assertTrue(String.format("Empty read result read after seqNo %d after adding op with seqNo %d", readId + 1, operation.getSequenceNumber()), readResult.hasNext());
                // Verify the read result.
                Operation readOp = readResult.next();
                operationComparer.assertEquals(String.format("Unexpected result operation for read after seqNo %d after adding op with seqNo %d", readId + 1, operation.getSequenceNumber()), operation, readOp);
                // Verify that we don't have more than one read result.
                Assert.assertFalse(String.format("Not expecting more than one result for read after seqNo %d after adding op with seqNo %d", readId + 1, operation.getSequenceNumber()), readResult.hasNext());
            }
        }
    }
    // Verify that such reads are cancelled when the DurableLog is closed.
    CompletableFuture<Iterator<Operation>> readFuture = durableLog.read(operationCount + 2, operationCount, TIMEOUT);
    Assert.assertFalse("read() returned a completed future when there is no data available (afterSeqNo = MAX).", readFuture.isDone());
    durableLog.stopAsync().awaitTerminated();
    Assert.assertTrue("A tail read was not cancelled when the DurableLog was stopped.", readFuture.isCompletedExceptionally());
}

Example 14 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class MemoryStateUpdaterTests method populate.

private ArrayList<Operation> populate(MemoryStateUpdater updater, int segmentCount, int operationCountPerType) throws DataCorruptionException {
    ArrayList<Operation> operations = new ArrayList<>();
    long offset = 0;
    for (int i = 0; i < segmentCount; i++) {
        for (int j = 0; j < operationCountPerType; j++) {
            StreamSegmentMapOperation mapOp = new StreamSegmentMapOperation(StreamSegmentInformation.builder().name("a").length(i * j).build());
            mapOp.setStreamSegmentId(i);
            operations.add(mapOp);
            StreamSegmentAppendOperation appendOp = new StreamSegmentAppendOperation(i, Integer.toString(i).getBytes(), null);
            appendOp.setStreamSegmentOffset(offset);
            offset += appendOp.getData().length;
            operations.add(appendOp);
            operations.add(new MergeTransactionOperation(i, j));
        }
    }
    for (int i = 0; i < operations.size(); i++) {
        operations.get(i).setSequenceNumber(i);
    }
    updater.process(operations.iterator());
    return operations;
}

Example 15 with StreamSegmentAppendOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentAppendOperation in project pravega by pravega.

the class MemoryStateUpdaterTests method testProcess.

/**
 * Tests the functionality of the process() method.
 */
@Test
public void testProcess() throws Exception {
    int segmentCount = 10;
    int operationCountPerType = 5;
    // Add to MTL + Add to ReadIndex (append; beginMerge).
    SequencedItemList<Operation> opLog = new SequencedItemList<>();
    ArrayList<TestReadIndex.MethodInvocation> methodInvocations = new ArrayList<>();
    TestReadIndex readIndex = new TestReadIndex(methodInvocations::add);
    AtomicInteger flushCallbackCallCount = new AtomicInteger();
    MemoryStateUpdater updater = new MemoryStateUpdater(opLog, readIndex, flushCallbackCallCount::incrementAndGet);
    ArrayList<Operation> operations = populate(updater, segmentCount, operationCountPerType);
    // Verify they were properly processed.
    int triggerFutureCount = (int) methodInvocations.stream().filter(mi -> mi.methodName.equals(TestReadIndex.TRIGGER_FUTURE_READS)).count();
    int addCount = methodInvocations.size() - triggerFutureCount;
    Assert.assertEquals("Unexpected number of items added to ReadIndex.", operations.size() - segmentCount * operationCountPerType, addCount);
    Assert.assertEquals("Unexpected number of calls to the ReadIndex triggerFutureReads method.", 1, triggerFutureCount);
    Assert.assertEquals("Unexpected number of calls to the flushCallback provided in the constructor.", 1, flushCallbackCallCount.get());
    // Verify add calls.
    Iterator<Operation> logIterator = opLog.read(-1, operations.size());
    int currentIndex = -1;
    int currentReadIndex = -1;
    while (logIterator.hasNext()) {
        currentIndex++;
        Operation expected = operations.get(currentIndex);
        Operation actual = logIterator.next();
        if (expected instanceof StorageOperation) {
            currentReadIndex++;
            TestReadIndex.MethodInvocation invokedMethod = methodInvocations.get(currentReadIndex);
            if (expected instanceof StreamSegmentAppendOperation) {
                Assert.assertTrue("StreamSegmentAppendOperation was not added as a CachedStreamSegmentAppendOperation to the Memory Log.", actual instanceof CachedStreamSegmentAppendOperation);
                StreamSegmentAppendOperation appendOp = (StreamSegmentAppendOperation) expected;
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was not added to the ReadIndex.", TestReadIndex.APPEND, invokedMethod.methodName);
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getStreamSegmentId(), invokedMethod.args.get("streamSegmentId"));
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getStreamSegmentOffset(), invokedMethod.args.get("offset"));
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getData(), invokedMethod.args.get("data"));
            } else if (expected instanceof MergeTransactionOperation) {
                MergeTransactionOperation mergeOp = (MergeTransactionOperation) expected;
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was not added to the ReadIndex.", TestReadIndex.BEGIN_MERGE, invokedMethod.methodName);
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getStreamSegmentId(), invokedMethod.args.get("targetStreamSegmentId"));
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getStreamSegmentOffset(), invokedMethod.args.get("offset"));
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getTransactionSegmentId(), invokedMethod.args.get("sourceStreamSegmentId"));
            }
        }
    }
    // Verify triggerFutureReads args.
    @SuppressWarnings("unchecked") Collection<Long> triggerSegmentIds = (Collection<Long>) methodInvocations.stream().filter(mi -> mi.methodName.equals(TestReadIndex.TRIGGER_FUTURE_READS)).findFirst().get().args.get("streamSegmentIds");
    val expectedSegmentIds = operations.stream().filter(op -> op instanceof SegmentOperation).map(op -> ((SegmentOperation) op).getStreamSegmentId()).collect(Collectors.toSet());
    AssertExtensions.assertContainsSameElements("ReadIndex.triggerFutureReads() was called with the wrong set of StreamSegmentIds.", expectedSegmentIds, triggerSegmentIds);
    // Test DataCorruptionException.
    AssertExtensions.assertThrows("MemoryStateUpdater accepted an operation that was out of order.", // This does not have a SequenceNumber set, so it should trigger a DCE.
    () -> updater.process(new MergeTransactionOperation(1, 2)), ex -> ex instanceof DataCorruptionException);
}