Examples with UpdateableSegmentMetadata io.pravega.segmentstore.server.UpdateableSegmentMetadata used on opensource projects

Example 11 with UpdateableSegmentMetadata

use of io.pravega.segmentstore.server.UpdateableSegmentMetadata in project pravega by pravega.

the class StorageWriter method getSegmentAggregator.

// endregion
// region Helpers
/**
 * Gets, or creates, a SegmentAggregator for the given StorageOperation.
 *
 * @param streamSegmentId The Id of the StreamSegment to get the aggregator for.
 */
private CompletableFuture<SegmentAggregator> getSegmentAggregator(long streamSegmentId) {
    SegmentAggregator existingAggregator = this.aggregators.getOrDefault(streamSegmentId, null);
    if (existingAggregator != null) {
        if (closeIfNecessary(existingAggregator).isClosed()) {
            // Existing SegmentAggregator has become stale (most likely due to its SegmentMetadata being evicted),
            // so it has been closed and we need to create a new one.
            this.aggregators.remove(streamSegmentId);
        } else {
            return CompletableFuture.completedFuture(existingAggregator);
        }
    }
    // Get the SegmentAggregator's Metadata.
    UpdateableSegmentMetadata segmentMetadata = this.dataSource.getStreamSegmentMetadata(streamSegmentId);
    if (segmentMetadata == null) {
        return Futures.failedFuture(new DataCorruptionException(String.format("No StreamSegment with id '%d' is registered in the metadata.", streamSegmentId)));
    }
    // Then create the aggregator, and only register it after a successful initialization. Otherwise we risk
    // having a registered aggregator that is not initialized.
    SegmentAggregator newAggregator = new SegmentAggregator(segmentMetadata, this.dataSource, this.storage, this.config, this.timer, this.executor);
    try {
        CompletableFuture<Void> init = newAggregator.initialize(this.config.getFlushTimeout());
        Futures.exceptionListener(init, ex -> newAggregator.close());
        return init.thenApply(ignored -> {
            this.aggregators.put(streamSegmentId, newAggregator);
            return newAggregator;
        });
    } catch (Exception ex) {
        newAggregator.close();
        throw ex;
    }
}

Example 12 with UpdateableSegmentMetadata

use of io.pravega.segmentstore.server.UpdateableSegmentMetadata in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method createSegmentMetadata.

/**
 * Creates a new UpdateableSegmentMetadata for the given Segment and registers it.
 */
private UpdateableSegmentMetadata createSegmentMetadata(String segmentName, long segmentId, long parentId) {
    UpdateableSegmentMetadata metadata;
    if (parentId == ContainerMetadata.NO_STREAM_SEGMENT_ID) {
        metadata = new StreamSegmentMetadata(segmentName, segmentId, this.containerId);
    } else {
        metadata = new StreamSegmentMetadata(segmentName, segmentId, parentId, this.containerId);
    }
    this.newSegments.put(metadata.getId(), metadata);
    this.newSegmentNames.put(metadata.getName(), metadata.getId());
    return metadata;
}

Example 13 with UpdateableSegmentMetadata

use of io.pravega.segmentstore.server.UpdateableSegmentMetadata in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method getOrCreateSegmentUpdateTransaction.

/**
 * Gets an UpdateableSegmentMetadata for the given Segment. If already registered, it returns that instance,
 * otherwise it creates and records a new Segment metadata.
 */
private SegmentMetadataUpdateTransaction getOrCreateSegmentUpdateTransaction(String segmentName, long segmentId, long parentId) {
    SegmentMetadataUpdateTransaction sm = tryGetSegmentUpdateTransaction(segmentId);
    if (sm == null) {
        SegmentMetadata baseSegmentMetadata = createSegmentMetadata(segmentName, segmentId, parentId);
        sm = new SegmentMetadataUpdateTransaction(baseSegmentMetadata, this.recoveryMode);
        this.segmentUpdates.put(segmentId, sm);
    }
    return sm;
}

Example 14 with UpdateableSegmentMetadata

use of io.pravega.segmentstore.server.UpdateableSegmentMetadata in project pravega by pravega.

the class ContainerMetadataUpdateTransaction method commit.

/**
 * Commits all pending changes to the given target Container Metadata.
 * @param target The UpdateableContainerMetadata to commit to.
 */
void commit(UpdateableContainerMetadata target) {
    Preconditions.checkArgument(target.getContainerId() == this.containerId, "ContainerId mismatch");
    Preconditions.checkArgument(target.isRecoveryMode() == this.isRecoveryMode(), "isRecoveryMode mismatch");
    if (target.isRecoveryMode()) {
        if (this.processedCheckpoint) {
            // If we processed a checkpoint during recovery, we need to wipe the metadata clean. We are setting
            // a brand new one.
            target.reset();
        }
        // we have in our UpdateTransaction. If we have nothing, we'll just set it to the default.
        assert this.newSequenceNumber >= ContainerMetadata.INITIAL_OPERATION_SEQUENCE_NUMBER : "Invalid Sequence Number " + this.newSequenceNumber;
        target.setOperationSequenceNumber(this.newSequenceNumber);
    }
    // Commit all segment-related transactional changes to their respective sources.
    this.segmentUpdates.values().forEach(txn -> {
        UpdateableSegmentMetadata targetSegmentMetadata = target.getStreamSegmentMetadata(txn.getId());
        if (targetSegmentMetadata == null) {
            targetSegmentMetadata = this.newSegments.get(txn.getId());
        }
        txn.apply(targetSegmentMetadata);
    });
    // We must first copy the Standalone Segments, and then the Transaction Segments. That's because
    // the Transaction Segments may refer to one of these newly created Segments, and the metadata
    // will reject the operation if it can't find the parent.
    // We need this because HashMap does not necessarily preserve order when iterating via values().
    copySegmentMetadata(this.newSegments.values(), s -> !s.isTransaction(), target);
    copySegmentMetadata(this.newSegments.values(), SegmentMetadata::isTransaction, target);
    // Copy truncation points.
    this.newTruncationPoints.forEach(target::setValidTruncationPoint);
    // We are done. Clear the transaction.
    clear();
}

Example 15 with UpdateableSegmentMetadata

use of io.pravega.segmentstore.server.UpdateableSegmentMetadata in project pravega by pravega.

the class DurableLogTests method testTailReads.

/**
 * Tests the ability to block reads if the read is at the tail and no more data is available (for now).
 */
@Test
public void testTailReads() throws Exception {
    final int operationCount = 10;
    final long segmentId = 1;
    final String segmentName = Long.toString(segmentId);
    // Setup a DurableLog and start it.
    @Cleanup ContainerSetup setup = new ContainerSetup(executorService());
    @Cleanup DurableLog durableLog = setup.createDurableLog();
    durableLog.startAsync().awaitRunning();
    // Create a segment, which will be used for testing later.
    UpdateableSegmentMetadata segmentMetadata = setup.metadata.mapStreamSegmentId(segmentName, segmentId);
    segmentMetadata.setLength(0);
    segmentMetadata.setStorageLength(0);
    // Setup a bunch of read operations, and make sure they are blocked (since there is no data).
    ArrayList<CompletableFuture<Iterator<Operation>>> readFutures = new ArrayList<>();
    for (int i = 0; i < operationCount; i++) {
        long afterSeqNo = i + 1;
        CompletableFuture<Iterator<Operation>> readFuture = durableLog.read(afterSeqNo, operationCount, TIMEOUT);
        Assert.assertFalse("read() returned a completed future when there is no data available (afterSeqNo = " + afterSeqNo + ").", readFuture.isDone());
        readFutures.add(readFuture);
    }
    // Add one operation at at time, and each time, verify that the correct Read got activated.
    OperationComparer operationComparer = new OperationComparer(true);
    for (int appendId = 0; appendId < operationCount; appendId++) {
        Operation operation = new StreamSegmentAppendOperation(segmentId, ("foo" + Integer.toString(appendId)).getBytes(), null);
        durableLog.add(operation, TIMEOUT).join();
        for (int readId = 0; readId < readFutures.size(); readId++) {
            val readFuture = readFutures.get(readId);
            boolean expectedComplete = readId <= appendId;
            if (expectedComplete) {
                // The internal callback happens asynchronously, so wait for this future to complete in a bit.
                readFuture.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            }
            Assert.assertEquals(String.format("Unexpected read completion status for read after seqNo %d after adding op with seqNo %d", readId + 1, operation.getSequenceNumber()), expectedComplete, readFutures.get(readId).isDone());
            if (appendId == readId) {
                // Verify that the read result matches the operation.
                Iterator<Operation> readResult = readFuture.join();
                // Verify that we actually have a non-empty read result.
                Assert.assertTrue(String.format("Empty read result read after seqNo %d after adding op with seqNo %d", readId + 1, operation.getSequenceNumber()), readResult.hasNext());
                // Verify the read result.
                Operation readOp = readResult.next();
                operationComparer.assertEquals(String.format("Unexpected result operation for read after seqNo %d after adding op with seqNo %d", readId + 1, operation.getSequenceNumber()), operation, readOp);
                // Verify that we don't have more than one read result.
                Assert.assertFalse(String.format("Not expecting more than one result for read after seqNo %d after adding op with seqNo %d", readId + 1, operation.getSequenceNumber()), readResult.hasNext());
            }
        }
    }
    // Verify that such reads are cancelled when the DurableLog is closed.
    CompletableFuture<Iterator<Operation>> readFuture = durableLog.read(operationCount + 2, operationCount, TIMEOUT);
    Assert.assertFalse("read() returned a completed future when there is no data available (afterSeqNo = MAX).", readFuture.isDone());
    durableLog.stopAsync().awaitTerminated();
    Assert.assertTrue("A tail read was not cancelled when the DurableLog was stopped.", readFuture.isCompletedExceptionally());
}