Examples with SegmentMetadata io.pravega.segmentstore.storage.metadata.SegmentMetadata used on opensource projects

Example 6 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class TruncateOperation method call.

@Override
public CompletableFuture<Void> call() {
    checkPreconditions();
    log.debug("{} truncate - started op={}, segment={}, offset={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), offset);
    val streamSegmentName = handle.getSegmentName();
    return ChunkedSegmentStorage.tryWith(chunkedSegmentStorage.getMetadataStore().beginTransaction(false, streamSegmentName), txn -> txn.get(streamSegmentName).thenComposeAsync(storageMetadata -> {
        segmentMetadata = (SegmentMetadata) storageMetadata;
        // Check preconditions
        checkPreconditions(streamSegmentName, segmentMetadata);
        if (segmentMetadata.getStartOffset() >= offset) {
            // Nothing to do
            logEnd();
            return CompletableFuture.completedFuture(null);
        }
        val oldChunkCount = segmentMetadata.getChunkCount();
        val oldStartOffset = segmentMetadata.getStartOffset();
        return updateFirstChunk(txn).thenComposeAsync(v -> relocateFirstChunkIfRequired(txn), chunkedSegmentStorage.getExecutor()).thenComposeAsync(v -> deleteChunks(txn).thenComposeAsync(vvv -> {
            txn.update(segmentMetadata);
            // Check invariants.
            segmentMetadata.checkInvariants();
            Preconditions.checkState(segmentMetadata.getLength() == oldLength, "truncate should not change segment length. oldLength=%s Segment=%s", oldLength, segmentMetadata);
            Preconditions.checkState(oldChunkCount - chunksToDelete.size() + (isFirstChunkRelocated ? 1 : 0) == segmentMetadata.getChunkCount(), "Number of chunks do not match. old value (%s) - number of chunks deleted (%s) + number of chunks added (%s) must match current chunk count(%s)", oldChunkCount, chunksToDelete.size(), segmentMetadata.getChunkCount());
            if (isFirstChunkRelocated) {
                Preconditions.checkState(segmentMetadata.getFirstChunkStartOffset() == segmentMetadata.getStartOffset(), "After relocation of first chunk FirstChunkStartOffset (%) must match StartOffset (%s)", segmentMetadata.getFirstChunkStartOffset(), segmentMetadata.getStartOffset());
            }
            if (null != currentMetadata && null != segmentMetadata.getFirstChunk()) {
                Preconditions.checkState(segmentMetadata.getFirstChunk().equals(currentMetadata.getName()), "First chunk name must match current metadata. Expected = %s Actual = %s", segmentMetadata.getFirstChunk(), currentMetadata.getName());
                Preconditions.checkState(segmentMetadata.getStartOffset() <= segmentMetadata.getFirstChunkStartOffset() + currentMetadata.getLength(), "segment start offset (%s) must be less than or equal to first chunk start offset (%s)+ first chunk length (%s)", segmentMetadata.getStartOffset(), segmentMetadata.getFirstChunkStartOffset(), currentMetadata.getLength());
                if (segmentMetadata.getChunkCount() == 1) {
                    Preconditions.checkState(segmentMetadata.getLength() - segmentMetadata.getFirstChunkStartOffset() == currentMetadata.getLength(), "Length of first chunk (%s) must match segment length (%s) - first chunk start offset (%s) when there is only one chunk", currentMetadata.getLength(), segmentMetadata.getLength(), segmentMetadata.getFirstChunkStartOffset());
                }
            }
            // To avoid possibility of unintentional deadlock, skip this step for storage system segments.
            if (!segmentMetadata.isStorageSystemSegment()) {
                chunkedSegmentStorage.deleteBlockIndexEntriesForChunk(txn, streamSegmentName, oldStartOffset, segmentMetadata.getStartOffset());
            }
            // Collect garbage.
            return chunkedSegmentStorage.getGarbageCollector().addChunksToGarbage(txn.getVersion(), chunksToDelete).thenComposeAsync(vv -> {
                // Finally, commit.
                return commit(txn).handleAsync(this::handleException, chunkedSegmentStorage.getExecutor()).thenRunAsync(this::postCommit, chunkedSegmentStorage.getExecutor());
            }, chunkedSegmentStorage.getExecutor());
        }, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor());
    }, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor());
}

Example 7 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class ChunkedSegmentStorage method claimOwnership.

/**
 * Checks ownership and adjusts the length of the segment if required.
 *
 * @param txn             Active {@link MetadataTransaction}.
 * @param segmentMetadata {@link SegmentMetadata} for the segment to change ownership for.
 *                        throws ChunkStorageException    In case of any chunk storage related errors.
 *                        throws StorageMetadataException In case of any chunk metadata store related errors.
 */
private CompletableFuture<Void> claimOwnership(MetadataTransaction txn, SegmentMetadata segmentMetadata) {
    // Get the last chunk
    val lastChunkName = segmentMetadata.getLastChunk();
    final CompletableFuture<Boolean> f;
    if (shouldAppend() && null != lastChunkName) {
        f = txn.get(lastChunkName).thenComposeAsync(storageMetadata -> {
            val lastChunk = (ChunkMetadata) storageMetadata;
            Preconditions.checkState(null != lastChunk, "last chunk metadata must not be null.");
            Preconditions.checkState(null != lastChunk.getName(), "Name of last chunk must not be null.");
            log.debug("{} claimOwnership - current last chunk - segment={}, last chunk={}, Length={}.", logPrefix, segmentMetadata.getName(), lastChunk.getName(), lastChunk.getLength());
            return chunkStorage.getInfo(lastChunkName).thenApplyAsync(chunkInfo -> {
                Preconditions.checkState(chunkInfo != null, "chunkInfo for last chunk must not be null.");
                Preconditions.checkState(lastChunk != null, "last chunk metadata must not be null.");
                // Adjust its length;
                if (chunkInfo.getLength() != lastChunk.getLength()) {
                    Preconditions.checkState(chunkInfo.getLength() > lastChunk.getLength(), "Length of last chunk on LTS must be greater than what is in metadata. Chunk=%s length=%s", lastChunk, chunkInfo.getLength());
                    // Whatever length you see right now is the final "sealed" length of the last chunk.
                    val oldLength = segmentMetadata.getLength();
                    lastChunk.setLength(chunkInfo.getLength());
                    segmentMetadata.setLength(segmentMetadata.getLastChunkStartOffset() + lastChunk.getLength());
                    if (!segmentMetadata.isStorageSystemSegment()) {
                        addBlockIndexEntriesForChunk(txn, segmentMetadata.getName(), lastChunk.getName(), segmentMetadata.getLastChunkStartOffset(), oldLength, segmentMetadata.getLength());
                    }
                    txn.update(lastChunk);
                    log.debug("{} claimOwnership - Length of last chunk adjusted - segment={}, last chunk={}, Length={}.", logPrefix, segmentMetadata.getName(), lastChunk.getName(), chunkInfo.getLength());
                }
                return true;
            }, executor).exceptionally(e -> {
                val ex = Exceptions.unwrap(e);
                if (ex instanceof ChunkNotFoundException) {
                    // This probably means that this instance is fenced out and newer instance truncated this segment.
                    // Try a commit of unmodified data to fail fast.
                    log.debug("{} claimOwnership - Last chunk was missing, failing fast - segment={}, last chunk={}.", logPrefix, segmentMetadata.getName(), lastChunk.getName());
                    txn.update(segmentMetadata);
                    return false;
                }
                throw new CompletionException(ex);
            });
        }, executor);
    } else {
        f = CompletableFuture.completedFuture(true);
    }
    return f.thenComposeAsync(shouldChange -> {
        // If this instance is no more owner, then transaction commit will fail.So it is still safe.
        if (shouldChange) {
            segmentMetadata.setOwnerEpoch(this.epoch);
            segmentMetadata.setOwnershipChanged(true);
        }
        // Update and commit
        // If This instance is fenced this update will fail.
        txn.update(segmentMetadata);
        return txn.commit();
    }, executor);
}

Example 8 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class ChunkedSegmentStorage method delete.

@Override
public CompletableFuture<Void> delete(SegmentHandle handle, Duration timeout) {
    checkInitialized();
    if (null == handle) {
        return CompletableFuture.failedFuture(new IllegalArgumentException("handle must not be null"));
    }
    return executeSerialized(() -> {
        val traceId = LoggerHelpers.traceEnter(log, "delete", handle);
        log.debug("{} delete - started segment={}.", logPrefix, handle.getSegmentName());
        val timer = new Timer();
        val streamSegmentName = handle.getSegmentName();
        return tryWith(metadataStore.beginTransaction(false, streamSegmentName), txn -> txn.get(streamSegmentName).thenComposeAsync(storageMetadata -> {
            val segmentMetadata = (SegmentMetadata) storageMetadata;
            // Check preconditions
            checkSegmentExists(streamSegmentName, segmentMetadata);
            checkOwnership(streamSegmentName, segmentMetadata);
            segmentMetadata.setActive(false);
            txn.update(segmentMetadata);
            // Collect garbage
            return garbageCollector.addSegmentToGarbage(txn.getVersion(), streamSegmentName).thenComposeAsync(vv -> {
                // Commit metadata.
                return txn.commit().thenRunAsync(() -> {
                    // Update the read index.
                    readIndexCache.remove(streamSegmentName);
                    val elapsed = timer.getElapsed();
                    SLTS_DELETE_LATENCY.reportSuccessEvent(elapsed);
                    SLTS_DELETE_COUNT.inc();
                    log.debug("{} delete - finished segment={}, latency={}.", logPrefix, handle.getSegmentName(), elapsed.toMillis());
                    LoggerHelpers.traceLeave(log, "delete", traceId, handle);
                }, executor);
            }, executor);
        }, executor), executor).exceptionally(ex -> {
            log.warn("{} delete - exception segment={}, latency={}.", logPrefix, handle.getSegmentName(), timer.getElapsedMillis(), ex);
            handleException(streamSegmentName, ex);
            return null;
        });
    }, handle.getSegmentName());
}

Example 9 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class TestUtils method insertMetadata.

/**
 * Insert Metadata as given.
 *
 * @param testSegmentName Name of the segment
 * @param maxRollingLength Max rolling length.
 * @param ownerEpoch Owner epoch.
 * @param chunkLengthsInMetadata Chunk lengths to set in metadata.
 * @param chunkLengthsInStorage Chunk lengths to set in storage.
 * @param addIndex Whether to add index.
 * @param addIndexMetadata Whether to add index metadata.
 * @param metadataStore Instance of {@link ChunkMetadataStore}
 * @param chunkedSegmentStorage Instance of {@link ChunkedSegmentStorage}.
 * @return {@link SegmentMetadata} representing segment.
 */
public static SegmentMetadata insertMetadata(String testSegmentName, long maxRollingLength, int ownerEpoch, long[] chunkLengthsInMetadata, long[] chunkLengthsInStorage, boolean addIndex, boolean addIndexMetadata, ChunkMetadataStore metadataStore, ChunkedSegmentStorage chunkedSegmentStorage) {
    Preconditions.checkArgument(maxRollingLength > 0, "maxRollingLength");
    Preconditions.checkArgument(ownerEpoch > 0, "ownerEpoch");
    try (val txn = metadataStore.beginTransaction(false, new String[] { testSegmentName })) {
        String firstChunk = null;
        String lastChunk = null;
        TreeMap<Long, String> index = new TreeMap<>();
        // Add chunks.
        long length = 0;
        long startOfLast = 0;
        long startOffset = 0;
        int chunkCount = 0;
        for (int i = 0; i < chunkLengthsInMetadata.length; i++) {
            String chunkName = testSegmentName + "_chunk_" + Integer.toString(i);
            ChunkMetadata chunkMetadata = ChunkMetadata.builder().name(chunkName).length(chunkLengthsInMetadata[i]).nextChunk(i == chunkLengthsInMetadata.length - 1 ? null : testSegmentName + "_chunk_" + Integer.toString(i + 1)).build();
            chunkMetadata.setActive(true);
            if (addIndex) {
                chunkedSegmentStorage.getReadIndexCache().addIndexEntry(testSegmentName, chunkName, startOffset);
            }
            index.put(startOffset, chunkName);
            startOffset += chunkLengthsInMetadata[i];
            length += chunkLengthsInMetadata[i];
            txn.create(chunkMetadata);
            addChunk(chunkedSegmentStorage.getChunkStorage(), chunkName, chunkLengthsInStorage[i]);
            chunkCount++;
        }
        // Fix the first and last
        if (chunkLengthsInMetadata.length > 0) {
            firstChunk = testSegmentName + "_chunk_0";
            lastChunk = testSegmentName + "_chunk_" + Integer.toString(chunkLengthsInMetadata.length - 1);
            startOfLast = length - chunkLengthsInMetadata[chunkLengthsInMetadata.length - 1];
        }
        // Finally save
        SegmentMetadata segmentMetadata = SegmentMetadata.builder().maxRollinglength(maxRollingLength).name(testSegmentName).ownerEpoch(ownerEpoch).firstChunk(firstChunk).lastChunk(lastChunk).length(length).lastChunkStartOffset(startOfLast).build();
        segmentMetadata.setActive(true);
        segmentMetadata.setChunkCount(chunkCount);
        segmentMetadata.checkInvariants();
        txn.create(segmentMetadata);
        if (addIndexMetadata) {
            for (long blockStartOffset = 0; blockStartOffset < segmentMetadata.getLength(); blockStartOffset += chunkedSegmentStorage.getConfig().getIndexBlockSize()) {
                val floor = index.floorEntry(blockStartOffset);
                txn.create(ReadIndexBlockMetadata.builder().name(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), blockStartOffset)).startOffset(floor.getKey()).chunkName(floor.getValue()).status(StatusFlags.ACTIVE).build());
            }
        }
        txn.commit().join();
        return segmentMetadata;
    }
}

Example 10 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class ChunkedSegmentStorage method seal.

@Override
public CompletableFuture<Void> seal(SegmentHandle handle, Duration timeout) {
    checkInitialized();
    return executeSerialized(() -> {
        val traceId = LoggerHelpers.traceEnter(log, "seal", handle);
        Timer timer = new Timer();
        log.debug("{} seal - started segment={}.", logPrefix, handle.getSegmentName());
        Preconditions.checkNotNull(handle, "handle");
        String streamSegmentName = handle.getSegmentName();
        Preconditions.checkNotNull(streamSegmentName, "streamSegmentName");
        Preconditions.checkArgument(!handle.isReadOnly(), "handle must not be read only. Segment=%s", handle.getSegmentName());
        return tryWith(metadataStore.beginTransaction(false, handle.getSegmentName()), txn -> txn.get(streamSegmentName).thenComposeAsync(storageMetadata -> {
            val segmentMetadata = (SegmentMetadata) storageMetadata;
            // Validate preconditions.
            checkSegmentExists(streamSegmentName, segmentMetadata);
            checkOwnership(streamSegmentName, segmentMetadata);
            // seal if it is not already sealed.
            if (!segmentMetadata.isSealed()) {
                segmentMetadata.setSealed(true);
                txn.update(segmentMetadata);
                return txn.commit();
            } else {
                return CompletableFuture.completedFuture(null);
            }
        }, executor).thenRunAsync(() -> {
            log.debug("{} seal - finished segment={} latency={}.", logPrefix, handle.getSegmentName(), timer.getElapsedMillis());
            LoggerHelpers.traceLeave(log, "seal", traceId, handle);
        }, executor), executor).exceptionally(ex -> {
            log.warn("{} seal - exception segment={} latency={}.", logPrefix, handle.getSegmentName(), timer.getElapsedMillis(), ex);
            handleException(streamSegmentName, ex);
            return null;
        });
    }, handle.getSegmentName());
}