Examples with SegmentMetadata io.pravega.segmentstore.storage.metadata.SegmentMetadata used on opensource projects

Example 16 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class ReadOperation method findChunkForOffset.

private CompletableFuture<Void> findChunkForOffset(MetadataTransaction txn) {
    currentChunkName = segmentMetadata.getFirstChunk();
    chunkToReadFrom = null;
    Preconditions.checkState(null != currentChunkName, "currentChunkName must not be null. Segment=%s", segmentMetadata.getName());
    bytesRemaining.set(length);
    currentBufferOffset.set(bufferOffset);
    currentOffset.set(offset);
    totalBytesRead.set(0);
    // Find the first chunk that contains the data.
    startOffsetForCurrentChunk.set(segmentMetadata.getFirstChunkStartOffset());
    boolean shouldOnlyReadLastChunk = offset >= segmentMetadata.getLastChunkStartOffset();
    if (shouldOnlyReadLastChunk) {
        startOffsetForCurrentChunk.set(segmentMetadata.getLastChunkStartOffset());
        currentChunkName = segmentMetadata.getLastChunk();
    } else {
        // Find the name of the chunk in the cached read index that is floor to required offset.
        val floorEntry = chunkedSegmentStorage.getReadIndexCache().findFloor(handle.getSegmentName(), offset);
        if (null != floorEntry && startOffsetForCurrentChunk.get() < floorEntry.getOffset() && null != floorEntry.getChunkName()) {
            startOffsetForCurrentChunk.set(floorEntry.getOffset());
            currentChunkName = floorEntry.getChunkName();
        }
    }
    final long floorBlockStartOffset = getFloorBlockStartOffset(offset);
    CompletableFuture<Void> f;
    if (!shouldOnlyReadLastChunk && !segmentMetadata.isStorageSystemSegment() && startOffsetForCurrentChunk.get() < floorBlockStartOffset) {
        val indexLookupTimer = new Timer();
        f = txn.get(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), floorBlockStartOffset)).thenAcceptAsync(storageMetadata -> {
            if (null != storageMetadata) {
                ReadIndexBlockMetadata blockMetadata = (ReadIndexBlockMetadata) storageMetadata;
                if (blockMetadata.getStartOffset() <= offset) {
                    startOffsetForCurrentChunk.set(blockMetadata.getStartOffset());
                    currentChunkName = blockMetadata.getChunkName();
                    log.debug("{} read - found block index to start scanning - op={}, segment={}, chunk={}, startOffset={}, offset={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), currentChunkName, startOffsetForCurrentChunk.get(), offset);
                    // Note: This just is prefetch call. Do not wait.
                    val nextBlock = getFloorBlockStartOffset(offset + length);
                    if (nextBlock > floorBlockStartOffset + chunkedSegmentStorage.getConfig().getIndexBlockSize()) {
                        // We read multiple blocks already
                        txn.get(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), nextBlock));
                    } else {
                        // Prefetch next block index entry.
                        txn.get(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), floorBlockStartOffset + chunkedSegmentStorage.getConfig().getIndexBlockSize()));
                    }
                } else {
                    log.warn("{} read - block entry offset must be floor to requested offset. op={} segment={} offset={} length={} block={}", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), segmentMetadata, offset, length, blockMetadata);
                }
            }
            if (segmentMetadata.isStorageSystemSegment()) {
                SLTS_SYS_READ_INDEX_BLOCK_LOOKUP_LATENCY.reportSuccessEvent(indexLookupTimer.getElapsed());
            } else {
                SLTS_READ_INDEX_BLOCK_LOOKUP_LATENCY.reportSuccessEvent(indexLookupTimer.getElapsed());
            }
        }, chunkedSegmentStorage.getExecutor());
    } else {
        f = CompletableFuture.completedFuture(null);
    }
    val readIndexTimer = new Timer();
    // Navigate to the chunk that contains the first byte of requested data.
    return f.thenComposeAsync(vv -> Futures.loop(() -> currentChunkName != null && !isLoopExited, () -> txn.get(currentChunkName).thenAcceptAsync(storageMetadata -> {
        chunkToReadFrom = (ChunkMetadata) storageMetadata;
        Preconditions.checkState(null != chunkToReadFrom, "chunkToReadFrom is null. currentChunkName=%s Segment=%s", currentChunkName, segmentMetadata.getName());
        if (startOffsetForCurrentChunk.get() <= currentOffset.get() && startOffsetForCurrentChunk.get() + chunkToReadFrom.getLength() > currentOffset.get()) {
            // we have found a chunk that contains first byte we want to read
            log.debug("{} read - found chunk to read - op={}, segment={}, chunk={}, startOffset={}, length={}, readOffset={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), chunkToReadFrom, startOffsetForCurrentChunk.get(), chunkToReadFrom.getLength(), currentOffset);
            isLoopExited = true;
            return;
        }
        currentChunkName = chunkToReadFrom.getNextChunk();
        startOffsetForCurrentChunk.addAndGet(chunkToReadFrom.getLength());
        // Update read index with newly visited chunk.
        if (null != currentChunkName) {
            chunkedSegmentStorage.getReadIndexCache().addIndexEntry(handle.getSegmentName(), currentChunkName, startOffsetForCurrentChunk.get());
        }
        cntScanned.incrementAndGet();
    }, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor()).thenAcceptAsync(v -> {
        val elapsed = readIndexTimer.getElapsed();
        if (segmentMetadata.isStorageSystemSegment()) {
            SLTS_SYS_READ_INDEX_SCAN_LATENCY.reportSuccessEvent(elapsed);
            SLTS_SYS_READ_INDEX_NUM_SCANNED.reportSuccessValue(cntScanned.get());
        } else {
            SLTS_READ_INDEX_SCAN_LATENCY.reportSuccessEvent(elapsed);
            SLTS_READ_INDEX_NUM_SCANNED.reportSuccessValue(cntScanned.get());
        }
        // Prefetch possible chunks for next read.
        if (chunkToReadFrom.getNextChunk() != null) {
            // Do not wait.
            txn.get(chunkToReadFrom.getNextChunk());
        }
        log.debug("{} read - chunk lookup - op={}, segment={}, offset={}, scanned={}, latency={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), offset, cntScanned.get(), elapsed.toMillis());
    }, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor());
}

Example 17 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class WriteOperation method writeToChunk.

/**
 * Write to chunk.
 */
private CompletableFuture<Void> writeToChunk(MetadataTransaction txn, SegmentMetadata segmentMetadata, InputStream data, ChunkHandle chunkHandle, ChunkMetadata chunkWrittenMetadata, long offsetToWriteAt, int bytesCount) {
    Preconditions.checkState(0 != bytesCount, "Attempt to write zero bytes. Segment=%s Chunk=%s offsetToWriteAt=%s", segmentMetadata, chunkWrittenMetadata, offsetToWriteAt);
    // Finally write the data.
    val bis = new BoundedInputStream(data, bytesCount);
    CompletableFuture<Integer> retValue;
    if (chunkedSegmentStorage.shouldAppend()) {
        retValue = chunkedSegmentStorage.getChunkStorage().write(chunkHandle, offsetToWriteAt, bytesCount, bis);
    } else {
        retValue = chunkedSegmentStorage.getChunkStorage().createWithContent(chunkHandle.getChunkName(), bytesCount, bis).thenApplyAsync(h -> bytesCount, chunkedSegmentStorage.getExecutor());
    }
    return retValue.thenAcceptAsync(bytesWritten -> {
        // Update the metadata for segment and chunk.
        Preconditions.checkState(bytesWritten >= 0, "bytesWritten (%s) must be non-negative. Segment=%s Chunk=%s offsetToWriteAt=%s", bytesWritten, segmentMetadata, chunkWrittenMetadata, offsetToWriteAt);
        segmentMetadata.setLength(segmentMetadata.getLength() + bytesWritten);
        chunkWrittenMetadata.setLength(chunkWrittenMetadata.getLength() + bytesWritten);
        txn.update(chunkWrittenMetadata);
        txn.update(segmentMetadata);
        bytesRemaining.addAndGet(-bytesWritten);
        currentOffset.addAndGet(bytesWritten);
    }, chunkedSegmentStorage.getExecutor()).handleAsync((v, e) -> {
        if (null != e) {
            val ex = Exceptions.unwrap(e);
            if (ex instanceof InvalidOffsetException) {
                val invalidEx = (InvalidOffsetException) ex;
                // This could happen if the previous write failed while writing data and chunk was partially written.
                if (invalidEx.getExpectedOffset() > offsetToWriteAt) {
                    skipOverFailedChunk = true;
                    log.debug("{} write - skipping partially written chunk op={}, segment={}, chunk={} expected={} given={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), chunkHandle.getChunkName(), invalidEx.getExpectedOffset(), invalidEx.getGivenOffset());
                    return null;
                }
                throw new CompletionException(new BadOffsetException(segmentMetadata.getName(), currentOffset.get() + ((InvalidOffsetException) ex).getExpectedOffset(), currentOffset.get() + ((InvalidOffsetException) ex).getGivenOffset()));
            }
            throw new CompletionException(ex);
        }
        return v;
    }, chunkedSegmentStorage.getExecutor());
}

Example 18 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class WriteOperation method writeData.

private CompletableFuture<Void> writeData(MetadataTransaction txn) {
    val oldChunkCount = segmentMetadata.getChunkCount();
    val oldLength = segmentMetadata.getLength();
    return Futures.loop(() -> bytesRemaining.get() > 0, () -> {
        // This could be either because there are no existing chunks or last chunk has reached max rolling length.
        return openChunkToWrite(txn).thenComposeAsync(v -> {
            // Calculate the data that needs to be written.
            val oldOffset = currentOffset.get();
            val offsetToWriteAt = currentOffset.get() - segmentMetadata.getLastChunkStartOffset();
            val writeSize = (int) Math.min(bytesRemaining.get(), segmentMetadata.getMaxRollinglength() - offsetToWriteAt);
            // Write data to last chunk.
            return writeToChunk(txn, segmentMetadata, data, chunkHandle, lastChunkMetadata.get(), offsetToWriteAt, writeSize).thenRunAsync(() -> {
                // Update block index.
                if (!segmentMetadata.isStorageSystemSegment()) {
                    chunkedSegmentStorage.addBlockIndexEntriesForChunk(txn, segmentMetadata.getName(), chunkHandle.getChunkName(), segmentMetadata.getLastChunkStartOffset(), oldOffset, segmentMetadata.getLength());
                }
            }, chunkedSegmentStorage.getExecutor());
        }, chunkedSegmentStorage.getExecutor());
    }, chunkedSegmentStorage.getExecutor()).thenRunAsync(() -> {
        // Check invariants.
        segmentMetadata.checkInvariants();
        Preconditions.checkState(oldChunkCount + chunksAddedCount.get() == segmentMetadata.getChunkCount(), "Number of chunks do not match. old value (%s) + number of chunks added (%s) must match current chunk count(%s)", oldChunkCount, chunksAddedCount.get(), segmentMetadata.getChunkCount());
        Preconditions.checkState(oldLength + length == segmentMetadata.getLength(), "New length must match. old value (%s) + length (%s) must match current chunk count(%s)", oldLength, length, segmentMetadata.getLength());
        if (null != lastChunkMetadata.get()) {
            Preconditions.checkState(segmentMetadata.getLastChunkStartOffset() + lastChunkMetadata.get().getLength() == segmentMetadata.getLength(), "Last chunk start offset (%s) + Last chunk length (%s) must match segment length (%s)", segmentMetadata.getLastChunkStartOffset(), lastChunkMetadata.get().getLength(), segmentMetadata.getLength());
        }
    }, chunkedSegmentStorage.getExecutor());
}

Example 19 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class WriteOperation method call.

@Override
public CompletableFuture<Void> call() {
    // Validate preconditions.
    checkPreconditions();
    log.debug("{} write - started op={}, segment={}, offset={} length={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), offset, length);
    val streamSegmentName = handle.getSegmentName();
    return ChunkedSegmentStorage.tryWith(chunkedSegmentStorage.getMetadataStore().beginTransaction(false, handle.getSegmentName()), txn -> {
        didSegmentLayoutChange = false;
        // Retrieve metadata.
        return txn.get(streamSegmentName).thenComposeAsync(storageMetadata -> {
            segmentMetadata = (SegmentMetadata) storageMetadata;
            // Validate preconditions.
            checkState();
            isSystemSegment = chunkedSegmentStorage.isStorageSystemSegment(segmentMetadata);
            // Check if this is a first write after ownership changed.
            isFirstWriteAfterFailover = segmentMetadata.isOwnershipChanged();
            lastChunkMetadata.set(null);
            chunkHandle = null;
            bytesRemaining.set(length);
            currentOffset.set(offset);
            return getLastChunk(txn).thenComposeAsync(v -> writeData(txn).thenComposeAsync(vv -> commit(txn).thenApplyAsync(vvvv -> postCommit(), chunkedSegmentStorage.getExecutor()).exceptionally(this::handleException), chunkedSegmentStorage.getExecutor()).thenRunAsync(this::logEnd, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor());
        }, chunkedSegmentStorage.getExecutor());
    }, chunkedSegmentStorage.getExecutor()).exceptionally(ex -> (Void) handleException(ex));
}

Example 20 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class DefragmentOperation method concatChunks.

private CompletableFuture<Void> concatChunks() {
    val concatArgs = new ConcatArgument[chunksToConcat.size()];
    for (int i = 0; i < chunksToConcat.size(); i++) {
        concatArgs[i] = ConcatArgument.fromChunkInfo(chunksToConcat.get(i));
    }
    final CompletableFuture<Integer> f;
    if (!useAppend.get() && chunkedSegmentStorage.getChunkStorage().supportsConcat()) {
        for (int i = 0; i < chunksToConcat.size() - 1; i++) {
            Preconditions.checkState(concatArgs[i].getLength() < chunkedSegmentStorage.getConfig().getMaxSizeLimitForConcat(), "ConcatArgument out of bound. {}", concatArgs[i]);
            Preconditions.checkState(concatArgs[i].getLength() > chunkedSegmentStorage.getConfig().getMinSizeLimitForConcat(), "ConcatArgument out of bound. {}", concatArgs[i]);
        }
        f = chunkedSegmentStorage.getChunkStorage().concat(concatArgs);
    } else {
        if (chunkedSegmentStorage.shouldAppend()) {
            f = concatUsingAppend(concatArgs);
        } else {
            Preconditions.checkState(chunkedSegmentStorage.getChunkStorage().supportsConcat(), "ChunkStorage must support Concat.");
            Preconditions.checkState(concatArgs[0].getLength() > chunkedSegmentStorage.getConfig().getMinSizeLimitForConcat(), "ConcatArgument out of bound. {}", concatArgs[0]);
            f = concatUsingTailConcat(concatArgs);
        }
    }
    return f.thenComposeAsync(v -> {
        // Delete chunks.
        for (int i = 1; i < chunksToConcat.size(); i++) {
            chunksToDelete.add(chunksToConcat.get(i).getName());
        }
        // Set the pointers
        target.setLength(targetSizeAfterConcat.get());
        target.setNextChunk(nextChunkName);
        // If target is the last chunk after this then update metadata accordingly
        if (null == nextChunkName) {
            segmentMetadata.setLastChunk(target.getName());
            segmentMetadata.setLastChunkStartOffset(segmentMetadata.getLength() - target.getLength());
        }
        final List<CompletableFuture<Void>> futures = Collections.synchronizedList(new ArrayList<>());
        // Update metadata for affected chunks.
        for (int i = 1; i < concatArgs.length; i++) {
            final int n = i;
            futures.add(txn.get(concatArgs[n].getName()).thenAcceptAsync(metadata -> {
                ((ChunkMetadata) metadata).setActive(false);
                txn.update(metadata);
            }, chunkedSegmentStorage.getExecutor()));
            segmentMetadata.setChunkCount(segmentMetadata.getChunkCount() - 1);
        }
        return Futures.allOf(futures).thenRunAsync(() -> {
            txn.update(target);
            txn.update(segmentMetadata);
        }, chunkedSegmentStorage.getExecutor());
    }, chunkedSegmentStorage.getExecutor());
}