Examples with ChunkMetadata io.pravega.segmentstore.storage.metadata.ChunkMetadata used on opensource projects

Example 11 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class TestUtils method insertMetadata.

/**
 * Insert Metadata as given.
 *
 * @param testSegmentName Name of the segment
 * @param maxRollingLength Max rolling length.
 * @param ownerEpoch Owner epoch.
 * @param chunkLengthsInMetadata Chunk lengths to set in metadata.
 * @param chunkLengthsInStorage Chunk lengths to set in storage.
 * @param addIndex Whether to add index.
 * @param addIndexMetadata Whether to add index metadata.
 * @param metadataStore Instance of {@link ChunkMetadataStore}
 * @param chunkedSegmentStorage Instance of {@link ChunkedSegmentStorage}.
 * @return {@link SegmentMetadata} representing segment.
 */
public static SegmentMetadata insertMetadata(String testSegmentName, long maxRollingLength, int ownerEpoch, long[] chunkLengthsInMetadata, long[] chunkLengthsInStorage, boolean addIndex, boolean addIndexMetadata, ChunkMetadataStore metadataStore, ChunkedSegmentStorage chunkedSegmentStorage) {
    Preconditions.checkArgument(maxRollingLength > 0, "maxRollingLength");
    Preconditions.checkArgument(ownerEpoch > 0, "ownerEpoch");
    try (val txn = metadataStore.beginTransaction(false, new String[] { testSegmentName })) {
        String firstChunk = null;
        String lastChunk = null;
        TreeMap<Long, String> index = new TreeMap<>();
        // Add chunks.
        long length = 0;
        long startOfLast = 0;
        long startOffset = 0;
        int chunkCount = 0;
        for (int i = 0; i < chunkLengthsInMetadata.length; i++) {
            String chunkName = testSegmentName + "_chunk_" + Integer.toString(i);
            ChunkMetadata chunkMetadata = ChunkMetadata.builder().name(chunkName).length(chunkLengthsInMetadata[i]).nextChunk(i == chunkLengthsInMetadata.length - 1 ? null : testSegmentName + "_chunk_" + Integer.toString(i + 1)).build();
            chunkMetadata.setActive(true);
            if (addIndex) {
                chunkedSegmentStorage.getReadIndexCache().addIndexEntry(testSegmentName, chunkName, startOffset);
            }
            index.put(startOffset, chunkName);
            startOffset += chunkLengthsInMetadata[i];
            length += chunkLengthsInMetadata[i];
            txn.create(chunkMetadata);
            addChunk(chunkedSegmentStorage.getChunkStorage(), chunkName, chunkLengthsInStorage[i]);
            chunkCount++;
        }
        // Fix the first and last
        if (chunkLengthsInMetadata.length > 0) {
            firstChunk = testSegmentName + "_chunk_0";
            lastChunk = testSegmentName + "_chunk_" + Integer.toString(chunkLengthsInMetadata.length - 1);
            startOfLast = length - chunkLengthsInMetadata[chunkLengthsInMetadata.length - 1];
        }
        // Finally save
        SegmentMetadata segmentMetadata = SegmentMetadata.builder().maxRollinglength(maxRollingLength).name(testSegmentName).ownerEpoch(ownerEpoch).firstChunk(firstChunk).lastChunk(lastChunk).length(length).lastChunkStartOffset(startOfLast).build();
        segmentMetadata.setActive(true);
        segmentMetadata.setChunkCount(chunkCount);
        segmentMetadata.checkInvariants();
        txn.create(segmentMetadata);
        if (addIndexMetadata) {
            for (long blockStartOffset = 0; blockStartOffset < segmentMetadata.getLength(); blockStartOffset += chunkedSegmentStorage.getConfig().getIndexBlockSize()) {
                val floor = index.floorEntry(blockStartOffset);
                txn.create(ReadIndexBlockMetadata.builder().name(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), blockStartOffset)).startOffset(floor.getKey()).chunkName(floor.getValue()).status(StatusFlags.ACTIVE).build());
            }
        }
        txn.commit().join();
        return segmentMetadata;
    }
}

Example 12 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class ReadOperation method findChunkForOffset.

private CompletableFuture<Void> findChunkForOffset(MetadataTransaction txn) {
    currentChunkName = segmentMetadata.getFirstChunk();
    chunkToReadFrom = null;
    Preconditions.checkState(null != currentChunkName, "currentChunkName must not be null. Segment=%s", segmentMetadata.getName());
    bytesRemaining.set(length);
    currentBufferOffset.set(bufferOffset);
    currentOffset.set(offset);
    totalBytesRead.set(0);
    // Find the first chunk that contains the data.
    startOffsetForCurrentChunk.set(segmentMetadata.getFirstChunkStartOffset());
    boolean shouldOnlyReadLastChunk = offset >= segmentMetadata.getLastChunkStartOffset();
    if (shouldOnlyReadLastChunk) {
        startOffsetForCurrentChunk.set(segmentMetadata.getLastChunkStartOffset());
        currentChunkName = segmentMetadata.getLastChunk();
    } else {
        // Find the name of the chunk in the cached read index that is floor to required offset.
        val floorEntry = chunkedSegmentStorage.getReadIndexCache().findFloor(handle.getSegmentName(), offset);
        if (null != floorEntry && startOffsetForCurrentChunk.get() < floorEntry.getOffset() && null != floorEntry.getChunkName()) {
            startOffsetForCurrentChunk.set(floorEntry.getOffset());
            currentChunkName = floorEntry.getChunkName();
        }
    }
    final long floorBlockStartOffset = getFloorBlockStartOffset(offset);
    CompletableFuture<Void> f;
    if (!shouldOnlyReadLastChunk && !segmentMetadata.isStorageSystemSegment() && startOffsetForCurrentChunk.get() < floorBlockStartOffset) {
        val indexLookupTimer = new Timer();
        f = txn.get(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), floorBlockStartOffset)).thenAcceptAsync(storageMetadata -> {
            if (null != storageMetadata) {
                ReadIndexBlockMetadata blockMetadata = (ReadIndexBlockMetadata) storageMetadata;
                if (blockMetadata.getStartOffset() <= offset) {
                    startOffsetForCurrentChunk.set(blockMetadata.getStartOffset());
                    currentChunkName = blockMetadata.getChunkName();
                    log.debug("{} read - found block index to start scanning - op={}, segment={}, chunk={}, startOffset={}, offset={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), currentChunkName, startOffsetForCurrentChunk.get(), offset);
                    // Note: This just is prefetch call. Do not wait.
                    val nextBlock = getFloorBlockStartOffset(offset + length);
                    if (nextBlock > floorBlockStartOffset + chunkedSegmentStorage.getConfig().getIndexBlockSize()) {
                        // We read multiple blocks already
                        txn.get(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), nextBlock));
                    } else {
                        // Prefetch next block index entry.
                        txn.get(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), floorBlockStartOffset + chunkedSegmentStorage.getConfig().getIndexBlockSize()));
                    }
                } else {
                    log.warn("{} read - block entry offset must be floor to requested offset. op={} segment={} offset={} length={} block={}", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), segmentMetadata, offset, length, blockMetadata);
                }
            }
            if (segmentMetadata.isStorageSystemSegment()) {
                SLTS_SYS_READ_INDEX_BLOCK_LOOKUP_LATENCY.reportSuccessEvent(indexLookupTimer.getElapsed());
            } else {
                SLTS_READ_INDEX_BLOCK_LOOKUP_LATENCY.reportSuccessEvent(indexLookupTimer.getElapsed());
            }
        }, chunkedSegmentStorage.getExecutor());
    } else {
        f = CompletableFuture.completedFuture(null);
    }
    val readIndexTimer = new Timer();
    // Navigate to the chunk that contains the first byte of requested data.
    return f.thenComposeAsync(vv -> Futures.loop(() -> currentChunkName != null && !isLoopExited, () -> txn.get(currentChunkName).thenAcceptAsync(storageMetadata -> {
        chunkToReadFrom = (ChunkMetadata) storageMetadata;
        Preconditions.checkState(null != chunkToReadFrom, "chunkToReadFrom is null. currentChunkName=%s Segment=%s", currentChunkName, segmentMetadata.getName());
        if (startOffsetForCurrentChunk.get() <= currentOffset.get() && startOffsetForCurrentChunk.get() + chunkToReadFrom.getLength() > currentOffset.get()) {
            // we have found a chunk that contains first byte we want to read
            log.debug("{} read - found chunk to read - op={}, segment={}, chunk={}, startOffset={}, length={}, readOffset={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), chunkToReadFrom, startOffsetForCurrentChunk.get(), chunkToReadFrom.getLength(), currentOffset);
            isLoopExited = true;
            return;
        }
        currentChunkName = chunkToReadFrom.getNextChunk();
        startOffsetForCurrentChunk.addAndGet(chunkToReadFrom.getLength());
        // Update read index with newly visited chunk.
        if (null != currentChunkName) {
            chunkedSegmentStorage.getReadIndexCache().addIndexEntry(handle.getSegmentName(), currentChunkName, startOffsetForCurrentChunk.get());
        }
        cntScanned.incrementAndGet();
    }, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor()).thenAcceptAsync(v -> {
        val elapsed = readIndexTimer.getElapsed();
        if (segmentMetadata.isStorageSystemSegment()) {
            SLTS_SYS_READ_INDEX_SCAN_LATENCY.reportSuccessEvent(elapsed);
            SLTS_SYS_READ_INDEX_NUM_SCANNED.reportSuccessValue(cntScanned.get());
        } else {
            SLTS_READ_INDEX_SCAN_LATENCY.reportSuccessEvent(elapsed);
            SLTS_READ_INDEX_NUM_SCANNED.reportSuccessValue(cntScanned.get());
        }
        // Prefetch possible chunks for next read.
        if (chunkToReadFrom.getNextChunk() != null) {
            // Do not wait.
            txn.get(chunkToReadFrom.getNextChunk());
        }
        log.debug("{} read - chunk lookup - op={}, segment={}, offset={}, scanned={}, latency={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), offset, cntScanned.get(), elapsed.toMillis());
    }, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor());
}

Example 13 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class WriteOperation method writeToChunk.

/**
 * Write to chunk.
 */
private CompletableFuture<Void> writeToChunk(MetadataTransaction txn, SegmentMetadata segmentMetadata, InputStream data, ChunkHandle chunkHandle, ChunkMetadata chunkWrittenMetadata, long offsetToWriteAt, int bytesCount) {
    Preconditions.checkState(0 != bytesCount, "Attempt to write zero bytes. Segment=%s Chunk=%s offsetToWriteAt=%s", segmentMetadata, chunkWrittenMetadata, offsetToWriteAt);
    // Finally write the data.
    val bis = new BoundedInputStream(data, bytesCount);
    CompletableFuture<Integer> retValue;
    if (chunkedSegmentStorage.shouldAppend()) {
        retValue = chunkedSegmentStorage.getChunkStorage().write(chunkHandle, offsetToWriteAt, bytesCount, bis);
    } else {
        retValue = chunkedSegmentStorage.getChunkStorage().createWithContent(chunkHandle.getChunkName(), bytesCount, bis).thenApplyAsync(h -> bytesCount, chunkedSegmentStorage.getExecutor());
    }
    return retValue.thenAcceptAsync(bytesWritten -> {
        // Update the metadata for segment and chunk.
        Preconditions.checkState(bytesWritten >= 0, "bytesWritten (%s) must be non-negative. Segment=%s Chunk=%s offsetToWriteAt=%s", bytesWritten, segmentMetadata, chunkWrittenMetadata, offsetToWriteAt);
        segmentMetadata.setLength(segmentMetadata.getLength() + bytesWritten);
        chunkWrittenMetadata.setLength(chunkWrittenMetadata.getLength() + bytesWritten);
        txn.update(chunkWrittenMetadata);
        txn.update(segmentMetadata);
        bytesRemaining.addAndGet(-bytesWritten);
        currentOffset.addAndGet(bytesWritten);
    }, chunkedSegmentStorage.getExecutor()).handleAsync((v, e) -> {
        if (null != e) {
            val ex = Exceptions.unwrap(e);
            if (ex instanceof InvalidOffsetException) {
                val invalidEx = (InvalidOffsetException) ex;
                // This could happen if the previous write failed while writing data and chunk was partially written.
                if (invalidEx.getExpectedOffset() > offsetToWriteAt) {
                    skipOverFailedChunk = true;
                    log.debug("{} write - skipping partially written chunk op={}, segment={}, chunk={} expected={} given={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), chunkHandle.getChunkName(), invalidEx.getExpectedOffset(), invalidEx.getGivenOffset());
                    return null;
                }
                throw new CompletionException(new BadOffsetException(segmentMetadata.getName(), currentOffset.get() + ((InvalidOffsetException) ex).getExpectedOffset(), currentOffset.get() + ((InvalidOffsetException) ex).getGivenOffset()));
            }
            throw new CompletionException(ex);
        }
        return v;
    }, chunkedSegmentStorage.getExecutor());
}

Example 14 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class DefragmentOperation method concatChunks.

private CompletableFuture<Void> concatChunks() {
    val concatArgs = new ConcatArgument[chunksToConcat.size()];
    for (int i = 0; i < chunksToConcat.size(); i++) {
        concatArgs[i] = ConcatArgument.fromChunkInfo(chunksToConcat.get(i));
    }
    final CompletableFuture<Integer> f;
    if (!useAppend.get() && chunkedSegmentStorage.getChunkStorage().supportsConcat()) {
        for (int i = 0; i < chunksToConcat.size() - 1; i++) {
            Preconditions.checkState(concatArgs[i].getLength() < chunkedSegmentStorage.getConfig().getMaxSizeLimitForConcat(), "ConcatArgument out of bound. {}", concatArgs[i]);
            Preconditions.checkState(concatArgs[i].getLength() > chunkedSegmentStorage.getConfig().getMinSizeLimitForConcat(), "ConcatArgument out of bound. {}", concatArgs[i]);
        }
        f = chunkedSegmentStorage.getChunkStorage().concat(concatArgs);
    } else {
        if (chunkedSegmentStorage.shouldAppend()) {
            f = concatUsingAppend(concatArgs);
        } else {
            Preconditions.checkState(chunkedSegmentStorage.getChunkStorage().supportsConcat(), "ChunkStorage must support Concat.");
            Preconditions.checkState(concatArgs[0].getLength() > chunkedSegmentStorage.getConfig().getMinSizeLimitForConcat(), "ConcatArgument out of bound. {}", concatArgs[0]);
            f = concatUsingTailConcat(concatArgs);
        }
    }
    return f.thenComposeAsync(v -> {
        // Delete chunks.
        for (int i = 1; i < chunksToConcat.size(); i++) {
            chunksToDelete.add(chunksToConcat.get(i).getName());
        }
        // Set the pointers
        target.setLength(targetSizeAfterConcat.get());
        target.setNextChunk(nextChunkName);
        // If target is the last chunk after this then update metadata accordingly
        if (null == nextChunkName) {
            segmentMetadata.setLastChunk(target.getName());
            segmentMetadata.setLastChunkStartOffset(segmentMetadata.getLength() - target.getLength());
        }
        final List<CompletableFuture<Void>> futures = Collections.synchronizedList(new ArrayList<>());
        // Update metadata for affected chunks.
        for (int i = 1; i < concatArgs.length; i++) {
            final int n = i;
            futures.add(txn.get(concatArgs[n].getName()).thenAcceptAsync(metadata -> {
                ((ChunkMetadata) metadata).setActive(false);
                txn.update(metadata);
            }, chunkedSegmentStorage.getExecutor()));
            segmentMetadata.setChunkCount(segmentMetadata.getChunkCount() - 1);
        }
        return Futures.allOf(futures).thenRunAsync(() -> {
            txn.update(target);
            txn.update(segmentMetadata);
        }, chunkedSegmentStorage.getExecutor());
    }, chunkedSegmentStorage.getExecutor());
}

Example 15 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class TestUtils method checkReadIndexEntries.

/**
 * Checks the existence of read index block metadata records for given segment.
 * @param chunkedSegmentStorage Instance of {@link ChunkedSegmentStorage}.
 * @param metadataStore  Metadata store to query.
 * @param segmentName    Name of the segment.
 * @param startOffset    Start offset of the segment.
 * @param endOffset      End offset of the segment.
 * @param checkReadIndex True if readIndex entries should be checked.
 * @throws Exception Exceptions are thrown in case of any errors.
 */
public static void checkReadIndexEntries(ChunkedSegmentStorage chunkedSegmentStorage, ChunkMetadataStore metadataStore, String segmentName, long startOffset, long endOffset, boolean checkReadIndex) throws Exception {
    val blockSize = chunkedSegmentStorage.getConfig().getIndexBlockSize();
    val segmentReadIndex = chunkedSegmentStorage.getReadIndexCache().getSegmentsReadIndexCache().getIfPresent(segmentName);
    try (val txn = metadataStore.beginTransaction(true, new String[] { segmentName })) {
        val segmentMetadata = (SegmentMetadata) txn.get(segmentName).get();
        Assert.assertNotNull(segmentMetadata);
        TreeMap<Long, String> index = new TreeMap<>();
        String current = segmentMetadata.getFirstChunk();
        long offset = segmentMetadata.getFirstChunkStartOffset();
        while (null != current) {
            val chunk = (ChunkMetadata) txn.get(current).get();
            Assert.assertNotNull(chunk);
            if (checkReadIndex && startOffset <= offset) {
                Assert.assertNotNull("Offset=" + offset, segmentReadIndex.getOffsetToChunkNameIndex().get(offset));
                Assert.assertEquals("Offset=" + offset, chunk.getName(), segmentReadIndex.getOffsetToChunkNameIndex().get(offset).getChunkName());
            }
            index.put(offset, chunk.getName());
            offset += chunk.getLength();
            current = chunk.getNextChunk();
        }
        if (checkReadIndex) {
            for (val entry : segmentReadIndex.getOffsetToChunkNameIndex().entrySet()) {
                Assert.assertNotNull("Offset=" + entry.getKey(), index.get(entry.getKey()));
                Assert.assertEquals("Offset=" + entry.getKey(), entry.getValue().getChunkName(), index.get(entry.getKey()));
            }
        }
        long blockStartOffset;
        for (blockStartOffset = 0; blockStartOffset < segmentMetadata.getLength(); blockStartOffset += blockSize) {
            // For all offsets below start offset, there should not be any index entries.
            if (segmentMetadata.getStartOffset() > blockStartOffset) {
                Assert.assertNull("for offset:" + blockStartOffset, txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, blockStartOffset)).get());
            }
            // For all valid offsets, there should be index entries.
            if (segmentMetadata.getStartOffset() <= blockStartOffset) {
                val blockIndexEntry = (ReadIndexBlockMetadata) txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, blockStartOffset)).get();
                Assert.assertNotNull("for offset:" + blockStartOffset, blockIndexEntry);
                Assert.assertNotNull("for offset:" + blockStartOffset, txn.get(blockIndexEntry.getChunkName()));
                val mappedChunk = index.floorEntry(blockStartOffset);
                Assert.assertNotNull(mappedChunk);
                Assert.assertEquals("for offset:" + blockStartOffset, mappedChunk.getValue(), blockIndexEntry.getChunkName());
            }
        }
        // For all offsets after end of the segment, there should not be any index entries
        Assert.assertNull("for offset:" + segmentMetadata.getLength(), txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, segmentMetadata.getLength())).get());
        Assert.assertNull("for offset:" + segmentMetadata.getLength() + blockSize, txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, segmentMetadata.getLength() + blockSize)).get());
    }
}