Examples with ReadIndexBlockMetadata io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata used on opensource projects

Example 1 with ReadIndexBlockMetadata

use of io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata in project pravega by pravega.

the class ReadOperation method findChunkForOffset.

private CompletableFuture<Void> findChunkForOffset(MetadataTransaction txn) {
    currentChunkName = segmentMetadata.getFirstChunk();
    chunkToReadFrom = null;
    Preconditions.checkState(null != currentChunkName, "currentChunkName must not be null. Segment=%s", segmentMetadata.getName());
    bytesRemaining.set(length);
    currentBufferOffset.set(bufferOffset);
    currentOffset.set(offset);
    totalBytesRead.set(0);
    // Find the first chunk that contains the data.
    startOffsetForCurrentChunk.set(segmentMetadata.getFirstChunkStartOffset());
    boolean shouldOnlyReadLastChunk = offset >= segmentMetadata.getLastChunkStartOffset();
    if (shouldOnlyReadLastChunk) {
        startOffsetForCurrentChunk.set(segmentMetadata.getLastChunkStartOffset());
        currentChunkName = segmentMetadata.getLastChunk();
    } else {
        // Find the name of the chunk in the cached read index that is floor to required offset.
        val floorEntry = chunkedSegmentStorage.getReadIndexCache().findFloor(handle.getSegmentName(), offset);
        if (null != floorEntry && startOffsetForCurrentChunk.get() < floorEntry.getOffset() && null != floorEntry.getChunkName()) {
            startOffsetForCurrentChunk.set(floorEntry.getOffset());
            currentChunkName = floorEntry.getChunkName();
        }
    }
    final long floorBlockStartOffset = getFloorBlockStartOffset(offset);
    CompletableFuture<Void> f;
    if (!shouldOnlyReadLastChunk && !segmentMetadata.isStorageSystemSegment() && startOffsetForCurrentChunk.get() < floorBlockStartOffset) {
        val indexLookupTimer = new Timer();
        f = txn.get(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), floorBlockStartOffset)).thenAcceptAsync(storageMetadata -> {
            if (null != storageMetadata) {
                ReadIndexBlockMetadata blockMetadata = (ReadIndexBlockMetadata) storageMetadata;
                if (blockMetadata.getStartOffset() <= offset) {
                    startOffsetForCurrentChunk.set(blockMetadata.getStartOffset());
                    currentChunkName = blockMetadata.getChunkName();
                    log.debug("{} read - found block index to start scanning - op={}, segment={}, chunk={}, startOffset={}, offset={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), currentChunkName, startOffsetForCurrentChunk.get(), offset);
                    // Note: This just is prefetch call. Do not wait.
                    val nextBlock = getFloorBlockStartOffset(offset + length);
                    if (nextBlock > floorBlockStartOffset + chunkedSegmentStorage.getConfig().getIndexBlockSize()) {
                        // We read multiple blocks already
                        txn.get(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), nextBlock));
                    } else {
                        // Prefetch next block index entry.
                        txn.get(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), floorBlockStartOffset + chunkedSegmentStorage.getConfig().getIndexBlockSize()));
                    }
                } else {
                    log.warn("{} read - block entry offset must be floor to requested offset. op={} segment={} offset={} length={} block={}", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), segmentMetadata, offset, length, blockMetadata);
                }
            }
            if (segmentMetadata.isStorageSystemSegment()) {
                SLTS_SYS_READ_INDEX_BLOCK_LOOKUP_LATENCY.reportSuccessEvent(indexLookupTimer.getElapsed());
            } else {
                SLTS_READ_INDEX_BLOCK_LOOKUP_LATENCY.reportSuccessEvent(indexLookupTimer.getElapsed());
            }
        }, chunkedSegmentStorage.getExecutor());
    } else {
        f = CompletableFuture.completedFuture(null);
    }
    val readIndexTimer = new Timer();
    // Navigate to the chunk that contains the first byte of requested data.
    return f.thenComposeAsync(vv -> Futures.loop(() -> currentChunkName != null && !isLoopExited, () -> txn.get(currentChunkName).thenAcceptAsync(storageMetadata -> {
        chunkToReadFrom = (ChunkMetadata) storageMetadata;
        Preconditions.checkState(null != chunkToReadFrom, "chunkToReadFrom is null. currentChunkName=%s Segment=%s", currentChunkName, segmentMetadata.getName());
        if (startOffsetForCurrentChunk.get() <= currentOffset.get() && startOffsetForCurrentChunk.get() + chunkToReadFrom.getLength() > currentOffset.get()) {
            // we have found a chunk that contains first byte we want to read
            log.debug("{} read - found chunk to read - op={}, segment={}, chunk={}, startOffset={}, length={}, readOffset={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), chunkToReadFrom, startOffsetForCurrentChunk.get(), chunkToReadFrom.getLength(), currentOffset);
            isLoopExited = true;
            return;
        }
        currentChunkName = chunkToReadFrom.getNextChunk();
        startOffsetForCurrentChunk.addAndGet(chunkToReadFrom.getLength());
        // Update read index with newly visited chunk.
        if (null != currentChunkName) {
            chunkedSegmentStorage.getReadIndexCache().addIndexEntry(handle.getSegmentName(), currentChunkName, startOffsetForCurrentChunk.get());
        }
        cntScanned.incrementAndGet();
    }, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor()).thenAcceptAsync(v -> {
        val elapsed = readIndexTimer.getElapsed();
        if (segmentMetadata.isStorageSystemSegment()) {
            SLTS_SYS_READ_INDEX_SCAN_LATENCY.reportSuccessEvent(elapsed);
            SLTS_SYS_READ_INDEX_NUM_SCANNED.reportSuccessValue(cntScanned.get());
        } else {
            SLTS_READ_INDEX_SCAN_LATENCY.reportSuccessEvent(elapsed);
            SLTS_READ_INDEX_NUM_SCANNED.reportSuccessValue(cntScanned.get());
        }
        // Prefetch possible chunks for next read.
        if (chunkToReadFrom.getNextChunk() != null) {
            // Do not wait.
            txn.get(chunkToReadFrom.getNextChunk());
        }
        log.debug("{} read - chunk lookup - op={}, segment={}, offset={}, scanned={}, latency={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), offset, cntScanned.get(), elapsed.toMillis());
    }, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor());
}

Example 2 with ReadIndexBlockMetadata

use of io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata in project pravega by pravega.

the class TestUtils method checkReadIndexEntries.

/**
 * Checks the existence of read index block metadata records for given segment.
 * @param chunkedSegmentStorage Instance of {@link ChunkedSegmentStorage}.
 * @param metadataStore  Metadata store to query.
 * @param segmentName    Name of the segment.
 * @param startOffset    Start offset of the segment.
 * @param endOffset      End offset of the segment.
 * @param checkReadIndex True if readIndex entries should be checked.
 * @throws Exception Exceptions are thrown in case of any errors.
 */
public static void checkReadIndexEntries(ChunkedSegmentStorage chunkedSegmentStorage, ChunkMetadataStore metadataStore, String segmentName, long startOffset, long endOffset, boolean checkReadIndex) throws Exception {
    val blockSize = chunkedSegmentStorage.getConfig().getIndexBlockSize();
    val segmentReadIndex = chunkedSegmentStorage.getReadIndexCache().getSegmentsReadIndexCache().getIfPresent(segmentName);
    try (val txn = metadataStore.beginTransaction(true, new String[] { segmentName })) {
        val segmentMetadata = (SegmentMetadata) txn.get(segmentName).get();
        Assert.assertNotNull(segmentMetadata);
        TreeMap<Long, String> index = new TreeMap<>();
        String current = segmentMetadata.getFirstChunk();
        long offset = segmentMetadata.getFirstChunkStartOffset();
        while (null != current) {
            val chunk = (ChunkMetadata) txn.get(current).get();
            Assert.assertNotNull(chunk);
            if (checkReadIndex && startOffset <= offset) {
                Assert.assertNotNull("Offset=" + offset, segmentReadIndex.getOffsetToChunkNameIndex().get(offset));
                Assert.assertEquals("Offset=" + offset, chunk.getName(), segmentReadIndex.getOffsetToChunkNameIndex().get(offset).getChunkName());
            }
            index.put(offset, chunk.getName());
            offset += chunk.getLength();
            current = chunk.getNextChunk();
        }
        if (checkReadIndex) {
            for (val entry : segmentReadIndex.getOffsetToChunkNameIndex().entrySet()) {
                Assert.assertNotNull("Offset=" + entry.getKey(), index.get(entry.getKey()));
                Assert.assertEquals("Offset=" + entry.getKey(), entry.getValue().getChunkName(), index.get(entry.getKey()));
            }
        }
        long blockStartOffset;
        for (blockStartOffset = 0; blockStartOffset < segmentMetadata.getLength(); blockStartOffset += blockSize) {
            // For all offsets below start offset, there should not be any index entries.
            if (segmentMetadata.getStartOffset() > blockStartOffset) {
                Assert.assertNull("for offset:" + blockStartOffset, txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, blockStartOffset)).get());
            }
            // For all valid offsets, there should be index entries.
            if (segmentMetadata.getStartOffset() <= blockStartOffset) {
                val blockIndexEntry = (ReadIndexBlockMetadata) txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, blockStartOffset)).get();
                Assert.assertNotNull("for offset:" + blockStartOffset, blockIndexEntry);
                Assert.assertNotNull("for offset:" + blockStartOffset, txn.get(blockIndexEntry.getChunkName()));
                val mappedChunk = index.floorEntry(blockStartOffset);
                Assert.assertNotNull(mappedChunk);
                Assert.assertEquals("for offset:" + blockStartOffset, mappedChunk.getValue(), blockIndexEntry.getChunkName());
            }
        }
        // For all offsets after end of the segment, there should not be any index entries
        Assert.assertNull("for offset:" + segmentMetadata.getLength(), txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, segmentMetadata.getLength())).get());
        Assert.assertNull("for offset:" + segmentMetadata.getLength() + blockSize, txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, segmentMetadata.getLength() + blockSize)).get());
    }
}

Example 3 with ReadIndexBlockMetadata

use of io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata in project pravega by pravega.

the class SltsMetadataSerializer method handleStorageMetadataValue.

/**
 * Convert {@link StorageMetadata} into string of fields and values to be appended it into the given StringBuilder.
 *
 * @param builder  The given StringBuilder.
 * @param metadata The StorageMetadata instance.
 */
private void handleStorageMetadataValue(StringBuilder builder, StorageMetadata metadata) {
    if (metadata instanceof ChunkMetadata) {
        appendField(builder, METADATA_TYPE, CHUNK_METADATA);
        ChunkMetadata chunkMetadata = (ChunkMetadata) metadata;
        CHUNK_METADATA_FIELD_MAP.forEach((name, f) -> appendField(builder, name, String.valueOf(f.apply(chunkMetadata))));
    } else if (metadata instanceof SegmentMetadata) {
        appendField(builder, METADATA_TYPE, SEGMENT_METADATA);
        SegmentMetadata segmentMetadata = (SegmentMetadata) metadata;
        SEGMENT_METADATA_FIELD_MAP.forEach((name, f) -> appendField(builder, name, String.valueOf(f.apply(segmentMetadata))));
    } else if (metadata instanceof ReadIndexBlockMetadata) {
        appendField(builder, METADATA_TYPE, READ_INDEX_BLOCK_METADATA);
        ReadIndexBlockMetadata readIndexBlockMetadata = (ReadIndexBlockMetadata) metadata;
        READ_INDEX_BLOCK_METADATA_FIELD_MAP.forEach((name, f) -> appendField(builder, name, String.valueOf(f.apply(readIndexBlockMetadata))));
    }
}