Examples with SegmentMetadata io.pravega.segmentstore.storage.metadata.SegmentMetadata used on opensource projects

Example 21 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class TestUtils method checkReadIndexEntries.

/**
 * Checks the existence of read index block metadata records for given segment.
 * @param chunkedSegmentStorage Instance of {@link ChunkedSegmentStorage}.
 * @param metadataStore  Metadata store to query.
 * @param segmentName    Name of the segment.
 * @param startOffset    Start offset of the segment.
 * @param endOffset      End offset of the segment.
 * @param checkReadIndex True if readIndex entries should be checked.
 * @throws Exception Exceptions are thrown in case of any errors.
 */
public static void checkReadIndexEntries(ChunkedSegmentStorage chunkedSegmentStorage, ChunkMetadataStore metadataStore, String segmentName, long startOffset, long endOffset, boolean checkReadIndex) throws Exception {
    val blockSize = chunkedSegmentStorage.getConfig().getIndexBlockSize();
    val segmentReadIndex = chunkedSegmentStorage.getReadIndexCache().getSegmentsReadIndexCache().getIfPresent(segmentName);
    try (val txn = metadataStore.beginTransaction(true, new String[] { segmentName })) {
        val segmentMetadata = (SegmentMetadata) txn.get(segmentName).get();
        Assert.assertNotNull(segmentMetadata);
        TreeMap<Long, String> index = new TreeMap<>();
        String current = segmentMetadata.getFirstChunk();
        long offset = segmentMetadata.getFirstChunkStartOffset();
        while (null != current) {
            val chunk = (ChunkMetadata) txn.get(current).get();
            Assert.assertNotNull(chunk);
            if (checkReadIndex && startOffset <= offset) {
                Assert.assertNotNull("Offset=" + offset, segmentReadIndex.getOffsetToChunkNameIndex().get(offset));
                Assert.assertEquals("Offset=" + offset, chunk.getName(), segmentReadIndex.getOffsetToChunkNameIndex().get(offset).getChunkName());
            }
            index.put(offset, chunk.getName());
            offset += chunk.getLength();
            current = chunk.getNextChunk();
        }
        if (checkReadIndex) {
            for (val entry : segmentReadIndex.getOffsetToChunkNameIndex().entrySet()) {
                Assert.assertNotNull("Offset=" + entry.getKey(), index.get(entry.getKey()));
                Assert.assertEquals("Offset=" + entry.getKey(), entry.getValue().getChunkName(), index.get(entry.getKey()));
            }
        }
        long blockStartOffset;
        for (blockStartOffset = 0; blockStartOffset < segmentMetadata.getLength(); blockStartOffset += blockSize) {
            // For all offsets below start offset, there should not be any index entries.
            if (segmentMetadata.getStartOffset() > blockStartOffset) {
                Assert.assertNull("for offset:" + blockStartOffset, txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, blockStartOffset)).get());
            }
            // For all valid offsets, there should be index entries.
            if (segmentMetadata.getStartOffset() <= blockStartOffset) {
                val blockIndexEntry = (ReadIndexBlockMetadata) txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, blockStartOffset)).get();
                Assert.assertNotNull("for offset:" + blockStartOffset, blockIndexEntry);
                Assert.assertNotNull("for offset:" + blockStartOffset, txn.get(blockIndexEntry.getChunkName()));
                val mappedChunk = index.floorEntry(blockStartOffset);
                Assert.assertNotNull(mappedChunk);
                Assert.assertEquals("for offset:" + blockStartOffset, mappedChunk.getValue(), blockIndexEntry.getChunkName());
            }
        }
        // For all offsets after end of the segment, there should not be any index entries
        Assert.assertNull("for offset:" + segmentMetadata.getLength(), txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, segmentMetadata.getLength())).get());
        Assert.assertNull("for offset:" + segmentMetadata.getLength() + blockSize, txn.get(NameUtils.getSegmentReadIndexBlockName(segmentName, segmentMetadata.getLength() + blockSize)).get());
    }
}

Example 22 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class TestUtils method insertMetadata.

/**
 * Insert Metadata as given.
 *
 * @param testSegmentName Name of the segment
 * @param maxRollingLength Max rolling length.
 * @param ownerEpoch Owner epoch.
 * @param metadataStore Instance of {@link ChunkMetadataStore}
 * @return {@link SegmentMetadata} representing segment.
 */
public static SegmentMetadata insertMetadata(String testSegmentName, int maxRollingLength, int ownerEpoch, ChunkMetadataStore metadataStore) {
    Preconditions.checkArgument(maxRollingLength > 0, "maxRollingLength");
    Preconditions.checkArgument(ownerEpoch > 0, "ownerEpoch");
    try (val txn = metadataStore.beginTransaction(false, new String[] { testSegmentName })) {
        SegmentMetadata segmentMetadata = SegmentMetadata.builder().maxRollinglength(maxRollingLength).name(testSegmentName).ownerEpoch(ownerEpoch).build();
        segmentMetadata.setActive(true);
        txn.create(segmentMetadata);
        txn.commit().join();
        return segmentMetadata;
    }
}

Example 23 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class SltsMetadataSerializer method handleStorageMetadataValue.

/**
 * Convert {@link StorageMetadata} into string of fields and values to be appended it into the given StringBuilder.
 *
 * @param builder  The given StringBuilder.
 * @param metadata The StorageMetadata instance.
 */
private void handleStorageMetadataValue(StringBuilder builder, StorageMetadata metadata) {
    if (metadata instanceof ChunkMetadata) {
        appendField(builder, METADATA_TYPE, CHUNK_METADATA);
        ChunkMetadata chunkMetadata = (ChunkMetadata) metadata;
        CHUNK_METADATA_FIELD_MAP.forEach((name, f) -> appendField(builder, name, String.valueOf(f.apply(chunkMetadata))));
    } else if (metadata instanceof SegmentMetadata) {
        appendField(builder, METADATA_TYPE, SEGMENT_METADATA);
        SegmentMetadata segmentMetadata = (SegmentMetadata) metadata;
        SEGMENT_METADATA_FIELD_MAP.forEach((name, f) -> appendField(builder, name, String.valueOf(f.apply(segmentMetadata))));
    } else if (metadata instanceof ReadIndexBlockMetadata) {
        appendField(builder, METADATA_TYPE, READ_INDEX_BLOCK_METADATA);
        ReadIndexBlockMetadata readIndexBlockMetadata = (ReadIndexBlockMetadata) metadata;
        READ_INDEX_BLOCK_METADATA_FIELD_MAP.forEach((name, f) -> appendField(builder, name, String.valueOf(f.apply(readIndexBlockMetadata))));
    }
}

Example 24 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class GarbageCollectorTests method insertSegment.

public SegmentMetadata insertSegment(ChunkMetadataStore metadataStore, ChunkStorage chunkStorage, ChunkedSegmentStorageConfig config, String testSegmentName, long maxRollingLength, int ownerEpoch, long[] chunkLengths, boolean addIndexMetadata, int status) throws Exception {
    Preconditions.checkArgument(maxRollingLength > 0, "maxRollingLength");
    Preconditions.checkArgument(ownerEpoch > 0, "ownerEpoch");
    try (val txn = metadataStore.beginTransaction(false, new String[] { testSegmentName })) {
        String firstChunk = null;
        String lastChunk = null;
        TreeMap<Long, String> index = new TreeMap<>();
        // Add chunks.
        long length = 0;
        long startOfLast = 0;
        long startOffset = 0;
        int chunkCount = 0;
        for (int i = 0; i < chunkLengths.length; i++) {
            String chunkName = testSegmentName + "_chunk_" + Integer.toString(i);
            ChunkMetadata chunkMetadata = ChunkMetadata.builder().name(chunkName).length(chunkLengths[i]).nextChunk(i == chunkLengths.length - 1 ? null : testSegmentName + "_chunk_" + Integer.toString(i + 1)).build();
            chunkMetadata.setActive(true);
            index.put(startOffset, chunkName);
            startOffset += chunkLengths[i];
            length += chunkLengths[i];
            txn.create(chunkMetadata);
            insertChunk(chunkStorage, chunkName, Math.toIntExact(chunkLengths[i]));
            chunkCount++;
        }
        // Fix the first and last
        if (chunkLengths.length > 0) {
            firstChunk = testSegmentName + "_chunk_0";
            lastChunk = testSegmentName + "_chunk_" + Integer.toString(chunkLengths.length - 1);
            startOfLast = length - chunkLengths[chunkLengths.length - 1];
        }
        // Finally save
        SegmentMetadata segmentMetadata = SegmentMetadata.builder().maxRollinglength(maxRollingLength).name(testSegmentName).ownerEpoch(ownerEpoch).firstChunk(firstChunk).lastChunk(lastChunk).length(length).lastChunkStartOffset(startOfLast).build();
        segmentMetadata.setStatus(status);
        segmentMetadata.setChunkCount(chunkCount);
        segmentMetadata.checkInvariants();
        txn.create(segmentMetadata);
        if (addIndexMetadata) {
            for (long blockStartOffset = 0; blockStartOffset < segmentMetadata.getLength(); blockStartOffset += config.getIndexBlockSize()) {
                val floor = index.floorEntry(blockStartOffset);
                txn.create(ReadIndexBlockMetadata.builder().name(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), blockStartOffset)).startOffset(floor.getKey()).chunkName(floor.getValue()).status(StatusFlags.ACTIVE).build());
            }
        }
        txn.commit().join();
        return segmentMetadata;
    }
}

Example 25 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class UtilsWrapper method getExtendedChunkInfoList.

/**
 * Returns the list of {@link ExtendedChunkInfo} which contain data about all chunks for the segment.
 *
 * @param streamSegmentName Name of the segment.
 * @param checkStorage Whether to retrieve information from underlying {@link ChunkStorage}.
 * @return A CompletableFuture that, when completed, will contain a list of {@link ExtendedChunkInfo} objects associated with the segment.
 *  If the operation failed, it will be completed with the appropriate exception.
 */
public CompletableFuture<List<ExtendedChunkInfo>> getExtendedChunkInfoList(String streamSegmentName, boolean checkStorage) {
    Preconditions.checkNotNull(streamSegmentName, "streamSegmentName");
    val infoList = Collections.synchronizedList(new ArrayList<ExtendedChunkInfo>());
    return chunkedSegmentStorage.executeSerialized(() -> chunkedSegmentStorage.tryWith(chunkedSegmentStorage.getMetadataStore().beginTransaction(true, streamSegmentName), txn -> txn.get(streamSegmentName).thenComposeAsync(storageMetadata -> {
        val segmentMetadata = (SegmentMetadata) storageMetadata;
        segmentMetadata.checkInvariants();
        val iterator = new ChunkIterator(chunkedSegmentStorage.getExecutor(), txn, segmentMetadata);
        val startOffset = new AtomicLong(segmentMetadata.getFirstChunkStartOffset());
        iterator.forEach((metadata, name) -> {
            infoList.add(ExtendedChunkInfo.builder().chunkName(name).startOffset(startOffset.get()).lengthInMetadata(metadata.getLength()).build());
            startOffset.addAndGet(metadata.getLength());
        });
        return completedFuture(infoList);
    }, chunkedSegmentStorage.getExecutor()).thenComposeAsync(v -> {
        val futures = new ArrayList<CompletableFuture<Void>>();
        if (checkStorage) {
            for (val info : infoList) {
                futures.add(chunkedSegmentStorage.getChunkStorage().exists(info.getChunkName()).thenComposeAsync(doesExist -> {
                    if (doesExist) {
                        return chunkedSegmentStorage.getChunkStorage().getInfo(info.getChunkName()).thenAcceptAsync(chunkInfo -> {
                            info.setLengthInStorage(chunkInfo.getLength());
                            info.setExistsInStorage(true);
                        }, chunkedSegmentStorage.getExecutor());
                    } else {
                        return completedFuture(null);
                    }
                }, chunkedSegmentStorage.getExecutor()));
            }
        }
        return Futures.allOf(futures);
    }, chunkedSegmentStorage.getExecutor()).thenApplyAsync(vv -> infoList, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor()), streamSegmentName);
}