Examples with ChunkMetadata io.pravega.segmentstore.storage.metadata.ChunkMetadata used on opensource projects

Example 16 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class TestUtils method getChunkList.

/**
 * Gets the list of chunks for the given segment.
 *
 * @param metadataStore Metadata store to query.
 * @param key           Key.
 * @return List of {@link ChunkMetadata} for the segment.
 * @throws Exception Exceptions are thrown in case of any errors.
 */
public static ArrayList<ChunkMetadata> getChunkList(ChunkMetadataStore metadataStore, String key) throws Exception {
    try (val txn = metadataStore.beginTransaction(true, new String[] { key })) {
        val segmentMetadata = getSegmentMetadata(metadataStore, key);
        Assert.assertNotNull(segmentMetadata);
        ArrayList<ChunkMetadata> chunkList = new ArrayList<ChunkMetadata>();
        String current = segmentMetadata.getFirstChunk();
        while (null != current) {
            val chunk = (ChunkMetadata) txn.get(current).get();
            Assert.assertNotNull(chunk);
            chunkList.add((ChunkMetadata) chunk.deepCopy());
            current = chunk.getNextChunk();
        }
        return chunkList;
    }
}

Example 17 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class TestUtils method checkChunksExistInStorage.

/**
 * Checks if all chunks actually exist in storage for given segment.
 *
 * @param chunkStorage {@link ChunkStorage} instance to check.
 * @param metadataStore   {@link ChunkMetadataStore} instance to check.
 * @param segmentName     Segment name to check.
 * @throws Exception Exceptions are thrown in case of any errors.
 */
public static void checkChunksExistInStorage(ChunkStorage chunkStorage, ChunkMetadataStore metadataStore, String segmentName) throws Exception {
    int chunkCount = 0;
    long dataSize = 0;
    val segmentMetadata = getSegmentMetadata(metadataStore, segmentName);
    HashSet<String> visited = new HashSet<>();
    val chunkList = getChunkList(metadataStore, segmentName);
    for (ChunkMetadata chunkMetadata : chunkList) {
        Assert.assertTrue(chunkStorage.exists(chunkMetadata.getName()).get());
        val info = chunkStorage.getInfo(chunkMetadata.getName()).get();
        Assert.assertTrue(String.format("Actual %s, Expected %d", chunkMetadata, info.getLength()), chunkMetadata.getLength() <= info.getLength());
        chunkCount++;
        Assert.assertTrue("Chunk length should be non negative", info.getLength() >= 0);
        Assert.assertTrue(info.getLength() <= segmentMetadata.getMaxRollinglength());
        Assert.assertTrue(info.getLength() >= chunkMetadata.getLength());
        Assert.assertFalse("All chunks should be unique", visited.contains(info.getName()));
        visited.add(info.getName());
        dataSize += chunkMetadata.getLength();
    }
    Assert.assertEquals(chunkCount, segmentMetadata.getChunkCount());
    Assert.assertEquals(dataSize, segmentMetadata.getLength() - segmentMetadata.getFirstChunkStartOffset());
}

Example 18 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class SltsMetadataSerializer method handleStorageMetadataValue.

/**
 * Convert {@link StorageMetadata} into string of fields and values to be appended it into the given StringBuilder.
 *
 * @param builder  The given StringBuilder.
 * @param metadata The StorageMetadata instance.
 */
private void handleStorageMetadataValue(StringBuilder builder, StorageMetadata metadata) {
    if (metadata instanceof ChunkMetadata) {
        appendField(builder, METADATA_TYPE, CHUNK_METADATA);
        ChunkMetadata chunkMetadata = (ChunkMetadata) metadata;
        CHUNK_METADATA_FIELD_MAP.forEach((name, f) -> appendField(builder, name, String.valueOf(f.apply(chunkMetadata))));
    } else if (metadata instanceof SegmentMetadata) {
        appendField(builder, METADATA_TYPE, SEGMENT_METADATA);
        SegmentMetadata segmentMetadata = (SegmentMetadata) metadata;
        SEGMENT_METADATA_FIELD_MAP.forEach((name, f) -> appendField(builder, name, String.valueOf(f.apply(segmentMetadata))));
    } else if (metadata instanceof ReadIndexBlockMetadata) {
        appendField(builder, METADATA_TYPE, READ_INDEX_BLOCK_METADATA);
        ReadIndexBlockMetadata readIndexBlockMetadata = (ReadIndexBlockMetadata) metadata;
        READ_INDEX_BLOCK_METADATA_FIELD_MAP.forEach((name, f) -> appendField(builder, name, String.valueOf(f.apply(readIndexBlockMetadata))));
    }
}

Example 19 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class GarbageCollectorTests method insertSegment.

public SegmentMetadata insertSegment(ChunkMetadataStore metadataStore, ChunkStorage chunkStorage, ChunkedSegmentStorageConfig config, String testSegmentName, long maxRollingLength, int ownerEpoch, long[] chunkLengths, boolean addIndexMetadata, int status) throws Exception {
    Preconditions.checkArgument(maxRollingLength > 0, "maxRollingLength");
    Preconditions.checkArgument(ownerEpoch > 0, "ownerEpoch");
    try (val txn = metadataStore.beginTransaction(false, new String[] { testSegmentName })) {
        String firstChunk = null;
        String lastChunk = null;
        TreeMap<Long, String> index = new TreeMap<>();
        // Add chunks.
        long length = 0;
        long startOfLast = 0;
        long startOffset = 0;
        int chunkCount = 0;
        for (int i = 0; i < chunkLengths.length; i++) {
            String chunkName = testSegmentName + "_chunk_" + Integer.toString(i);
            ChunkMetadata chunkMetadata = ChunkMetadata.builder().name(chunkName).length(chunkLengths[i]).nextChunk(i == chunkLengths.length - 1 ? null : testSegmentName + "_chunk_" + Integer.toString(i + 1)).build();
            chunkMetadata.setActive(true);
            index.put(startOffset, chunkName);
            startOffset += chunkLengths[i];
            length += chunkLengths[i];
            txn.create(chunkMetadata);
            insertChunk(chunkStorage, chunkName, Math.toIntExact(chunkLengths[i]));
            chunkCount++;
        }
        // Fix the first and last
        if (chunkLengths.length > 0) {
            firstChunk = testSegmentName + "_chunk_0";
            lastChunk = testSegmentName + "_chunk_" + Integer.toString(chunkLengths.length - 1);
            startOfLast = length - chunkLengths[chunkLengths.length - 1];
        }
        // Finally save
        SegmentMetadata segmentMetadata = SegmentMetadata.builder().maxRollinglength(maxRollingLength).name(testSegmentName).ownerEpoch(ownerEpoch).firstChunk(firstChunk).lastChunk(lastChunk).length(length).lastChunkStartOffset(startOfLast).build();
        segmentMetadata.setStatus(status);
        segmentMetadata.setChunkCount(chunkCount);
        segmentMetadata.checkInvariants();
        txn.create(segmentMetadata);
        if (addIndexMetadata) {
            for (long blockStartOffset = 0; blockStartOffset < segmentMetadata.getLength(); blockStartOffset += config.getIndexBlockSize()) {
                val floor = index.floorEntry(blockStartOffset);
                txn.create(ReadIndexBlockMetadata.builder().name(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), blockStartOffset)).startOffset(floor.getKey()).chunkName(floor.getValue()).status(StatusFlags.ACTIVE).build());
            }
        }
        txn.commit().join();
        return segmentMetadata;
    }
}

Example 20 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class GarbageCollectorTests method insertChunkMetadata.

private void insertChunkMetadata(ChunkMetadataStore metadataStore, String chunkName, int dataSize, int status) throws Exception {
    try (val txn = metadataStore.beginTransaction(false, chunkName)) {
        txn.create(ChunkMetadata.builder().name(chunkName).length(dataSize).status(status).build());
        txn.commit().get();
    }
    try (val txn = metadataStore.beginTransaction(true, chunkName)) {
        val metadata = (ChunkMetadata) txn.get(chunkName).get();
        Assert.assertNotNull(metadata);
        Assert.assertEquals(chunkName, metadata.getName());
        Assert.assertEquals(status, metadata.getStatus());
    }
}