Examples with ChunkMetadata io.pravega.segmentstore.storage.metadata.ChunkMetadata used on opensource projects

Example 1 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class ChunkedSegmentStorage method claimOwnership.

/**
 * Checks ownership and adjusts the length of the segment if required.
 *
 * @param txn             Active {@link MetadataTransaction}.
 * @param segmentMetadata {@link SegmentMetadata} for the segment to change ownership for.
 *                        throws ChunkStorageException    In case of any chunk storage related errors.
 *                        throws StorageMetadataException In case of any chunk metadata store related errors.
 */
private CompletableFuture<Void> claimOwnership(MetadataTransaction txn, SegmentMetadata segmentMetadata) {
    // Get the last chunk
    val lastChunkName = segmentMetadata.getLastChunk();
    final CompletableFuture<Boolean> f;
    if (shouldAppend() && null != lastChunkName) {
        f = txn.get(lastChunkName).thenComposeAsync(storageMetadata -> {
            val lastChunk = (ChunkMetadata) storageMetadata;
            Preconditions.checkState(null != lastChunk, "last chunk metadata must not be null.");
            Preconditions.checkState(null != lastChunk.getName(), "Name of last chunk must not be null.");
            log.debug("{} claimOwnership - current last chunk - segment={}, last chunk={}, Length={}.", logPrefix, segmentMetadata.getName(), lastChunk.getName(), lastChunk.getLength());
            return chunkStorage.getInfo(lastChunkName).thenApplyAsync(chunkInfo -> {
                Preconditions.checkState(chunkInfo != null, "chunkInfo for last chunk must not be null.");
                Preconditions.checkState(lastChunk != null, "last chunk metadata must not be null.");
                // Adjust its length;
                if (chunkInfo.getLength() != lastChunk.getLength()) {
                    Preconditions.checkState(chunkInfo.getLength() > lastChunk.getLength(), "Length of last chunk on LTS must be greater than what is in metadata. Chunk=%s length=%s", lastChunk, chunkInfo.getLength());
                    // Whatever length you see right now is the final "sealed" length of the last chunk.
                    val oldLength = segmentMetadata.getLength();
                    lastChunk.setLength(chunkInfo.getLength());
                    segmentMetadata.setLength(segmentMetadata.getLastChunkStartOffset() + lastChunk.getLength());
                    if (!segmentMetadata.isStorageSystemSegment()) {
                        addBlockIndexEntriesForChunk(txn, segmentMetadata.getName(), lastChunk.getName(), segmentMetadata.getLastChunkStartOffset(), oldLength, segmentMetadata.getLength());
                    }
                    txn.update(lastChunk);
                    log.debug("{} claimOwnership - Length of last chunk adjusted - segment={}, last chunk={}, Length={}.", logPrefix, segmentMetadata.getName(), lastChunk.getName(), chunkInfo.getLength());
                }
                return true;
            }, executor).exceptionally(e -> {
                val ex = Exceptions.unwrap(e);
                if (ex instanceof ChunkNotFoundException) {
                    // This probably means that this instance is fenced out and newer instance truncated this segment.
                    // Try a commit of unmodified data to fail fast.
                    log.debug("{} claimOwnership - Last chunk was missing, failing fast - segment={}, last chunk={}.", logPrefix, segmentMetadata.getName(), lastChunk.getName());
                    txn.update(segmentMetadata);
                    return false;
                }
                throw new CompletionException(ex);
            });
        }, executor);
    } else {
        f = CompletableFuture.completedFuture(true);
    }
    return f.thenComposeAsync(shouldChange -> {
        // If this instance is no more owner, then transaction commit will fail.So it is still safe.
        if (shouldChange) {
            segmentMetadata.setOwnerEpoch(this.epoch);
            segmentMetadata.setOwnershipChanged(true);
        }
        // Update and commit
        // If This instance is fenced this update will fail.
        txn.update(segmentMetadata);
        return txn.commit();
    }, executor);
}

Example 2 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class TestUtils method insertMetadata.

/**
 * Insert Metadata as given.
 *
 * @param testSegmentName Name of the segment
 * @param maxRollingLength Max rolling length.
 * @param ownerEpoch Owner epoch.
 * @param chunkLengthsInMetadata Chunk lengths to set in metadata.
 * @param chunkLengthsInStorage Chunk lengths to set in storage.
 * @param addIndex Whether to add index.
 * @param addIndexMetadata Whether to add index metadata.
 * @param metadataStore Instance of {@link ChunkMetadataStore}
 * @param chunkedSegmentStorage Instance of {@link ChunkedSegmentStorage}.
 * @return {@link SegmentMetadata} representing segment.
 */
public static SegmentMetadata insertMetadata(String testSegmentName, long maxRollingLength, int ownerEpoch, long[] chunkLengthsInMetadata, long[] chunkLengthsInStorage, boolean addIndex, boolean addIndexMetadata, ChunkMetadataStore metadataStore, ChunkedSegmentStorage chunkedSegmentStorage) {
    Preconditions.checkArgument(maxRollingLength > 0, "maxRollingLength");
    Preconditions.checkArgument(ownerEpoch > 0, "ownerEpoch");
    try (val txn = metadataStore.beginTransaction(false, new String[] { testSegmentName })) {
        String firstChunk = null;
        String lastChunk = null;
        TreeMap<Long, String> index = new TreeMap<>();
        // Add chunks.
        long length = 0;
        long startOfLast = 0;
        long startOffset = 0;
        int chunkCount = 0;
        for (int i = 0; i < chunkLengthsInMetadata.length; i++) {
            String chunkName = testSegmentName + "_chunk_" + Integer.toString(i);
            ChunkMetadata chunkMetadata = ChunkMetadata.builder().name(chunkName).length(chunkLengthsInMetadata[i]).nextChunk(i == chunkLengthsInMetadata.length - 1 ? null : testSegmentName + "_chunk_" + Integer.toString(i + 1)).build();
            chunkMetadata.setActive(true);
            if (addIndex) {
                chunkedSegmentStorage.getReadIndexCache().addIndexEntry(testSegmentName, chunkName, startOffset);
            }
            index.put(startOffset, chunkName);
            startOffset += chunkLengthsInMetadata[i];
            length += chunkLengthsInMetadata[i];
            txn.create(chunkMetadata);
            addChunk(chunkedSegmentStorage.getChunkStorage(), chunkName, chunkLengthsInStorage[i]);
            chunkCount++;
        }
        // Fix the first and last
        if (chunkLengthsInMetadata.length > 0) {
            firstChunk = testSegmentName + "_chunk_0";
            lastChunk = testSegmentName + "_chunk_" + Integer.toString(chunkLengthsInMetadata.length - 1);
            startOfLast = length - chunkLengthsInMetadata[chunkLengthsInMetadata.length - 1];
        }
        // Finally save
        SegmentMetadata segmentMetadata = SegmentMetadata.builder().maxRollinglength(maxRollingLength).name(testSegmentName).ownerEpoch(ownerEpoch).firstChunk(firstChunk).lastChunk(lastChunk).length(length).lastChunkStartOffset(startOfLast).build();
        segmentMetadata.setActive(true);
        segmentMetadata.setChunkCount(chunkCount);
        segmentMetadata.checkInvariants();
        txn.create(segmentMetadata);
        if (addIndexMetadata) {
            for (long blockStartOffset = 0; blockStartOffset < segmentMetadata.getLength(); blockStartOffset += chunkedSegmentStorage.getConfig().getIndexBlockSize()) {
                val floor = index.floorEntry(blockStartOffset);
                txn.create(ReadIndexBlockMetadata.builder().name(NameUtils.getSegmentReadIndexBlockName(segmentMetadata.getName(), blockStartOffset)).startOffset(floor.getKey()).chunkName(floor.getValue()).status(StatusFlags.ACTIVE).build());
            }
        }
        txn.commit().join();
        return segmentMetadata;
    }
}

Example 3 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class ChunkedSegmentStorageTests method testWritesWithFlakyMetadataStore.

public void testWritesWithFlakyMetadataStore(int failFrequency, int length) throws Exception {
    String testSegmentName = "foo";
    @Cleanup TestContext testContext = getTestContext(ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().lazyCommitEnabled(false).build());
    val invocationCount = new AtomicInteger(0);
    val testMetadataStore = (InMemoryMetadataStore) testContext.metadataStore;
    testMetadataStore.setMaxEntriesInTxnBuffer(1);
    testMetadataStore.setWriteCallback(dummy -> {
        if (invocationCount.incrementAndGet() % failFrequency == 0) {
            return CompletableFuture.failedFuture(new IntentionalException("Intentional"));
        }
        return CompletableFuture.completedFuture(null);
    });
    val h = testContext.chunkedSegmentStorage.create(testSegmentName, null).get();
    byte[] data = populate(100);
    int currentOffset = 0;
    SegmentMetadata expectedSegmentMetadata = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
    ChunkMetadata expectedChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, expectedSegmentMetadata.getLastChunk());
    testMetadataStore.evictAllEligibleEntriesFromBuffer();
    testMetadataStore.evictFromCache();
    while (currentOffset < data.length) {
        try {
            int toWrite = Math.min(length, data.length - currentOffset);
            expectedSegmentMetadata = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
            expectedChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, expectedSegmentMetadata.getLastChunk());
            testContext.chunkedSegmentStorage.write(h, currentOffset, new ByteArrayInputStream(data, currentOffset, toWrite), toWrite, null).get();
            currentOffset += toWrite;
        } catch (Exception e) {
            if (!(Exceptions.unwrap(e) instanceof IntentionalException)) {
                throw e;
            }
            val actual = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
            val actualChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, expectedSegmentMetadata.getLastChunk());
            Assert.assertEquals(expectedSegmentMetadata, actual);
            Assert.assertEquals(expectedChunkMetadata, actualChunkMetadata);
        } finally {
            val info = testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).get();
            Assert.assertEquals(info.getLength(), currentOffset);
        }
    }
    testMetadataStore.setWriteCallback(null);
    checkDataRead(testSegmentName, testContext, 0, data.length, data);
}

Example 4 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class SystemJournalRecordsTests method testSystemSnapshotRecordSerialization.

@Test
public void testSystemSnapshotRecordSerialization() throws Exception {
    ArrayList<ChunkMetadata> list1 = new ArrayList<>();
    list1.add(ChunkMetadata.builder().name("name1").nextChunk("nextChunk1").length(1).status(2).build());
    list1.add(ChunkMetadata.builder().name("name12").length(1).status(2).build());
    ArrayList<ChunkMetadata> list2 = new ArrayList<>();
    list2.add(ChunkMetadata.builder().name("name2").nextChunk("nextChunk2").length(1).status(3).build());
    list2.add(ChunkMetadata.builder().name("name22").length(1).status(3).build());
    ArrayList<SystemJournal.SegmentSnapshotRecord> segmentlist = new ArrayList<>();
    segmentlist.add(SystemJournal.SegmentSnapshotRecord.builder().segmentMetadata(SegmentMetadata.builder().name("name1").length(1).chunkCount(2).startOffset(3).status(5).maxRollinglength(6).firstChunk("firstChunk111").lastChunk("lastChun111k").lastModified(7).firstChunkStartOffset(8).lastChunkStartOffset(9).ownerEpoch(10).build()).chunkMetadataCollection(list1).build());
    segmentlist.add(SystemJournal.SegmentSnapshotRecord.builder().segmentMetadata(SegmentMetadata.builder().name("name2").length(1).chunkCount(2).startOffset(3).status(5).maxRollinglength(6).firstChunk(null).lastChunk(null).lastModified(7).firstChunkStartOffset(8).lastChunkStartOffset(9).ownerEpoch(10).build()).chunkMetadataCollection(list2).build());
    val systemSnapshot = SystemJournal.SystemSnapshotRecord.builder().epoch(42).fileIndex(7).segmentSnapshotRecords(segmentlist).build();
    testSystemSnapshotRecordSerialization(systemSnapshot);
}

Example 5 with ChunkMetadata

use of io.pravega.segmentstore.storage.metadata.ChunkMetadata in project pravega by pravega.

the class GarbageCollectorTests method testMetadataExceptionForSegmentPartialMetadataUpdate.

/**
 * Test for segment that for which metadata is partially updated already in previous attempt.
 */
@Test
public void testMetadataExceptionForSegmentPartialMetadataUpdate() throws Exception {
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    @Cleanup ChunkMetadataStore metadataStore = spy(getMetadataStore());
    int containerId = CONTAINER_ID;
    int dataSize = 1;
    Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
    val testTaskQueue = new InMemoryTaskQueueManager();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
    @Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
    // Now actually start run
    garbageCollector.initialize(testTaskQueue).join();
    Assert.assertNotNull(garbageCollector.getTaskQueue());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, new long[] { 1, 2, 3, 4 }, false, 0);
    val chunkNames = TestUtils.getChunkNameList(metadataStore, "testSegment");
    // Simulate partial update of metadata.
    @Cleanup val txn = metadataStore.beginTransaction(false, "testSegment");
    val metadata = (ChunkMetadata) txn.get(chunkNames.stream().findFirst().get()).join();
    metadata.setActive(false);
    txn.update(metadata);
    txn.commit().join();
    // Add some garbage
    garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
    // Validate state before
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
    val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(1, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(list).join();
    // Validate state after
    Assert.assertEquals(4, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(4, garbageCollector.getQueueSize().get());
    garbageCollector.processBatch(testTaskQueue.drain(garbageCollector.getTaskQueueName(), 4)).join();
    Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
    Assert.assertEquals(0, garbageCollector.getQueueSize().get());
    chunkNames.stream().forEach(chunkName -> Assert.assertFalse(chunkName + " should not exist", chunkStorage.exists(chunkName).join()));
}