Examples with SegmentMetadata io.pravega.segmentstore.storage.metadata.SegmentMetadata used on opensource projects

Example 1 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class SystemJournal method applyChunkAddition.

/**
 * Apply chunk addition.
 */
private CompletableFuture<Void> applyChunkAddition(MetadataTransaction txn, Map<String, Long> chunkStartOffsets, String segmentName, String oldChunkName, String newChunkName, long offset) {
    Preconditions.checkState(null != oldChunkName, "oldChunkName must not be null");
    Preconditions.checkState(null != newChunkName && !newChunkName.isEmpty(), "newChunkName must not be null or empty");
    return txn.get(segmentName).thenComposeAsync(m -> {
        val segmentMetadata = (SegmentMetadata) m;
        segmentMetadata.checkInvariants();
        validateSegment(txn, segmentName);
        // set length.
        segmentMetadata.setLength(offset);
        val newChunkMetadata = ChunkMetadata.builder().name(newChunkName).build();
        newChunkMetadata.setActive(true);
        txn.create(newChunkMetadata);
        txn.markPinned(newChunkMetadata);
        chunkStartOffsets.put(newChunkName, offset);
        CompletableFuture<Void> f;
        // Set first and last pointers.
        if (!oldChunkName.isEmpty()) {
            Preconditions.checkState(txn.getData().containsKey(oldChunkName), "Txn must contain old key", oldChunkName);
            f = txn.get(oldChunkName).thenComposeAsync(mm -> {
                val oldChunk = (ChunkMetadata) mm;
                Preconditions.checkState(null != oldChunk, "oldChunk must not be null. oldChunkName=%s", oldChunkName);
                // In case the old segment store was still writing some zombie chunks when ownership changed
                // then new offset may invalidate tail part of chunk list.
                // Note that chunk with oldChunkName is still valid, it is the chunks after this that become invalid.
                val toDelete = new AtomicReference<>(oldChunk.getNextChunk());
                return Futures.loop(() -> toDelete.get() != null, () -> txn.get(toDelete.get()).thenAcceptAsync(mmm -> {
                    val chunkToDelete = (ChunkMetadata) mmm;
                    txn.delete(toDelete.get());
                    segmentMetadata.setChunkCount(segmentMetadata.getChunkCount() - 1);
                    // move to next chunk in list of now zombie chunks
                    toDelete.set(chunkToDelete.getNextChunk());
                }, executor), executor).thenAcceptAsync(v -> {
                    // Set next chunk
                    oldChunk.setNextChunk(newChunkName);
                    // Set length
                    val oldLength = chunkStartOffsets.get(oldChunkName);
                    oldChunk.setLength(offset - oldLength);
                    txn.update(oldChunk);
                }, executor);
            }, executor);
        } else {
            segmentMetadata.setFirstChunk(newChunkName);
            segmentMetadata.setStartOffset(offset);
            Preconditions.checkState(segmentMetadata.getChunkCount() == 0, "Chunk count must be 0. %s", segmentMetadata);
            f = CompletableFuture.completedFuture(null);
        }
        return f.thenComposeAsync(v -> {
            segmentMetadata.setLastChunk(newChunkName);
            segmentMetadata.setLastChunkStartOffset(offset);
            segmentMetadata.setChunkCount(segmentMetadata.getChunkCount() + 1);
            segmentMetadata.checkInvariants();
            // Save the segment metadata.
            txn.update(segmentMetadata);
            if (config.isSelfCheckEnabled()) {
                return validateSegment(txn, segmentName);
            } else {
                return CompletableFuture.completedFuture(null);
            }
        }, executor);
    }, executor);
}

Example 2 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class SystemJournal method adjustLastChunkLengths.

/**
 * Adjusts the lengths of last chunks for each segment.
 */
private CompletableFuture<Void> adjustLastChunkLengths(MetadataTransaction txn) {
    val futures = new ArrayList<CompletableFuture<Void>>();
    for (val systemSegment : systemSegments) {
        val f = txn.get(systemSegment).thenComposeAsync(m -> {
            val segmentMetadata = (SegmentMetadata) m;
            segmentMetadata.checkInvariants();
            CompletableFuture<Void> ff;
            // Update length of last chunk in metadata to what we actually find on LTS.
            if (null != segmentMetadata.getLastChunk()) {
                ff = chunkStorage.getInfo(segmentMetadata.getLastChunk()).thenComposeAsync(chunkInfo -> {
                    long length = chunkInfo.getLength();
                    return txn.get(segmentMetadata.getLastChunk()).thenAcceptAsync(mm -> {
                        val lastChunk = (ChunkMetadata) mm;
                        Preconditions.checkState(null != lastChunk, "lastChunk must not be null. Segment=%s", segmentMetadata);
                        lastChunk.setLength(length);
                        txn.update(lastChunk);
                        val newLength = segmentMetadata.getLastChunkStartOffset() + length;
                        segmentMetadata.setLength(newLength);
                        log.debug("SystemJournal[{}] Adjusting length of last chunk segment. segment={}, length={} chunk={}, chunk length={}", containerId, segmentMetadata.getName(), length, lastChunk.getName(), newLength);
                    }, executor);
                }, executor);
            } else {
                ff = CompletableFuture.completedFuture(null);
            }
            return ff.thenApplyAsync(v -> {
                Preconditions.checkState(segmentMetadata.isOwnershipChanged(), "ownershipChanged must be true. Segment=%s", segmentMetadata);
                segmentMetadata.checkInvariants();
                return segmentMetadata;
            }, executor);
        }, executor).thenAcceptAsync(segmentMetadata -> txn.update(segmentMetadata), executor);
        futures.add(f);
    }
    return Futures.allOf(futures);
}

Example 3 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class ChunkedSegmentStorageTests method testWritesWithFlakyMetadataStore.

public void testWritesWithFlakyMetadataStore(int failFrequency, int length) throws Exception {
    String testSegmentName = "foo";
    @Cleanup TestContext testContext = getTestContext(ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().lazyCommitEnabled(false).build());
    val invocationCount = new AtomicInteger(0);
    val testMetadataStore = (InMemoryMetadataStore) testContext.metadataStore;
    testMetadataStore.setMaxEntriesInTxnBuffer(1);
    testMetadataStore.setWriteCallback(dummy -> {
        if (invocationCount.incrementAndGet() % failFrequency == 0) {
            return CompletableFuture.failedFuture(new IntentionalException("Intentional"));
        }
        return CompletableFuture.completedFuture(null);
    });
    val h = testContext.chunkedSegmentStorage.create(testSegmentName, null).get();
    byte[] data = populate(100);
    int currentOffset = 0;
    SegmentMetadata expectedSegmentMetadata = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
    ChunkMetadata expectedChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, expectedSegmentMetadata.getLastChunk());
    testMetadataStore.evictAllEligibleEntriesFromBuffer();
    testMetadataStore.evictFromCache();
    while (currentOffset < data.length) {
        try {
            int toWrite = Math.min(length, data.length - currentOffset);
            expectedSegmentMetadata = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
            expectedChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, expectedSegmentMetadata.getLastChunk());
            testContext.chunkedSegmentStorage.write(h, currentOffset, new ByteArrayInputStream(data, currentOffset, toWrite), toWrite, null).get();
            currentOffset += toWrite;
        } catch (Exception e) {
            if (!(Exceptions.unwrap(e) instanceof IntentionalException)) {
                throw e;
            }
            val actual = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
            val actualChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, expectedSegmentMetadata.getLastChunk());
            Assert.assertEquals(expectedSegmentMetadata, actual);
            Assert.assertEquals(expectedChunkMetadata, actualChunkMetadata);
        } finally {
            val info = testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).get();
            Assert.assertEquals(info.getLength(), currentOffset);
        }
    }
    testMetadataStore.setWriteCallback(null);
    checkDataRead(testSegmentName, testContext, 0, data.length, data);
}

Example 4 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class GarbageCollector method deleteSegment.

/**
 * Perform delete segment related tasks.
 */
private CompletableFuture<Void> deleteSegment(TaskInfo taskInfo) {
    val streamSegmentName = taskInfo.getName();
    val txn = metadataStore.beginTransaction(true, streamSegmentName);
    return txn.get(streamSegmentName).thenComposeAsync(storageMetadata -> {
        val segmentMetadata = (SegmentMetadata) storageMetadata;
        if (null == segmentMetadata) {
            log.debug("{}: deleteGarbage - Segment metadata does not exist. segment={}.", traceObjectId, streamSegmentName);
            return CompletableFuture.completedFuture(null);
        } else if (segmentMetadata.isActive()) {
            log.debug("{}: deleteGarbage - Segment is not marked as deleted. segment={}.", traceObjectId, streamSegmentName);
            return CompletableFuture.completedFuture(null);
        } else {
            val chunksToDelete = Collections.synchronizedSet(new HashSet<String>());
            val currentBatch = Collections.synchronizedSet(new HashSet<ChunkMetadata>());
            val currentChunkName = new AtomicReference<String>(segmentMetadata.getFirstChunk());
            return Futures.loop(() -> null != currentChunkName.get(), () -> txn.get(currentChunkName.get()).thenComposeAsync(metadata -> {
                val chunkMetadata = (ChunkMetadata) metadata;
                CompletableFuture<Void> retFuture = CompletableFuture.completedFuture(null);
                // Skip if metadata is possibly deleted in last attempt, we are done.
                if (null == chunkMetadata) {
                    currentChunkName.set(null);
                    return retFuture;
                }
                // Add to list of chunks to delete
                chunksToDelete.add(chunkMetadata.getName());
                // Add to batch and commit batch if required.
                currentBatch.add(chunkMetadata);
                if (chunkMetadata.isActive()) {
                    if (currentBatch.size() > config.getGarbageCollectionTransactionBatchSize()) {
                        // Commit batch
                        retFuture = addTransactionForUpdateBatch(currentBatch, streamSegmentName);
                        // Clear batch
                        currentBatch.clear();
                    }
                }
                // Move next
                currentChunkName.set(chunkMetadata.getNextChunk());
                return retFuture;
            }, storageExecutor), storageExecutor).thenComposeAsync(v -> {
                if (currentBatch.size() > 0) {
                    return addTransactionForUpdateBatch(currentBatch, streamSegmentName);
                }
                return CompletableFuture.completedFuture(null);
            }, storageExecutor).thenComposeAsync(v -> this.addChunksToGarbage(txn.getVersion(), chunksToDelete), storageExecutor).thenComposeAsync(v -> deleteBlockIndexEntriesForSegment(streamSegmentName, segmentMetadata.getStartOffset(), segmentMetadata.getLength())).thenComposeAsync(v -> {
                val innerTxn = metadataStore.beginTransaction(false, segmentMetadata.getName());
                innerTxn.delete(segmentMetadata.getName());
                return innerTxn.commit().whenCompleteAsync((vv, ex) -> innerTxn.close(), storageExecutor);
            }, storageExecutor).handleAsync((v, e) -> {
                txn.close();
                if (null != e) {
                    log.error(String.format("%s deleteGarbage - Could not delete metadata for garbage segment=%s.", traceObjectId, streamSegmentName), e);
                    return true;
                }
                return false;
            }, storageExecutor).thenComposeAsync(failed -> {
                if (failed) {
                    if (taskInfo.getAttempts() < config.getGarbageCollectionMaxAttempts()) {
                        val attempts = taskInfo.attempts + 1;
                        SLTS_GC_SEGMENT_RETRY.inc();
                        return addSegmentToGarbage(taskInfo.toBuilder().attempts(attempts).build());
                    } else {
                        SLTS_GC_SEGMENT_FAILED.inc();
                        log.info("{}: deleteGarbage - could not delete after max attempts segment={}.", traceObjectId, taskInfo.getName());
                        return failTask(taskInfo);
                    }
                } else {
                    SLTS_GC_SEGMENT_PROCESSED.inc();
                    return CompletableFuture.completedFuture(null);
                }
            }, storageExecutor);
        }
    }, storageExecutor);
}

Example 5 with SegmentMetadata

use of io.pravega.segmentstore.storage.metadata.SegmentMetadata in project pravega by pravega.

the class ReadOperation method call.

@Override
public CompletableFuture<Integer> call() {
    // Validate preconditions.
    checkPreconditions();
    log.debug("{} read - started op={}, segment={}, offset={}, length={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), offset, length);
    val streamSegmentName = handle.getSegmentName();
    return ChunkedSegmentStorage.tryWith(chunkedSegmentStorage.getMetadataStore().beginTransaction(true, streamSegmentName), txn -> txn.get(streamSegmentName).thenComposeAsync(storageMetadata -> {
        segmentMetadata = (SegmentMetadata) storageMetadata;
        // Validate preconditions.
        checkState();
        if (length == 0) {
            return CompletableFuture.completedFuture(0);
        }
        return findChunkForOffset(txn).thenComposeAsync(v -> {
            // Now read.
            return readData(txn);
        }, chunkedSegmentStorage.getExecutor()).exceptionally(ex -> {
            log.debug("{} read - exception op={}, segment={}, offset={}, bytesRead={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), offset, totalBytesRead);
            if (ex instanceof CompletionException) {
                throw (CompletionException) ex;
            }
            throw new CompletionException(ex);
        }).thenApplyAsync(v -> {
            logEnd();
            return totalBytesRead.get();
        }, chunkedSegmentStorage.getExecutor());
    }, chunkedSegmentStorage.getExecutor()), chunkedSegmentStorage.getExecutor());
}