Examples with MetadataTransaction io.pravega.segmentstore.storage.metadata.MetadataTransaction used on opensource projects

Example 6 with MetadataTransaction

use of io.pravega.segmentstore.storage.metadata.MetadataTransaction in project pravega by pravega.

the class TruncateOperation method relocateFirstChunkIfRequired.

private CompletableFuture<Void> relocateFirstChunkIfRequired(MetadataTransaction txn) {
    if (shouldRelocate()) {
        String oldChunkName = segmentMetadata.getFirstChunk();
        String newChunkName = chunkedSegmentStorage.getNewChunkName(handle.getSegmentName(), segmentMetadata.getStartOffset());
        val startOffsetInChunk = segmentMetadata.getStartOffset() - segmentMetadata.getFirstChunkStartOffset();
        val newLength = Math.toIntExact(currentMetadata.getLength() - startOffsetInChunk);
        log.debug("{} truncate - relocating first chunk op={}, segment={}, offset={} old={} new={} relocatedBytes={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), offset, oldChunkName, newChunkName, newLength);
        return chunkedSegmentStorage.getGarbageCollector().trackNewChunk(txn.getVersion(), newChunkName).thenComposeAsync(v -> chunkedSegmentStorage.getChunkStorage().create(newChunkName), chunkedSegmentStorage.getExecutor()).thenComposeAsync(chunkHandle -> copyBytes(chunkHandle, ChunkHandle.readHandle(oldChunkName), startOffsetInChunk, newLength), chunkedSegmentStorage.getExecutor()).thenRunAsync(() -> {
            // Create metadata for new chunk.
            val newFirstChunkMetadata = ChunkMetadata.builder().name(newChunkName).nextChunk(currentMetadata.getNextChunk()).length(newLength).build().setActive(true);
            txn.create(newFirstChunkMetadata);
            // Update segment metadata.
            segmentMetadata.setFirstChunkStartOffset(segmentMetadata.getStartOffset());
            segmentMetadata.setFirstChunk(newChunkName);
            // Handle case when there is only one chunk
            if (segmentMetadata.getChunkCount() == 1) {
                segmentMetadata.setLastChunk(newChunkName);
                segmentMetadata.setLastChunkStartOffset(segmentMetadata.getStartOffset());
            }
            // Mark old first chunk for deletion.
            chunksToDelete.add(oldChunkName);
            currentMetadata.setActive(false);
            // Add block index entries.
            chunkedSegmentStorage.addBlockIndexEntriesForChunk(txn, segmentMetadata.getName(), newChunkName, segmentMetadata.getFirstChunkStartOffset(), segmentMetadata.getFirstChunkStartOffset(), segmentMetadata.getFirstChunkStartOffset() + newFirstChunkMetadata.getLength());
            isFirstChunkRelocated = true;
            currentMetadata = newFirstChunkMetadata;
            currentChunkName = newChunkName;
        }, chunkedSegmentStorage.getExecutor());
    }
    return CompletableFuture.completedFuture(null);
}

Example 7 with MetadataTransaction

use of io.pravega.segmentstore.storage.metadata.MetadataTransaction in project pravega by pravega.

the class WriteOperation method writeToChunk.

/**
 * Write to chunk.
 */
private CompletableFuture<Void> writeToChunk(MetadataTransaction txn, SegmentMetadata segmentMetadata, InputStream data, ChunkHandle chunkHandle, ChunkMetadata chunkWrittenMetadata, long offsetToWriteAt, int bytesCount) {
    Preconditions.checkState(0 != bytesCount, "Attempt to write zero bytes. Segment=%s Chunk=%s offsetToWriteAt=%s", segmentMetadata, chunkWrittenMetadata, offsetToWriteAt);
    // Finally write the data.
    val bis = new BoundedInputStream(data, bytesCount);
    CompletableFuture<Integer> retValue;
    if (chunkedSegmentStorage.shouldAppend()) {
        retValue = chunkedSegmentStorage.getChunkStorage().write(chunkHandle, offsetToWriteAt, bytesCount, bis);
    } else {
        retValue = chunkedSegmentStorage.getChunkStorage().createWithContent(chunkHandle.getChunkName(), bytesCount, bis).thenApplyAsync(h -> bytesCount, chunkedSegmentStorage.getExecutor());
    }
    return retValue.thenAcceptAsync(bytesWritten -> {
        // Update the metadata for segment and chunk.
        Preconditions.checkState(bytesWritten >= 0, "bytesWritten (%s) must be non-negative. Segment=%s Chunk=%s offsetToWriteAt=%s", bytesWritten, segmentMetadata, chunkWrittenMetadata, offsetToWriteAt);
        segmentMetadata.setLength(segmentMetadata.getLength() + bytesWritten);
        chunkWrittenMetadata.setLength(chunkWrittenMetadata.getLength() + bytesWritten);
        txn.update(chunkWrittenMetadata);
        txn.update(segmentMetadata);
        bytesRemaining.addAndGet(-bytesWritten);
        currentOffset.addAndGet(bytesWritten);
    }, chunkedSegmentStorage.getExecutor()).handleAsync((v, e) -> {
        if (null != e) {
            val ex = Exceptions.unwrap(e);
            if (ex instanceof InvalidOffsetException) {
                val invalidEx = (InvalidOffsetException) ex;
                // This could happen if the previous write failed while writing data and chunk was partially written.
                if (invalidEx.getExpectedOffset() > offsetToWriteAt) {
                    skipOverFailedChunk = true;
                    log.debug("{} write - skipping partially written chunk op={}, segment={}, chunk={} expected={} given={}.", chunkedSegmentStorage.getLogPrefix(), System.identityHashCode(this), handle.getSegmentName(), chunkHandle.getChunkName(), invalidEx.getExpectedOffset(), invalidEx.getGivenOffset());
                    return null;
                }
                throw new CompletionException(new BadOffsetException(segmentMetadata.getName(), currentOffset.get() + ((InvalidOffsetException) ex).getExpectedOffset(), currentOffset.get() + ((InvalidOffsetException) ex).getGivenOffset()));
            }
            throw new CompletionException(ex);
        }
        return v;
    }, chunkedSegmentStorage.getExecutor());
}

Example 8 with MetadataTransaction

use of io.pravega.segmentstore.storage.metadata.MetadataTransaction in project pravega by pravega.

the class WriteOperation method writeData.

private CompletableFuture<Void> writeData(MetadataTransaction txn) {
    val oldChunkCount = segmentMetadata.getChunkCount();
    val oldLength = segmentMetadata.getLength();
    return Futures.loop(() -> bytesRemaining.get() > 0, () -> {
        // This could be either because there are no existing chunks or last chunk has reached max rolling length.
        return openChunkToWrite(txn).thenComposeAsync(v -> {
            // Calculate the data that needs to be written.
            val oldOffset = currentOffset.get();
            val offsetToWriteAt = currentOffset.get() - segmentMetadata.getLastChunkStartOffset();
            val writeSize = (int) Math.min(bytesRemaining.get(), segmentMetadata.getMaxRollinglength() - offsetToWriteAt);
            // Write data to last chunk.
            return writeToChunk(txn, segmentMetadata, data, chunkHandle, lastChunkMetadata.get(), offsetToWriteAt, writeSize).thenRunAsync(() -> {
                // Update block index.
                if (!segmentMetadata.isStorageSystemSegment()) {
                    chunkedSegmentStorage.addBlockIndexEntriesForChunk(txn, segmentMetadata.getName(), chunkHandle.getChunkName(), segmentMetadata.getLastChunkStartOffset(), oldOffset, segmentMetadata.getLength());
                }
            }, chunkedSegmentStorage.getExecutor());
        }, chunkedSegmentStorage.getExecutor());
    }, chunkedSegmentStorage.getExecutor()).thenRunAsync(() -> {
        // Check invariants.
        segmentMetadata.checkInvariants();
        Preconditions.checkState(oldChunkCount + chunksAddedCount.get() == segmentMetadata.getChunkCount(), "Number of chunks do not match. old value (%s) + number of chunks added (%s) must match current chunk count(%s)", oldChunkCount, chunksAddedCount.get(), segmentMetadata.getChunkCount());
        Preconditions.checkState(oldLength + length == segmentMetadata.getLength(), "New length must match. old value (%s) + length (%s) must match current chunk count(%s)", oldLength, length, segmentMetadata.getLength());
        if (null != lastChunkMetadata.get()) {
            Preconditions.checkState(segmentMetadata.getLastChunkStartOffset() + lastChunkMetadata.get().getLength() == segmentMetadata.getLength(), "Last chunk start offset (%s) + Last chunk length (%s) must match segment length (%s)", segmentMetadata.getLastChunkStartOffset(), lastChunkMetadata.get().getLength(), segmentMetadata.getLength());
        }
    }, chunkedSegmentStorage.getExecutor());
}

Example 9 with MetadataTransaction

use of io.pravega.segmentstore.storage.metadata.MetadataTransaction in project pravega by pravega.

the class SystemJournal method createSystemSnapshotRecord.

private CompletableFuture<SystemSnapshotRecord> createSystemSnapshotRecord(MetadataTransaction txn, boolean validateSegment, boolean validateChunks) {
    val systemSnapshot = SystemSnapshotRecord.builder().epoch(epoch).fileIndex(currentFileIndex.get()).segmentSnapshotRecords(new ArrayList<>()).build();
    val futures = Collections.synchronizedList(new ArrayList<CompletableFuture<Void>>());
    for (val systemSegment : systemSegments) {
        // Find segment metadata.
        val future = txn.get(systemSegment).thenComposeAsync(metadata -> {
            val segmentMetadata = (SegmentMetadata) metadata;
            segmentMetadata.checkInvariants();
            val segmentSnapshot = SegmentSnapshotRecord.builder().segmentMetadata(segmentMetadata).chunkMetadataCollection(new ArrayList<>()).build();
            // Enumerate all chunks.
            val currentChunkName = new AtomicReference<>(segmentMetadata.getFirstChunk());
            val dataSize = new AtomicLong();
            val chunkCount = new AtomicLong();
            // For each chunk
            return Futures.loop(() -> null != currentChunkName.get(), () -> txn.get(currentChunkName.get()).thenComposeAsync(m -> {
                val currentChunkMetadata = (ChunkMetadata) m;
                CompletableFuture<Void> f;
                Preconditions.checkState(null != currentChunkMetadata, "currentChunkMetadata must not be null");
                if (validateChunks) {
                    f = chunkStorage.getInfo(currentChunkName.get()).thenAcceptAsync(chunkInfo -> Preconditions.checkState(chunkInfo.getLength() >= currentChunkMetadata.getLength(), "Wrong chunk length chunkInfo=%d, currentMetadata=%d.", chunkInfo.getLength(), currentChunkMetadata.getLength()), executor);
                } else {
                    f = CompletableFuture.completedFuture(null);
                }
                return f.thenAcceptAsync(v -> {
                    chunkCount.getAndIncrement();
                    dataSize.addAndGet(currentChunkMetadata.getLength());
                    segmentSnapshot.chunkMetadataCollection.add(currentChunkMetadata);
                    // move to next chunk
                    currentChunkName.set(currentChunkMetadata.getNextChunk());
                }, executor);
            }, executor), executor).thenAcceptAsync(v -> {
                // Validate
                if (validateSegment) {
                    Preconditions.checkState(chunkCount.get() == segmentMetadata.getChunkCount(), "Wrong chunk count. Segment=%s", segmentMetadata);
                    Preconditions.checkState(dataSize.get() == segmentMetadata.getLength() - segmentMetadata.getFirstChunkStartOffset(), "Data size does not match dataSize (%s). Segment=%s", dataSize.get(), segmentMetadata);
                }
                // Add to the system snapshot.
                synchronized (systemSnapshot) {
                    systemSnapshot.segmentSnapshotRecords.add(segmentSnapshot);
                }
            }, executor);
        }, executor);
        futures.add(future);
    }
    return Futures.allOf(futures).thenApplyAsync(vv -> {
        systemSnapshot.checkInvariants();
        return systemSnapshot;
    }, executor);
}

Example 10 with MetadataTransaction

use of io.pravega.segmentstore.storage.metadata.MetadataTransaction in project pravega by pravega.

the class SystemJournal method processJournalContents.

private CompletableFuture<Void> processJournalContents(MetadataTransaction txn, BootstrapState state, String systemLogName, ByteArrayInputStream input) {
    // Loop is exited with eventual EOFException.
    val isBatchDone = new AtomicBoolean();
    return Futures.loop(() -> !isBatchDone.get(), () -> {
        try {
            log.debug("SystemJournal[{}] Processing journal {}.", containerId, systemLogName);
            val batch = BATCH_SERIALIZER.deserialize(input);
            return Futures.loop(batch.getSystemJournalRecords(), record -> applyRecord(txn, state, record).thenApply(r -> true), executor);
        } catch (EOFException e) {
            log.debug("SystemJournal[{}] Done processing journal {}.", containerId, systemLogName);
            isBatchDone.set(true);
        } catch (Exception e) {
            log.error("SystemJournal[{}] Error while processing journal {}.", containerId, systemLogName, e);
            throw new CompletionException(e);
        }
        return CompletableFuture.completedFuture(null);
    }, executor);
}