Examples with SegmentMetadata io.pravega.segmentstore.server.SegmentMetadata used on opensource projects

Example 11 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class ContainerReadIndexTests method testReadDirect.

/**
 * Tests the readDirect() method on the ReadIndex.
 */
@Test
public void testReadDirect() throws Exception {
    final int randomAppendLength = 1024;
    @Cleanup TestContext context = new TestContext();
    ArrayList<Long> segmentIds = new ArrayList<>();
    final long segmentId = createSegment(0, context);
    final UpdateableSegmentMetadata segmentMetadata = context.metadata.getStreamSegmentMetadata(segmentId);
    segmentIds.add(segmentId);
    HashMap<Long, ArrayList<Long>> transactionsBySegment = createTransactions(segmentIds, 1, context);
    final long mergedTxId = transactionsBySegment.get(segmentId).get(0);
    // Add data to all segments.
    HashMap<Long, ByteArrayOutputStream> segmentContents = new HashMap<>();
    transactionsBySegment.values().forEach(segmentIds::addAll);
    appendData(segmentIds, segmentContents, context);
    // Mark everything so far (minus a few bytes) as being written to storage.
    segmentMetadata.setStorageLength(segmentMetadata.getLength() - 100);
    // Now partially merge a second transaction
    final long mergedTxOffset = beginMergeTransaction(mergedTxId, segmentMetadata, segmentContents, context);
    // Add one more append after all of this.
    final long endOfMergedDataOffset = segmentMetadata.getLength();
    byte[] appendData = new byte[randomAppendLength];
    new Random(0).nextBytes(appendData);
    appendSingleWrite(segmentId, new ByteArraySegment(appendData), context);
    recordAppend(segmentId, new ByteArraySegment(appendData), segmentContents);
    // Verify we are not allowed to read from the range which has already been committed to Storage (invalid arguments).
    for (AtomicLong offset = new AtomicLong(0); offset.get() < segmentMetadata.getStorageLength(); offset.incrementAndGet()) {
        AssertExtensions.assertThrows(String.format("readDirect allowed reading from an illegal offset (%s).", offset), () -> context.readIndex.readDirect(segmentId, offset.get(), 1), ex -> ex instanceof IllegalArgumentException);
    }
    // Verify that any reads overlapping a merged transaction return null (that is, we cannot retrieve the requested data).
    for (long offset = mergedTxOffset - 1; offset < endOfMergedDataOffset; offset++) {
        val resultData = context.readIndex.readDirect(segmentId, offset, 2);
        Assert.assertNull("readDirect() returned data overlapping a partially merged transaction", resultData);
    }
    // Verify that we can read from any other offset.
    final byte[] expectedData = segmentContents.get(segmentId).toByteArray();
    BiConsumer<Long, Long> verifyReadResult = (startOffset, endOffset) -> {
        int readLength = (int) (endOffset - startOffset);
        while (readLength > 0) {
            BufferView actualDataBuffer;
            try {
                actualDataBuffer = context.readIndex.readDirect(segmentId, startOffset, readLength);
            } catch (StreamSegmentNotExistsException ex) {
                throw new CompletionException(ex);
            }
            Assert.assertNotNull(String.format("Unexpected result when data is readily available for Offset = %s, Length = %s.", startOffset, readLength), actualDataBuffer);
            byte[] actualData = actualDataBuffer.getCopy();
            AssertExtensions.assertArrayEquals("Unexpected data read from the segment at offset " + startOffset, expectedData, startOffset.intValue(), actualData, 0, actualData.length);
            // Setup the read for the next test (where we read 1 less byte than now).
            readLength--;
            if (readLength % 2 == 0) {
                // For every 2 bytes of decreased read length, increase the start offset by 1. This allows for a greater
                // number of combinations to be tested.
                startOffset++;
            }
        }
    };
    // Verify that we can read the cached data just after the StorageLength but before the merged transaction.
    verifyReadResult.accept(segmentMetadata.getStorageLength(), mergedTxOffset);
    // Verify that we can read the cached data just after the merged transaction but before the end of the segment.
    verifyReadResult.accept(endOfMergedDataOffset, segmentMetadata.getLength());
}

Example 12 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class ContainerReadIndexTests method createSegmentsInStorage.

private void createSegmentsInStorage(TestContext context) {
    for (long segmentId : context.metadata.getAllStreamSegmentIds()) {
        SegmentMetadata sm = context.metadata.getStreamSegmentMetadata(segmentId);
        context.storage.create(sm.getName(), TIMEOUT).join();
    }
}

Example 13 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class StorageReadManagerTests method testInvalidRequests.

/**
 * Tests the execute method with invalid Requests.
 * * StreamSegment does not exist
 * * Invalid read offset
 * * Too long of a read (offset+length is beyond the Segment's length)
 */
@Test
public void testInvalidRequests() {
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    // Segment does not exist.
    AssertExtensions.assertThrows("Request was not failed when StreamSegment does not exist.", () -> {
        SegmentMetadata sm = new StreamSegmentMetadata(SEGMENT_METADATA.getName(), 0, 0);
        @Cleanup StorageReadManager nonExistentReader = new StorageReadManager(sm, storage, executorService());
        sendRequest(nonExistentReader, 0, 1).join();
    }, ex -> ex instanceof StreamSegmentNotExistsException);
    // Now create segment, it should exist and request should succeed.
    byte[] segmentData = populateSegment(storage);
    @Cleanup StorageReadManager reader = new StorageReadManager(SEGMENT_METADATA, storage, executorService());
    sendRequest(reader, 0, 1).join();
    // Invalid read offset.
    AssertExtensions.assertSuppliedFutureThrows("Request was not failed when bad offset was provided.", () -> sendRequest(reader, segmentData.length + 1, 1), ex -> ex instanceof ArrayIndexOutOfBoundsException);
    // Invalid read length.
    AssertExtensions.assertSuppliedFutureThrows("Request was not failed when bad offset + length was provided.", () -> sendRequest(reader, segmentData.length - 1, 2), ex -> ex instanceof ArrayIndexOutOfBoundsException);
    // Make sure valid request succeeds after invalid one
    sendRequest(reader, 0, 1).join();
}

Example 14 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class ContainerReadIndex method getOrCreateIndex.

/**
 * Gets a reference to the existing StreamSegmentRead index for the given StreamSegment Id. Creates a new one if
 * necessary.
 *
 * @param streamSegmentId    The Id of the StreamSegment whose ReadIndex to get.
 */
private StreamSegmentReadIndex getOrCreateIndex(long streamSegmentId) throws StreamSegmentNotExistsException {
    StreamSegmentReadIndex index;
    synchronized (this.lock) {
        // Try to see if we have the index already in memory.
        index = getIndex(streamSegmentId);
        if (index != null && !index.isActive()) {
            // Index is registered, but it points to a segment metadata that is inactive. We should not be using
            // it anymore.
            closeIndex(streamSegmentId, true);
            index = null;
        }
        if (index == null) {
            // Create a new Segment Read Index.
            SegmentMetadata segmentMetadata = this.metadata.getStreamSegmentMetadata(streamSegmentId);
            if (segmentMetadata == null) {
                throw new IllegalArgumentException(String.format("Segment Id %d does not exist in the metadata.", streamSegmentId));
            } else if (!segmentMetadata.isActive()) {
                throw new IllegalArgumentException(String.format("Segment Id %d does exist in the metadata but is inactive.", streamSegmentId));
            } else if (segmentMetadata.isDeleted()) {
                throw new StreamSegmentNotExistsException(segmentMetadata.getName());
            }
            index = createSegmentIndex(this.config, segmentMetadata, this.cacheManager.getCacheStorage(), this.storage, this.executor, isRecoveryMode());
            this.cacheManager.register(index);
            this.readIndices.put(streamSegmentId, index);
        }
    }
    return index;
}

Example 15 with SegmentMetadata

use of io.pravega.segmentstore.server.SegmentMetadata in project pravega by pravega.

the class ContainerReadIndex method completeMerge.

@Override
public void completeMerge(long targetStreamSegmentId, long sourceStreamSegmentId) throws StreamSegmentNotExistsException {
    Exceptions.checkNotClosed(this.closed.get(), this);
    log.debug("{}: completeMerge (TargetId = {}, SourceId = {}.", this.traceObjectId, targetStreamSegmentId, sourceStreamSegmentId);
    SegmentMetadata sourceMetadata;
    synchronized (this.lock) {
        sourceMetadata = this.metadata.getStreamSegmentMetadata(sourceStreamSegmentId);
    }
    Preconditions.checkState(sourceMetadata != null, "No Metadata found for Segment Id %s.", sourceStreamSegmentId);
    StreamSegmentReadIndex targetIndex = getOrCreateIndex(targetStreamSegmentId);
    targetIndex.completeMerge(sourceMetadata);
    synchronized (this.lock) {
        // Do not clear the Cache after merger - we are reusing the cache entries from the source index in the target one.
        closeIndex(sourceStreamSegmentId, false);
    }
}