Examples with StreamSegmentNotExistsException io.pravega.segmentstore.contracts.StreamSegmentNotExistsException used on opensource projects

Example 36 with StreamSegmentNotExistsException

use of io.pravega.segmentstore.contracts.StreamSegmentNotExistsException in project pravega by pravega.

the class OperationProcessorTests method testWithInvalidOperations.

/**
 * Tests the ability of the OperationProcessor to process Operations when encountering invalid operations (such as
 * appends to StreamSegments that do not exist or to those that are sealed). This covers the following exceptions:
 * * StreamSegmentNotExistsException
 * * StreamSegmentSealedException
 * * General MetadataUpdateException.
 */
@Test
public void testWithInvalidOperations() throws Exception {
    int streamSegmentCount = 10;
    int appendsPerStreamSegment = 40;
    // We are going to prematurely seal this StreamSegment.
    long sealedStreamSegmentId = 6;
    // We are going to prematurely mark this StreamSegment as deleted.
    long deletedStreamSegmentId = 8;
    // This is a bogus StreamSegment, that does not exist.
    long nonExistentStreamSegmentId;
    @Cleanup TestContext context = new TestContext();
    // Generate some test data (no need to complicate ourselves with Transactions here; that is tested in the no-failure test).
    HashSet<Long> streamSegmentIds = createStreamSegmentsInMetadata(streamSegmentCount, context.metadata);
    nonExistentStreamSegmentId = streamSegmentIds.size();
    streamSegmentIds.add(nonExistentStreamSegmentId);
    context.metadata.getStreamSegmentMetadata(sealedStreamSegmentId).markSealed();
    context.metadata.getStreamSegmentMetadata(deletedStreamSegmentId).markDeleted();
    List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
    // Setup an OperationProcessor and start it.
    @Cleanup TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, MAX_DATA_LOG_APPEND_SIZE, executorService());
    dataLog.initialize(TIMEOUT);
    @Cleanup OperationProcessor operationProcessor = new OperationProcessor(context.metadata, context.stateUpdater, dataLog, getNoOpCheckpointPolicy(), executorService());
    operationProcessor.startAsync().awaitRunning();
    // Process all generated operations.
    List<OperationWithCompletion> completionFutures = processOperations(operations, operationProcessor);
    // Wait for all such operations to complete. We are expecting exceptions, so verify that we do.
    AssertExtensions.assertThrows("No operations failed.", OperationWithCompletion.allOf(completionFutures)::join, ex -> ex instanceof MetadataUpdateException || ex instanceof StreamSegmentException);
    HashSet<Long> streamSegmentsWithNoContents = new HashSet<>();
    streamSegmentsWithNoContents.add(sealedStreamSegmentId);
    streamSegmentsWithNoContents.add(deletedStreamSegmentId);
    streamSegmentsWithNoContents.add(nonExistentStreamSegmentId);
    // Verify that the "right" operations failed, while the others succeeded.
    for (OperationWithCompletion oc : completionFutures) {
        if (oc.operation instanceof StorageOperation) {
            long streamSegmentId = ((StorageOperation) oc.operation).getStreamSegmentId();
            if (streamSegmentsWithNoContents.contains(streamSegmentId)) {
                Assert.assertTrue("Completion future for invalid StreamSegment " + streamSegmentId + " did not complete exceptionally.", oc.completion.isCompletedExceptionally());
                Predicate<Throwable> errorValidator;
                if (streamSegmentId == sealedStreamSegmentId) {
                    errorValidator = ex -> ex instanceof StreamSegmentSealedException;
                } else if (streamSegmentId == deletedStreamSegmentId) {
                    errorValidator = ex -> ex instanceof StreamSegmentNotExistsException;
                } else {
                    errorValidator = ex -> ex instanceof MetadataUpdateException;
                }
                AssertExtensions.assertThrows("Unexpected exception for failed Operation.", oc.completion::join, errorValidator);
                continue;
            }
        }
        // If we get here, we must verify no exception was thrown.
        oc.completion.join();
    }
    performLogOperationChecks(completionFutures, context.memoryLog, dataLog, context.metadata);
    performMetadataChecks(streamSegmentIds, streamSegmentsWithNoContents, new HashMap<>(), completionFutures, context.metadata, false, false);
    performReadIndexChecks(completionFutures, context.readIndex);
    operationProcessor.stopAsync().awaitTerminated();
}

Example 37 with StreamSegmentNotExistsException

use of io.pravega.segmentstore.contracts.StreamSegmentNotExistsException in project pravega by pravega.

the class ContainerReadIndexTests method testStorageFailedReads.

/**
 * Tests the ability to handle Storage read failures.
 */
@Test
public void testStorageFailedReads() {
    // Create all segments (Storage and Metadata).
    @Cleanup TestContext context = new TestContext();
    ArrayList<Long> segmentIds = createSegments(context);
    createSegmentsInStorage(context);
    // Read beyond Storage actual offset (metadata is corrupt)
    long testSegmentId = segmentIds.get(0);
    UpdateableSegmentMetadata sm = context.metadata.getStreamSegmentMetadata(testSegmentId);
    sm.setStorageLength(1024 * 1024);
    sm.setLength(1024 * 1024);
    AssertExtensions.assertThrows("Unexpected exception when attempting to read beyond the Segment length in Storage.", () -> {
        @Cleanup ReadResult readResult = context.readIndex.read(testSegmentId, 0, 100, TIMEOUT);
        Assert.assertTrue("Unexpected value from hasNext() when there should be at least one ReadResultEntry.", readResult.hasNext());
        ReadResultEntry entry = readResult.next();
        Assert.assertEquals("Unexpected ReadResultEntryType.", ReadResultEntryType.Storage, entry.getType());
        entry.requestContent(TIMEOUT);
        entry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    }, ex -> ex instanceof ArrayIndexOutOfBoundsException);
    // Segment not exists (exists in metadata, but not in Storage)
    val handle = context.storage.openWrite(sm.getName()).join();
    context.storage.delete(handle, TIMEOUT).join();
    AssertExtensions.assertThrows("Unexpected exception when attempting to from a segment that exists in Metadata, but not in Storage.", () -> {
        @Cleanup ReadResult readResult = context.readIndex.read(testSegmentId, 0, 100, TIMEOUT);
        Assert.assertTrue("Unexpected value from hasNext() when there should be at least one ReadResultEntry.", readResult.hasNext());
        ReadResultEntry entry = readResult.next();
        Assert.assertEquals("Unexpected ReadResultEntryType.", ReadResultEntryType.Storage, entry.getType());
        entry.requestContent(TIMEOUT);
        entry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    }, ex -> ex instanceof StreamSegmentNotExistsException);
}

Example 38 with StreamSegmentNotExistsException

use of io.pravega.segmentstore.contracts.StreamSegmentNotExistsException in project pravega by pravega.

the class RedirectedReadResultEntryTests method testGetContent.

/**
 * Tests the ability to retry (and switch base) when a failure occurred in getContent().
 */
@Test
public void testGetContent() {
    // More than one retry (by design, it will only retry one time; the next time it will simply throw).
    MockReadResultEntry t1 = new MockReadResultEntry(1, 1);
    RedirectedReadResultEntry e1 = new TestRedirectedReadResultEntry(t1, 0, (o, l) -> t1, executorService());
    t1.getContent().completeExceptionally(new StreamSegmentNotExistsException("foo"));
    AssertExtensions.assertThrows("getContent() did not throw when attempting to retry more than once.", e1::getContent, ex -> ex instanceof StreamSegmentNotExistsException);
    // Ineligible exception.
    MockReadResultEntry t2 = new MockReadResultEntry(1, 1);
    RedirectedReadResultEntry e2 = new TestRedirectedReadResultEntry(t2, 0, (o, l) -> t2, executorService());
    t2.getContent().completeExceptionally(new IntentionalException());
    AssertExtensions.assertThrows("getContent() did not throw when an ineligible exception got thrown.", e2::getContent, ex -> ex instanceof IntentionalException);
    // Given back another Redirect.
    MockReadResultEntry t3 = new MockReadResultEntry(1, 1);
    RedirectedReadResultEntry e3 = new TestRedirectedReadResultEntry(t3, 0, (o, l) -> e1, executorService());
    t3.getContent().completeExceptionally(new StreamSegmentNotExistsException("foo"));
    AssertExtensions.assertThrows("getContent() did not throw when a retry yielded another RedirectReadResultEntry.", e3::getContent, ex -> ex instanceof StreamSegmentNotExistsException);
    // Given redirect function fails.
    MockReadResultEntry t4 = new MockReadResultEntry(1, 1);
    t4.getContent().completeExceptionally(new StreamSegmentNotExistsException("foo"));
    RedirectedReadResultEntry e4 = new TestRedirectedReadResultEntry(t4, 0, (o, l) -> {
        throw new IntentionalException();
    }, executorService());
    AssertExtensions.assertThrows("getContent() did not throw when retry failed.", e4::getContent, ex -> ex instanceof StreamSegmentNotExistsException);
    // One that works correctly.
    MockReadResultEntry t5Bad = new MockReadResultEntry(1, 1);
    t5Bad.getContent().completeExceptionally(new StreamSegmentNotExistsException("foo"));
    MockReadResultEntry t5Good = new MockReadResultEntry(2, 1);
    t1.setCompletionCallback(i -> {
    // Do nothing.
    });
    t5Good.getContent().complete(new ReadResultEntryContents(new ByteArrayInputStream(new byte[1]), 1));
    RedirectedReadResultEntry e5 = new TestRedirectedReadResultEntry(t5Bad, 1, (o, l) -> t5Good, executorService());
    val finalResult = e5.getContent().join();
    Assert.assertEquals("Unexpected result from getCompletionCallback after successful redirect.", t5Bad.getCompletionCallback(), t5Good.getCompletionCallback());
    Assert.assertEquals("Unexpected result from getRequestedReadLength after successful redirect.", t5Bad.getRequestedReadLength(), e5.getRequestedReadLength());
    Assert.assertEquals("Unexpected result from getStreamSegmentOffset after successful redirect.", t5Good.getStreamSegmentOffset(), e5.getStreamSegmentOffset());
    Assert.assertEquals("Unexpected result from getContent after successful redirect.", t5Good.getContent().join(), finalResult);
}

Example 39 with StreamSegmentNotExistsException

use of io.pravega.segmentstore.contracts.StreamSegmentNotExistsException in project pravega by pravega.

the class StreamSegmentStorageReaderTests method testReadWithErrors.

/**
 * Tests the read() method when errors are present.
 */
@Test
public void testReadWithErrors() throws Exception {
    @Cleanup val s = createStorage();
    // 1. Segment does not exist.
    val si1 = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(SEGMENT_LENGTH).startOffset(0).sealed(true).build();
    val rr1 = StreamSegmentStorageReader.read(si1, 0, SEGMENT_LENGTH, SEGMENT_APPEND_SIZE - 1, s);
    val firstEntry1 = rr1.next();
    Assert.assertEquals("Unexpected ReadResultEntryType.", ReadResultEntryType.Storage, firstEntry1.getType());
    AssertExtensions.assertThrows("Unexpected exception when Segment does not exist initially.", () -> {
        firstEntry1.requestContent(TIMEOUT);
        return firstEntry1.getContent();
    }, ex -> ex instanceof StreamSegmentNotExistsException);
    populate(s);
    // 2. Segment exists initially, but is deleted while reading.
    val si2 = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(SEGMENT_LENGTH).startOffset(0).sealed(true).build();
    val rr2 = StreamSegmentStorageReader.read(si2, 0, SEGMENT_LENGTH, SEGMENT_APPEND_SIZE - 1, s);
    // Skip over the first entry.
    val firstEntry2 = rr2.next();
    firstEntry2.requestContent(TIMEOUT);
    firstEntry2.getContent().join();
    // Delete the Segment, then attempt to read again.
    s.delete(s.openWrite(SEGMENT_NAME).join(), TIMEOUT).join();
    val secondEntry = rr2.next();
    Assert.assertEquals("Unexpected ReadResultEntryType.", ReadResultEntryType.Storage, secondEntry.getType());
    AssertExtensions.assertThrows("Unexpected exception when Segment was deleted while reading.", () -> {
        secondEntry.requestContent(TIMEOUT);
        return secondEntry.getContent();
    }, ex -> ex instanceof StreamSegmentNotExistsException);
}

Example 40 with StreamSegmentNotExistsException

use of io.pravega.segmentstore.contracts.StreamSegmentNotExistsException in project pravega by pravega.

the class RollingStorage method read.

@Override
public int read(SegmentHandle handle, long offset, byte[] buffer, int bufferOffset, int length) throws StreamSegmentException {
    val h = asReadableHandle(handle);
    long traceId = LoggerHelpers.traceEnter(log, "read", handle, offset, length);
    ensureNotDeleted(h);
    Exceptions.checkArrayRange(bufferOffset, length, buffer.length, "bufferOffset", "length");
    if (offset < 0 || bufferOffset < 0 || length < 0 || buffer.length < bufferOffset + length) {
        throw new ArrayIndexOutOfBoundsException(String.format("Offset (%s) must be non-negative, and bufferOffset (%s) and length (%s) must be valid indices into buffer of size %s.", offset, bufferOffset, length, buffer.length));
    }
    if (h.isReadOnly() && !h.isSealed() && offset + length > h.length()) {
        // We have a non-sealed read-only handle. It's possible that the SegmentChunks may have been modified since
        // the last time we refreshed it, and we received a request for a read beyond our last known offset. Reload
        // the handle before attempting the read.
        val newHandle = (RollingSegmentHandle) openRead(handle.getSegmentName());
        h.refresh(newHandle);
        log.debug("Handle refreshed: {}.", h);
    }
    Preconditions.checkArgument(offset < h.length(), "Offset %s is beyond the last offset %s of the segment.", offset, h.length());
    Preconditions.checkArgument(offset + length <= h.length(), "Offset %s + length %s is beyond the last offset %s of the segment.", offset, length, h.length());
    // Read in a loop, from each SegmentChunk, until we can't read anymore.
    // If at any point we encounter a StreamSegmentNotExistsException, fail immediately with StreamSegmentTruncatedException (+inner).
    val chunks = h.chunks();
    int currentIndex = CollectionHelpers.binarySearch(chunks, s -> offset < s.getStartOffset() ? -1 : (offset >= s.getLastOffset() ? 1 : 0));
    assert currentIndex >= 0 : "unable to locate first SegmentChunk index.";
    try {
        int bytesRead = 0;
        while (bytesRead < length && currentIndex < chunks.size()) {
            // Verify if this is a known truncated SegmentChunk; if so, bail out quickly.
            SegmentChunk current = chunks.get(currentIndex);
            checkTruncatedSegment(null, h, current);
            if (current.getLength() == 0) {
                // Empty SegmentChunk; don't bother trying to read from it.
                continue;
            }
            long readOffset = offset + bytesRead - current.getStartOffset();
            int readLength = (int) Math.min(length - bytesRead, current.getLength() - readOffset);
            assert readOffset >= 0 && readLength >= 0 : "negative readOffset or readLength";
            // Read from the actual SegmentChunk into the given buffer.
            try {
                val sh = this.baseStorage.openRead(current.getName());
                int count = this.baseStorage.read(sh, readOffset, buffer, bufferOffset + bytesRead, readLength);
                bytesRead += count;
                if (readOffset + count >= current.getLength()) {
                    currentIndex++;
                }
            } catch (StreamSegmentNotExistsException ex) {
                log.debug("SegmentChunk '{}' does not exist anymore ({}).", current, h);
                checkTruncatedSegment(ex, h, current);
            }
        }
        LoggerHelpers.traceLeave(log, "read", traceId, handle, offset, bytesRead);
        return bytesRead;
    } catch (StreamSegmentTruncatedException ex) {
        // It's possible that the Segment has been truncated or deleted altogether using another handle. We need to
        // refresh the handle and throw the appropriate exception.
        val newHandle = (RollingSegmentHandle) openRead(handle.getSegmentName());
        h.refresh(newHandle);
        if (h.isDeleted()) {
            log.debug("Segment '{}' has been deleted. Cannot read anymore.", h);
            throw new StreamSegmentNotExistsException(handle.getSegmentName(), ex);
        } else {
            throw ex;
        }
    }
}