Examples with DataCorruptionException io.pravega.segmentstore.server.DataCorruptionException used on opensource projects

Example 6 with DataCorruptionException

use of io.pravega.segmentstore.server.DataCorruptionException in project pravega by pravega.

the class ContainerReadIndex method exitRecoveryMode.

@Override
public void exitRecoveryMode(boolean successfulRecovery) throws DataCorruptionException {
    Exceptions.checkNotClosed(this.closed.get(), this);
    Preconditions.checkState(this.isRecoveryMode(), "Read Index is not in recovery mode.");
    synchronized (this.lock) {
        assert this.preRecoveryMetadata != null : "preRecoveryMetadata is null, which should only be the case when we are not in recovery mode";
        Preconditions.checkState(!this.preRecoveryMetadata.isRecoveryMode(), "Cannot take ReadIndex out of recovery: ContainerMetadata is still in recovery mode.");
        if (successfulRecovery) {
            // Validate that the metadata has been properly recovered and that we are still in sync with it.
            for (Map.Entry<Long, StreamSegmentReadIndex> e : this.readIndices.entrySet()) {
                SegmentMetadata metadata = this.preRecoveryMetadata.getStreamSegmentMetadata(e.getKey());
                if (metadata == null) {
                    throw new DataCorruptionException(String.format("ContainerMetadata has no knowledge of StreamSegment Id %s.", e.getKey()));
                }
                e.getValue().exitRecoveryMode(metadata);
            }
        } else {
            // Recovery was unsuccessful. Clear the contents of the ReadIndex to avoid further issues.
            clear();
        }
        this.metadata = this.preRecoveryMetadata;
        this.preRecoveryMetadata = null;
    }
    log.info("{} Exit RecoveryMode.", this.traceObjectId);
}

Example 7 with DataCorruptionException

use of io.pravega.segmentstore.server.DataCorruptionException in project pravega by pravega.

the class StorageWriter method getSegmentAggregator.

// endregion
// region Helpers
/**
 * Gets, or creates, a SegmentAggregator for the given StorageOperation.
 *
 * @param streamSegmentId The Id of the StreamSegment to get the aggregator for.
 */
private CompletableFuture<SegmentAggregator> getSegmentAggregator(long streamSegmentId) {
    SegmentAggregator existingAggregator = this.aggregators.getOrDefault(streamSegmentId, null);
    if (existingAggregator != null) {
        if (closeIfNecessary(existingAggregator).isClosed()) {
            // Existing SegmentAggregator has become stale (most likely due to its SegmentMetadata being evicted),
            // so it has been closed and we need to create a new one.
            this.aggregators.remove(streamSegmentId);
        } else {
            return CompletableFuture.completedFuture(existingAggregator);
        }
    }
    // Get the SegmentAggregator's Metadata.
    UpdateableSegmentMetadata segmentMetadata = this.dataSource.getStreamSegmentMetadata(streamSegmentId);
    if (segmentMetadata == null) {
        return Futures.failedFuture(new DataCorruptionException(String.format("No StreamSegment with id '%d' is registered in the metadata.", streamSegmentId)));
    }
    // Then create the aggregator, and only register it after a successful initialization. Otherwise we risk
    // having a registered aggregator that is not initialized.
    SegmentAggregator newAggregator = new SegmentAggregator(segmentMetadata, this.dataSource, this.storage, this.config, this.timer, this.executor);
    try {
        CompletableFuture<Void> init = newAggregator.initialize(this.config.getFlushTimeout());
        Futures.exceptionListener(init, ex -> newAggregator.close());
        return init.thenApply(ignored -> {
            this.aggregators.put(streamSegmentId, newAggregator);
            return newAggregator;
        });
    } catch (Exception ex) {
        newAggregator.close();
        throw ex;
    }
}

Example 8 with DataCorruptionException

use of io.pravega.segmentstore.server.DataCorruptionException in project pravega by pravega.

the class SegmentStateStoreTests method testGetCorruptedState.

/**
 * Tests the get() method when there exists a state in Storage, however it is a corrupted file.
 */
@Test
public void testGetCorruptedState() throws Exception {
    final String segmentName = "foo";
    final String stateSegment = StreamSegmentNameUtils.getStateSegmentName(segmentName);
    val store = createStateStore();
    // Write some dummy contents in the file which is not how a SegmentState would be serialized.
    this.storage.create(stateSegment, TIMEOUT).thenCompose(si -> this.storage.openWrite(stateSegment)).thenCompose(handle -> this.storage.write(handle, 0, new ByteArrayInputStream(new byte[1]), 1, TIMEOUT)).join();
    AssertExtensions.assertThrows("Unexpected behavior when attempting to read a corrupted state file.", () -> store.get(segmentName, TIMEOUT), ex -> ex instanceof DataCorruptionException);
}

Example 9 with DataCorruptionException

use of io.pravega.segmentstore.server.DataCorruptionException in project pravega by pravega.

the class DataFrameReader method getNext.

// endregion
// region CloseableIterator Implementation
/**
 * Attempts to return the next Operation from the DataFrameLog.
 *
 * @return A DataFrameRecord with the requested operation. If no more Operations are available, null is returned.
 */
@Override
public DataFrameRecord<T> getNext() throws DataCorruptionException {
    Exceptions.checkNotClosed(this.closed, closed);
    try {
        while (!this.dataFrameInputStream.isClosed()) {
            try {
                if (!this.dataFrameInputStream.beginRecord()) {
                    // We've reached the end of the DataFrameInputStream.
                    return null;
                }
                // Attempt to deserialize the next record. If the serialization was bad, this will throw an exception which we'll pass along.
                T logItem = this.serializer.deserialize(this.dataFrameInputStream);
                DataFrameRecord.RecordInfo recordInfo = this.dataFrameInputStream.endRecord();
                long seqNo = logItem.getSequenceNumber();
                if (seqNo <= this.lastReadSequenceNumber) {
                    throw new DataCorruptionException(String.format("Invalid Operation Sequence Number. Expected: larger than %d, found: %d.", this.lastReadSequenceNumber, seqNo));
                }
                this.lastReadSequenceNumber = seqNo;
                return new DataFrameRecord<>(logItem, recordInfo);
            } catch (DataFrameInputStream.RecordResetException | DataFrameInputStream.NoMoreRecordsException ex) {
            // We partially "deserialized" a record, but realized it was garbage (a product of a failed, partial
            // serialization). Discard whatever we have and try again.
            } catch (IOException ex) {
                // Any other exceptions are considered to be non-DataCorruption.
                throw new DataCorruptionException("Deserialization failed.", ex);
            }
        }
        // No more data.
        return null;
    } catch (Exception ex) {
        // If we encountered any kind of reader exception, close the reader right away.
        // We do not do retries at this layer. Retries should be handled by the DataLog.
        // At this time, we close the reader for any kind of exception. In the future, we may decide to only do this
        // for critical exceptions, such as DataCorruptionException or DataLogNotAvailableException, but be able
        // to recover from other kinds of exceptions.
        // Since there are many layers of iterators (DataFrame, DataFrameRecord, LogItems), handling an exception at
        // the very top level is problematic, mostly because we would have to "rewind" some of the other iterators
        // to a previous position, otherwise any retries may read the wrong data.
        close();
        throw ex;
    }
}

Example 10 with DataCorruptionException

use of io.pravega.segmentstore.server.DataCorruptionException in project pravega by pravega.

the class MemoryStateUpdaterTests method testProcess.

/**
 * Tests the functionality of the process() method.
 */
@Test
public void testProcess() throws Exception {
    int segmentCount = 10;
    int operationCountPerType = 5;
    // Add to MTL + Add to ReadIndex (append; beginMerge).
    SequencedItemList<Operation> opLog = new SequencedItemList<>();
    ArrayList<TestReadIndex.MethodInvocation> methodInvocations = new ArrayList<>();
    TestReadIndex readIndex = new TestReadIndex(methodInvocations::add);
    AtomicInteger flushCallbackCallCount = new AtomicInteger();
    MemoryStateUpdater updater = new MemoryStateUpdater(opLog, readIndex, flushCallbackCallCount::incrementAndGet);
    ArrayList<Operation> operations = populate(updater, segmentCount, operationCountPerType);
    // Verify they were properly processed.
    int triggerFutureCount = (int) methodInvocations.stream().filter(mi -> mi.methodName.equals(TestReadIndex.TRIGGER_FUTURE_READS)).count();
    int addCount = methodInvocations.size() - triggerFutureCount;
    Assert.assertEquals("Unexpected number of items added to ReadIndex.", operations.size() - segmentCount * operationCountPerType, addCount);
    Assert.assertEquals("Unexpected number of calls to the ReadIndex triggerFutureReads method.", 1, triggerFutureCount);
    Assert.assertEquals("Unexpected number of calls to the flushCallback provided in the constructor.", 1, flushCallbackCallCount.get());
    // Verify add calls.
    Iterator<Operation> logIterator = opLog.read(-1, operations.size());
    int currentIndex = -1;
    int currentReadIndex = -1;
    while (logIterator.hasNext()) {
        currentIndex++;
        Operation expected = operations.get(currentIndex);
        Operation actual = logIterator.next();
        if (expected instanceof StorageOperation) {
            currentReadIndex++;
            TestReadIndex.MethodInvocation invokedMethod = methodInvocations.get(currentReadIndex);
            if (expected instanceof StreamSegmentAppendOperation) {
                Assert.assertTrue("StreamSegmentAppendOperation was not added as a CachedStreamSegmentAppendOperation to the Memory Log.", actual instanceof CachedStreamSegmentAppendOperation);
                StreamSegmentAppendOperation appendOp = (StreamSegmentAppendOperation) expected;
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was not added to the ReadIndex.", TestReadIndex.APPEND, invokedMethod.methodName);
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getStreamSegmentId(), invokedMethod.args.get("streamSegmentId"));
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getStreamSegmentOffset(), invokedMethod.args.get("offset"));
                Assert.assertEquals("Append with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", appendOp.getData(), invokedMethod.args.get("data"));
            } else if (expected instanceof MergeTransactionOperation) {
                MergeTransactionOperation mergeOp = (MergeTransactionOperation) expected;
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was not added to the ReadIndex.", TestReadIndex.BEGIN_MERGE, invokedMethod.methodName);
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getStreamSegmentId(), invokedMethod.args.get("targetStreamSegmentId"));
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getStreamSegmentOffset(), invokedMethod.args.get("offset"));
                Assert.assertEquals("Merge with SeqNo " + expected.getSequenceNumber() + " was added to the ReadIndex with wrong arguments.", mergeOp.getTransactionSegmentId(), invokedMethod.args.get("sourceStreamSegmentId"));
            }
        }
    }
    // Verify triggerFutureReads args.
    @SuppressWarnings("unchecked") Collection<Long> triggerSegmentIds = (Collection<Long>) methodInvocations.stream().filter(mi -> mi.methodName.equals(TestReadIndex.TRIGGER_FUTURE_READS)).findFirst().get().args.get("streamSegmentIds");
    val expectedSegmentIds = operations.stream().filter(op -> op instanceof SegmentOperation).map(op -> ((SegmentOperation) op).getStreamSegmentId()).collect(Collectors.toSet());
    AssertExtensions.assertContainsSameElements("ReadIndex.triggerFutureReads() was called with the wrong set of StreamSegmentIds.", expectedSegmentIds, triggerSegmentIds);
    // Test DataCorruptionException.
    AssertExtensions.assertThrows("MemoryStateUpdater accepted an operation that was out of order.", // This does not have a SequenceNumber set, so it should trigger a DCE.
    () -> updater.process(new MergeTransactionOperation(1, 2)), ex -> ex instanceof DataCorruptionException);
}