use of io.pravega.segmentstore.storage.DataLogNotAvailableException in project pravega by pravega.
the class BookKeeperLogTests method testAutoCloseOnBookieFailure.
/**
* Tests the ability to auto-close upon a permanent write failure caused by BookKeeper.
*
* @throws Exception If one got thrown.
*/
@Test
public void testAutoCloseOnBookieFailure() throws Exception {
try (DurableDataLog log = createDurableDataLog()) {
log.initialize(TIMEOUT);
try {
// Suspend a bookie (this will trigger write errors).
stopFirstBookie();
// First write should fail. Either a DataLogNotAvailableException (insufficient bookies) or
// WriteFailureException (general unable to write) should be thrown.
AssertExtensions.assertSuppliedFutureThrows("First write did not fail with the appropriate exception.", () -> log.append(new CompositeByteArraySegment(getWriteData()), TIMEOUT), ex -> ex instanceof RetriesExhaustedException && (ex.getCause() instanceof DataLogNotAvailableException || isLedgerClosedException(ex.getCause())) || ex instanceof ObjectClosedException || ex instanceof CancellationException);
// Subsequent writes should be rejected since the BookKeeperLog is now closed.
AssertExtensions.assertSuppliedFutureThrows("Second write did not fail with the appropriate exception.", () -> log.append(new CompositeByteArraySegment(getWriteData()), TIMEOUT), ex -> ex instanceof ObjectClosedException || ex instanceof CancellationException);
} finally {
// Don't forget to resume the bookie.
restartFirstBookie();
}
}
}
use of io.pravega.segmentstore.storage.DataLogNotAvailableException in project pravega by pravega.
the class DurableLogTests method testRecoveryFailures.
/**
* Tests the DurableLog recovery process in a scenario when there are failures during the process
* (these may or may not be DataCorruptionExceptions).
*/
@Test
public void testRecoveryFailures() throws Exception {
int streamSegmentCount = 50;
int appendsPerStreamSegment = 20;
// Fail DataLog reads after X reads.
int failReadAfter = 2;
// Setup a DurableLog and start it.
AtomicReference<TestDurableDataLog> dataLog = new AtomicReference<>();
@Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), dataLog::set);
@Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
storage.initialize(1);
Set<Long> streamSegmentIds;
List<OperationWithCompletion> completionFutures;
// First DurableLog. We use this for generating data.
UpdateableContainerMetadata metadata = new MetadataBuilder(CONTAINER_ID).build();
@Cleanup CacheStorage cacheStorage = new DirectMemoryCache(Integer.MAX_VALUE);
@Cleanup CacheManager cacheManager = new CacheManager(CachePolicy.INFINITE, cacheStorage, executorService());
try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, storage, cacheManager, executorService());
DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
durableLog.startAsync().awaitRunning();
// Generate some test data (we need to do this after we started the DurableLog because in the process of
// recovery, it wipes away all existing metadata).
streamSegmentIds = createStreamSegmentsWithOperations(streamSegmentCount, durableLog);
List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
// Process all generated operations and wait for them to complete
completionFutures = processOperations(operations, durableLog);
OperationWithCompletion.allOf(completionFutures).join();
// Stop the processor.
durableLog.stopAsync().awaitTerminated();
}
// Recovery failure due to DataLog Failures.
metadata = new MetadataBuilder(CONTAINER_ID).build();
dataLog.set(null);
try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, storage, cacheManager, executorService());
DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
// Inject some artificial error into the DataLogRead after a few reads.
ErrorInjector<Exception> readNextInjector = new ErrorInjector<>(count -> count > failReadAfter, () -> new DataLogNotAvailableException("intentional"));
dataLog.get().setReadErrorInjectors(null, readNextInjector);
// Verify the exception thrown from startAsync() is of the right kind. This exception will be wrapped in
// multiple layers, so we need to dig deep into it.
AssertExtensions.assertThrows("Recovery did not fail properly when expecting DurableDataLogException.", () -> durableLog.startAsync().awaitRunning(), ex -> {
if (ex instanceof IllegalStateException) {
ex = ex.getCause();
}
if (ex == null) {
try {
// We need this to enter a FAILED state to get its failure cause.
durableLog.awaitTerminated();
} catch (Exception ex2) {
ex = durableLog.failureCause();
}
}
ex = Exceptions.unwrap(ex);
return ex instanceof DataLogNotAvailableException && ex.getMessage().equals("intentional");
});
}
// Recovery failure due to DataCorruptionException.
metadata = new MetadataBuilder(CONTAINER_ID).build();
dataLog.set(null);
try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, storage, cacheManager, executorService());
DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
// Reset error injectors to nothing.
dataLog.get().setReadErrorInjectors(null, null);
AtomicInteger readCounter = new AtomicInteger();
dataLog.get().setReadInterceptor(readItem -> {
if (readCounter.incrementAndGet() > failReadAfter && readItem.getLength() > DataFrame.MIN_ENTRY_LENGTH_NEEDED) {
// Mangle with the payload and overwrite its contents with a DataFrame having a bogus
// previous sequence number.
DataFrame df = DataFrame.ofSize(readItem.getLength());
df.seal();
CompositeArrayView serialization = df.getData();
return new InjectedReadItem(serialization.getReader(), serialization.getLength(), readItem.getAddress());
}
return readItem;
});
// Verify the exception thrown from startAsync() is of the right kind. This exception will be wrapped in
// multiple layers, so we need to dig deep into it.
AssertExtensions.assertThrows("Recovery did not fail properly when expecting DataCorruptionException.", () -> durableLog.startAsync().awaitRunning(), ex -> {
if (ex instanceof IllegalStateException) {
ex = ex.getCause();
}
return Exceptions.unwrap(ex) instanceof DataCorruptionException;
});
// Verify that the underlying DurableDataLog has been disabled.
val disabledDataLog = dataLogFactory.createDurableDataLog(CONTAINER_ID);
AssertExtensions.assertThrows("DurableDataLog has not been disabled following a recovery failure with DataCorruptionException.", () -> disabledDataLog.initialize(TIMEOUT), ex -> ex instanceof DataLogDisabledException);
}
}
use of io.pravega.segmentstore.storage.DataLogNotAvailableException in project pravega by pravega.
the class DataFrameReaderTests method testReadsWithDataLogFailure.
/**
* Tests the case when the DataFrameReader reads from a log and it encounters log read failures.
* 1. Initial read failures.
* 2. Somewhere in the middle of reading.
*/
@Test
public void testReadsWithDataLogFailure() throws Exception {
// Fail reads synchronously every X attempts.
int failReadSyncEvery = 3;
ArrayList<TestLogItem> records = DataFrameTestHelpers.generateLogItems(100, SMALL_RECORD_MIN_SIZE, SMALL_RECORD_MAX_SIZE, 0);
records.addAll(DataFrameTestHelpers.generateLogItems(100, LARGE_RECORD_MIN_SIZE, LARGE_RECORD_MAX_SIZE, records.size()));
try (TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, FRAME_SIZE, executorService())) {
dataLog.initialize(TIMEOUT);
BiConsumer<Throwable, DataFrameBuilder.CommitArgs> errorCallback = (ex, a) -> Assert.fail(String.format("Unexpected error occurred upon commit. %s", ex));
val args = new DataFrameBuilder.Args(Callbacks::doNothing, Callbacks::doNothing, errorCallback, executorService());
try (DataFrameBuilder<TestLogItem> b = new DataFrameBuilder<>(dataLog, SERIALIZER, args)) {
for (TestLogItem r : records) {
b.append(r);
}
}
TestSerializer logItemFactory = new TestSerializer();
// Test 1: Initial call to getReader.
ErrorInjector<Exception> getReaderErrorInjector = new ErrorInjector<>(// Fail every time.
count -> true, () -> new DataLogNotAvailableException("intentional getReader exception"));
dataLog.setReadErrorInjectors(getReaderErrorInjector, null);
AssertExtensions.assertThrows("No exception or wrong type of exception thrown by getNext() with exception thrown by getReader().", () -> new DataFrameReader<>(dataLog, logItemFactory, CONTAINER_ID), ex -> Exceptions.unwrap(ex) == getReaderErrorInjector.getLastCycleException());
// Test 2: Failures during getNext().
ErrorInjector<Exception> readErrorInjector = new ErrorInjector<>(count -> count % failReadSyncEvery == 0, () -> new DataLogNotAvailableException("intentional getNext exception"));
dataLog.setReadErrorInjectors(null, readErrorInjector);
testReadWithException(dataLog, logItemFactory, ex -> ex == readErrorInjector.getLastCycleException());
}
}
use of io.pravega.segmentstore.storage.DataLogNotAvailableException in project pravega by pravega.
the class BookKeeperLog method handleWriteException.
/**
* Handles an exception after a Write operation, converts it to a Pravega Exception and completes the given future
* exceptionally using it.
*
* @param ex The exception from BookKeeper client.
* @param write The Write that failed.
*/
@VisibleForTesting
static void handleWriteException(Throwable ex, Write write, BookKeeperLog bookKeeperLog) {
try {
int code = Code.UnexpectedConditionException;
if (ex instanceof BKException) {
BKException bKException = (BKException) ex;
code = bKException.getCode();
}
switch(code) {
case Code.LedgerFencedException:
// We were fenced out.
ex = new DataLogWriterNotPrimaryException("BookKeeperLog is not primary anymore.", ex);
break;
case Code.NotEnoughBookiesException:
// Insufficient Bookies to complete the operation. This is a retryable exception.
ex = new DataLogNotAvailableException("BookKeeperLog is not available.", ex);
break;
case Code.LedgerClosedException:
// LedgerClosed can happen because we just rolled over the ledgers or because BookKeeper closed a ledger
// due to some error. In either case, this is a retryable exception.
ex = new WriteFailureException("Active Ledger is closed.", ex);
break;
case Code.WriteException:
// Write-related failure or current Ledger closed. This is a retryable exception.
ex = new WriteFailureException("Unable to write to active Ledger.", ex);
break;
case Code.ClientClosedException:
// The BookKeeper client was closed externally. We cannot restart it here. We should close.
ex = new ObjectClosedException(bookKeeperLog, ex);
break;
default:
// All the other kind of exceptions go in the same bucket.
ex = new DurableDataLogException("General exception while accessing BookKeeper.", ex);
}
} finally {
write.fail(ex, !isRetryable(ex));
}
}
use of io.pravega.segmentstore.storage.DataLogNotAvailableException in project pravega by pravega.
the class BookKeeperLogTests method testFactoryInitialize.
/**
* Tests the BookKeeperLogFactory and its initialization.
*/
@Test
public void testFactoryInitialize() {
BookKeeperConfig bkConfig = BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, "127.0.0.1:" + TestUtils.getAvailableListenPort()).with(BookKeeperConfig.BK_LEDGER_MAX_SIZE, // Very frequent rollovers.
WRITE_MAX_LENGTH * 10).with(BookKeeperConfig.ZK_METADATA_PATH, this.zkClient.get().getNamespace()).build();
@Cleanup val factory = new BookKeeperLogFactory(bkConfig, this.zkClient.get(), executorService());
AssertExtensions.assertThrows("", factory::initialize, ex -> ex instanceof DataLogNotAvailableException && ex.getCause() != null);
}
Aggregations