Examples with DurableDataLog io.pravega.segmentstore.storage.DurableDataLog used on opensource projects

Example 6 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class BookKeeperAdapter method shutDown.

@Override
protected void shutDown() {
    this.logs.values().forEach(DurableDataLog::close);
    this.logs.clear();
    BookKeeperLogFactory lf = this.logFactory;
    if (lf != null) {
        lf.close();
        this.logFactory = null;
    }
    stopBookKeeper();
    CuratorFramework zkClient = this.zkClient;
    if (zkClient != null) {
        zkClient.close();
        this.zkClient = null;
    }
    Runtime.getRuntime().removeShutdownHook(this.stopBookKeeperProcess);
}

Example 7 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class BookKeeperLogTests method testAutoCloseOnBookieFailure.

/**
 * Tests the ability to auto-close upon a permanent write failure caused by BookKeeper.
 *
 * @throws Exception If one got thrown.
 */
@Test
public void testAutoCloseOnBookieFailure() throws Exception {
    try (DurableDataLog log = createDurableDataLog()) {
        log.initialize(TIMEOUT);
        try {
            // Suspend a bookie (this will trigger write errors).
            stopFirstBookie();
            // First write should fail. Either a DataLogNotAvailableException (insufficient bookies) or
            // WriteFailureException (general unable to write) should be thrown.
            AssertExtensions.assertThrows("First write did not fail with the appropriate exception.", () -> log.append(new ByteArraySegment(getWriteData()), TIMEOUT), ex -> ex instanceof RetriesExhaustedException && (ex.getCause() instanceof DataLogNotAvailableException || isLedgerClosedException(ex.getCause())) || ex instanceof ObjectClosedException || ex instanceof CancellationException);
            // Subsequent writes should be rejected since the BookKeeperLog is now closed.
            AssertExtensions.assertThrows("Second write did not fail with the appropriate exception.", () -> log.append(new ByteArraySegment(getWriteData()), TIMEOUT), ex -> ex instanceof ObjectClosedException || ex instanceof CancellationException);
        } finally {
            // Don't forget to resume the bookie.
            restartFirstBookie();
        }
    }
}

Example 8 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class InMemoryDurableDataLogTests method testConstructor.

/**
 * Tests the constructor of InMemoryDurableDataLog. The constructor takes in an EntryCollection and this verifies
 * that information from a previous instance of an InMemoryDurableDataLog is still accessible.
 */
@Test(timeout = 5000)
public void testConstructor() throws Exception {
    InMemoryDurableDataLog.EntryCollection entries = new InMemoryDurableDataLog.EntryCollection();
    TreeMap<LogAddress, byte[]> writeData;
    // Create first log and write some data to it.
    try (DurableDataLog log = new InMemoryDurableDataLog(entries, executorService())) {
        log.initialize(TIMEOUT);
        writeData = populate(log, WRITE_COUNT);
    }
    // Close the first log, and open a second one, with the same EntryCollection in the constructor.
    try (DurableDataLog log = new InMemoryDurableDataLog(entries, executorService())) {
        log.initialize(TIMEOUT);
        // Verify it contains the same entries.
        verifyReads(log, writeData);
    }
}

Example 9 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class StreamSegmentContainerTests method testStartOffline.

/**
 * Tests the ability of the StreamSegmentContainer to start in Offline mode (due to an offline DurableLog) and eventually
 * become online when the DurableLog becomes too.
 */
@Test
public void testStartOffline() throws Exception {
    @Cleanup val context = new TestContext();
    AtomicReference<DurableDataLog> dataLog = new AtomicReference<>();
    @Cleanup val dataLogFactory = new TestDurableDataLogFactory(context.dataLogFactory, dataLog::set);
    AtomicReference<OperationLog> durableLog = new AtomicReference<>();
    val durableLogFactory = new WatchableOperationLogFactory(new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, dataLogFactory, executorService()), durableLog::set);
    val containerFactory = new StreamSegmentContainerFactory(DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.writerFactory, context.storageFactory, executorService());
    // Write some data
    ArrayList<String> segmentNames = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
        for (int i = 0; i < SEGMENT_COUNT; i++) {
            String segmentName = getSegmentName(i);
            segmentNames.add(segmentName);
            opFutures.add(container.createStreamSegment(segmentName, null, TIMEOUT));
        }
        for (int i = 0; i < APPENDS_PER_SEGMENT / 2; i++) {
            for (String segmentName : segmentNames) {
                byte[] appendData = getAppendData(segmentName, i);
                opFutures.add(container.append(segmentName, appendData, null, TIMEOUT));
                lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
                recordAppend(segmentName, appendData, segmentContents);
            }
        }
        Futures.allOf(opFutures).join();
        // Disable the DurableDataLog.
        dataLog.get().disable();
    }
    // Start in "Offline" mode, verify operations cannot execute and then shut down - make sure we can shut down an offline container.
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        Assert.assertTrue("Expecting Segment Container to be offline.", container.isOffline());
        AssertExtensions.assertThrows("append() worked in offline mode.", () -> container.append("foo", new byte[1], null, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("getStreamSegmentInfo() worked in offline mode.", () -> container.getStreamSegmentInfo("foo", false, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("read() worked in offline mode.", () -> container.read("foo", 0, 1, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        container.stopAsync().awaitTerminated();
    }
    // Start in "Offline" mode and verify we can resume a normal startup.
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        Assert.assertTrue("Expecting Segment Container to be offline.", container.isOffline());
        dataLog.get().enable();
        // Wait for the DurableLog to become online.
        durableLog.get().awaitOnline().get(DEFAULT_DURABLE_LOG_CONFIG.getStartRetryDelay().toMillis() * 100, TimeUnit.MILLISECONDS);
        // Verify we can execute regular operations now.
        ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
        for (int i = 0; i < APPENDS_PER_SEGMENT / 2; i++) {
            for (String segmentName : segmentNames) {
                byte[] appendData = getAppendData(segmentName, i);
                opFutures.add(container.append(segmentName, appendData, null, TIMEOUT));
                lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
                recordAppend(segmentName, appendData, segmentContents);
            }
        }
        Futures.allOf(opFutures).join();
        // Verify all operations arrived in Storage.
        ArrayList<CompletableFuture<Void>> segmentsCompletion = new ArrayList<>();
        for (String segmentName : segmentNames) {
            SegmentProperties sp = container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
            segmentsCompletion.add(waitForSegmentInStorage(sp, context));
        }
        Futures.allOf(segmentsCompletion).join();
        container.stopAsync().awaitTerminated();
    }
}

Example 10 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class BookKeeperAdapter method createStream.

@Override
public CompletableFuture<Void> createStream(String logName, Duration timeout) {
    ensureRunning();
    int id;
    synchronized (this.internalIds) {
        if (this.internalIds.containsKey(logName)) {
            return Futures.failedFuture(new StreamSegmentExistsException(logName));
        }
        id = this.internalIds.size();
        this.internalIds.put(logName, id);
    }
    return CompletableFuture.runAsync(() -> {
        DurableDataLog log = null;
        boolean success = false;
        try {
            log = this.logFactory.createDurableDataLog(id);
            this.logs.put(logName, log);
            log.initialize(timeout);
            success = true;
        } catch (DurableDataLogException ex) {
            throw new CompletionException(ex);
        } finally {
            if (!success) {
                this.logs.remove(logName);
                synchronized (this.internalIds) {
                    this.internalIds.remove(logName);
                }
                if (log != null) {
                    log.close();
                }
            }
        }
    }, this.executor);
}