Examples with DurableDataLog io.pravega.segmentstore.storage.DurableDataLog used on opensource projects

Example 16 with DurableDataLog

use of io.pravega.segmentstore.storage.DurableDataLog in project pravega by pravega.

the class StreamSegmentContainerTests method testStartOffline.

/**
 * Tests the ability of the StreamSegmentContainer to start in Offline mode (due to an offline DurableLog) and eventually
 * become online when the DurableLog becomes too.
 */
@Test
public void testStartOffline() throws Exception {
    @Cleanup val context = createContext();
    AtomicReference<DurableDataLog> dataLog = new AtomicReference<>();
    @Cleanup val dataLogFactory = new TestDurableDataLogFactory(context.dataLogFactory, dataLog::set);
    AtomicReference<OperationLog> durableLog = new AtomicReference<>();
    val durableLogFactory = new WatchableOperationLogFactory(new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, dataLogFactory, executorService()), durableLog::set);
    val containerFactory = new StreamSegmentContainerFactory(DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    // Write some data
    ArrayList<String> segmentNames = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        val creationFutures = new ArrayList<CompletableFuture<Void>>();
        for (int i = 0; i < SEGMENT_COUNT; i++) {
            String segmentName = getSegmentName(i);
            segmentNames.add(segmentName);
            creationFutures.add(container.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT));
        }
        // Wait for the segments to be created first.
        Futures.allOf(creationFutures).join();
        val opFutures = new ArrayList<CompletableFuture<Long>>();
        for (int i = 0; i < APPENDS_PER_SEGMENT / 2; i++) {
            for (String segmentName : segmentNames) {
                RefCountByteArraySegment appendData = getAppendData(segmentName, i);
                opFutures.add(container.append(segmentName, appendData, null, TIMEOUT));
                lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.getLength());
                recordAppend(segmentName, appendData, segmentContents, null);
            }
        }
        Futures.allOf(opFutures).join();
        // Disable the DurableDataLog.
        dataLog.get().disable();
    }
    // Start in "Offline" mode, verify operations cannot execute and then shut down - make sure we can shut down an offline container.
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        Assert.assertTrue("Expecting Segment Container to be offline.", container.isOffline());
        AssertExtensions.assertSuppliedFutureThrows("append() worked in offline mode.", () -> container.append("foo", new ByteArraySegment(new byte[1]), null, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertSuppliedFutureThrows("getStreamSegmentInfo() worked in offline mode.", () -> container.getStreamSegmentInfo("foo", TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertSuppliedFutureThrows("read() worked in offline mode.", () -> container.read("foo", 0, 1, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        container.stopAsync().awaitTerminated();
    }
    // Start in "Offline" mode and verify we can resume a normal startup.
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        Assert.assertTrue("Expecting Segment Container to be offline.", container.isOffline());
        dataLog.get().enable();
        // Wait for the DurableLog to become online.
        durableLog.get().awaitOnline().get(DEFAULT_DURABLE_LOG_CONFIG.getStartRetryDelay().toMillis() * 100, TimeUnit.MILLISECONDS);
        // Verify we can execute regular operations now.
        ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
        for (int i = 0; i < APPENDS_PER_SEGMENT / 2; i++) {
            for (String segmentName : segmentNames) {
                RefCountByteArraySegment appendData = getAppendData(segmentName, i);
                opFutures.add(Futures.toVoid(container.append(segmentName, appendData, null, TIMEOUT)));
                lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.getLength());
                recordAppend(segmentName, appendData, segmentContents, null);
            }
        }
        Futures.allOf(opFutures).join();
        // Verify all operations arrived in Storage.
        ArrayList<CompletableFuture<Void>> segmentsCompletion = new ArrayList<>();
        for (String segmentName : segmentNames) {
            SegmentProperties sp = container.getStreamSegmentInfo(segmentName, TIMEOUT).join();
            segmentsCompletion.add(waitForSegmentInStorage(sp, context));
        }
        Futures.allOf(segmentsCompletion).join();
        container.stopAsync().awaitTerminated();
    }
}