Examples with OperationLog io.pravega.segmentstore.server.OperationLog used on opensource projects

Example 1 with OperationLog

use of io.pravega.segmentstore.server.OperationLog in project pravega by pravega.

the class StreamSegmentContainerTests method testStartFailure.

/**
 * Tests the behavior when there is a startup failure (i.e., already started services need to shut down.
 */
@Test
public void testStartFailure() throws Exception {
    final Duration shutdownTimeout = Duration.ofSeconds(5);
    @Cleanup val context = new TestContext();
    val failedWriterFactory = new FailedWriterFactory();
    AtomicReference<OperationLog> log = new AtomicReference<>();
    val watchableDurableLogFactory = new WatchableOperationLogFactory(context.operationLogFactory, log::set);
    val containerFactory = new StreamSegmentContainerFactory(DEFAULT_CONFIG, watchableDurableLogFactory, context.readIndexFactory, failedWriterFactory, context.storageFactory, executorService());
    val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID);
    container.startAsync();
    // Wait for the container to be shut down and verify it is failed.
    ServiceListeners.awaitShutdown(container, shutdownTimeout, false);
    Assert.assertEquals("Container is not in a failed state after a failed startup.", Service.State.FAILED, container.state());
    Throwable actualException = Exceptions.unwrap(container.failureCause());
    boolean exceptionMatch = actualException instanceof IntentionalException;
    if (!exceptionMatch) {
        Assert.fail(String.format("Container did not fail with the correct exception. Expected '%s', Actual '%s'.", IntentionalException.class.getSimpleName(), actualException));
    }
    // Verify the OperationLog is also shut down, and make sure it is not in a Failed state.
    ServiceListeners.awaitShutdown(log.get(), shutdownTimeout, true);
}

Example 2 with OperationLog

use of io.pravega.segmentstore.server.OperationLog in project pravega by pravega.

the class OperationLogTestBase method createStreamSegmentsWithOperations.

/**
 * Creates a number of StreamSegments in the given Metadata and OperationLog.
 */
HashSet<Long> createStreamSegmentsWithOperations(int streamSegmentCount, ContainerMetadata containerMetadata, OperationLog durableLog, Storage storage) {
    StreamSegmentMapper mapper = new StreamSegmentMapper(containerMetadata, durableLog, new InMemoryStateStore(), NO_OP_METADATA_CLEANUP, storage, ForkJoinPool.commonPool());
    HashSet<Long> result = new HashSet<>();
    for (int i = 0; i < streamSegmentCount; i++) {
        String name = getStreamSegmentName(i);
        long streamSegmentId = mapper.createNewStreamSegment(name, null, Duration.ZERO).thenCompose(v -> mapper.getOrAssignStreamSegmentId(name, Duration.ZERO)).join();
        result.add(streamSegmentId);
    }
    return result;
}

Example 3 with OperationLog

use of io.pravega.segmentstore.server.OperationLog in project pravega by pravega.

the class OperationLogTestBase method createTransactionsWithOperations.

/**
 * Creates a number of Transaction Segments in the given Metadata and OperationLog.
 */
AbstractMap<Long, Long> createTransactionsWithOperations(HashSet<Long> streamSegmentIds, int transactionsPerStreamSegment, ContainerMetadata containerMetadata, OperationLog durableLog, Storage storage) {
    HashMap<Long, Long> result = new HashMap<>();
    StreamSegmentMapper mapper = new StreamSegmentMapper(containerMetadata, durableLog, new InMemoryStateStore(), NO_OP_METADATA_CLEANUP, storage, ForkJoinPool.commonPool());
    for (long streamSegmentId : streamSegmentIds) {
        String streamSegmentName = containerMetadata.getStreamSegmentMetadata(streamSegmentId).getName();
        for (int i = 0; i < transactionsPerStreamSegment; i++) {
            long transactionId = mapper.createNewTransactionStreamSegment(streamSegmentName, UUID.randomUUID(), null, Duration.ZERO).thenCompose(v -> mapper.getOrAssignStreamSegmentId(v, Duration.ZERO)).join();
            result.put(transactionId, streamSegmentId);
        }
    }
    return result;
}

Example 4 with OperationLog

use of io.pravega.segmentstore.server.OperationLog in project pravega by pravega.

the class StreamSegmentContainerTests method testStartOffline.

/**
 * Tests the ability of the StreamSegmentContainer to start in Offline mode (due to an offline DurableLog) and eventually
 * become online when the DurableLog becomes too.
 */
@Test
public void testStartOffline() throws Exception {
    @Cleanup val context = new TestContext();
    AtomicReference<DurableDataLog> dataLog = new AtomicReference<>();
    @Cleanup val dataLogFactory = new TestDurableDataLogFactory(context.dataLogFactory, dataLog::set);
    AtomicReference<OperationLog> durableLog = new AtomicReference<>();
    val durableLogFactory = new WatchableOperationLogFactory(new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, dataLogFactory, executorService()), durableLog::set);
    val containerFactory = new StreamSegmentContainerFactory(DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.writerFactory, context.storageFactory, executorService());
    // Write some data
    ArrayList<String> segmentNames = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
        for (int i = 0; i < SEGMENT_COUNT; i++) {
            String segmentName = getSegmentName(i);
            segmentNames.add(segmentName);
            opFutures.add(container.createStreamSegment(segmentName, null, TIMEOUT));
        }
        for (int i = 0; i < APPENDS_PER_SEGMENT / 2; i++) {
            for (String segmentName : segmentNames) {
                byte[] appendData = getAppendData(segmentName, i);
                opFutures.add(container.append(segmentName, appendData, null, TIMEOUT));
                lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
                recordAppend(segmentName, appendData, segmentContents);
            }
        }
        Futures.allOf(opFutures).join();
        // Disable the DurableDataLog.
        dataLog.get().disable();
    }
    // Start in "Offline" mode, verify operations cannot execute and then shut down - make sure we can shut down an offline container.
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        Assert.assertTrue("Expecting Segment Container to be offline.", container.isOffline());
        AssertExtensions.assertThrows("append() worked in offline mode.", () -> container.append("foo", new byte[1], null, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("getStreamSegmentInfo() worked in offline mode.", () -> container.getStreamSegmentInfo("foo", false, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        AssertExtensions.assertThrows("read() worked in offline mode.", () -> container.read("foo", 0, 1, TIMEOUT), ex -> ex instanceof ContainerOfflineException);
        container.stopAsync().awaitTerminated();
    }
    // Start in "Offline" mode and verify we can resume a normal startup.
    try (val container = containerFactory.createStreamSegmentContainer(CONTAINER_ID)) {
        container.startAsync().awaitRunning();
        Assert.assertTrue("Expecting Segment Container to be offline.", container.isOffline());
        dataLog.get().enable();
        // Wait for the DurableLog to become online.
        durableLog.get().awaitOnline().get(DEFAULT_DURABLE_LOG_CONFIG.getStartRetryDelay().toMillis() * 100, TimeUnit.MILLISECONDS);
        // Verify we can execute regular operations now.
        ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
        for (int i = 0; i < APPENDS_PER_SEGMENT / 2; i++) {
            for (String segmentName : segmentNames) {
                byte[] appendData = getAppendData(segmentName, i);
                opFutures.add(container.append(segmentName, appendData, null, TIMEOUT));
                lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
                recordAppend(segmentName, appendData, segmentContents);
            }
        }
        Futures.allOf(opFutures).join();
        // Verify all operations arrived in Storage.
        ArrayList<CompletableFuture<Void>> segmentsCompletion = new ArrayList<>();
        for (String segmentName : segmentNames) {
            SegmentProperties sp = container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
            segmentsCompletion.add(waitForSegmentInStorage(sp, context));
        }
        Futures.allOf(segmentsCompletion).join();
        container.stopAsync().awaitTerminated();
    }
}