Examples with RollingStorage io.pravega.segmentstore.storage.rolling.RollingStorage used on opensource projects

Example 1 with RollingStorage

use of io.pravega.segmentstore.storage.rolling.RollingStorage in project pravega by pravega.

the class ExtendedS3StorageFactory method createStorageAdapter.

@Override
public Storage createStorageAdapter() {
    S3Config s3Config = new S3Config(config.getUrl()).withIdentity(config.getAccessKey()).withSecretKey(config.getSecretKey()).withNamespace(config.getNamespace());
    S3JerseyClient client = new S3JerseyClient(s3Config);
    ExtendedS3Storage s = new ExtendedS3Storage(client, this.config);
    return new AsyncStorageWrapper(new RollingStorage(s), this.executor);
}

Example 2 with RollingStorage

use of io.pravega.segmentstore.storage.rolling.RollingStorage in project pravega by pravega.

the class RestoreBackUpDataRecoveryTest method testDurableDataLogFailRecoveryWatermarking.

/**
 * Tests the data recovery scenario with watermarking events.
 *  What test does, step by step:
 *  1. Starts Pravega locally with just 4 segment containers.
 *  2. Writes {@link #TOTAL_NUM_EVENTS} events to a segment with watermarks.
 *  3. Waits for all segments created to be flushed to the long term storage.
 *  4. Shuts down the controller, segment store and bookeeper/zookeeper.
 *  5. Creates back up of container metadata segment and its attribute segment before deleting them from the Long Term Storage .
 *  6. Starts 4 debug segment containers using a new bookeeper/zookeeper and the Long Term Storage.
 *  7. Re-creates the container metadata segments in DurableLog and lets them to be flushed to the Long Term Storage.
 *  8. Starts segment store and controller.
 *  9. Read all events and verify that all events are below the bounds.
 * @throws Exception    In case of an exception occurred while execution.
 */
@Test(timeout = 180000)
public void testDurableDataLogFailRecoveryWatermarking() throws Exception {
    int instanceId = 0;
    int bookieCount = 1;
    int containerCount = 4;
    String readerGroup = "rgTx";
    // Creating a long term storage only once here.
    this.storageFactory = new InMemoryStorageFactory(executorService());
    log.info("Created a long term storage.");
    // Start a new BK & ZK, segment store and controller
    @Cleanup PravegaRunner pravegaRunner = new PravegaRunner(instanceId++, bookieCount, containerCount, this.storageFactory);
    // Create a scope and a stream
    createScopeStream(pravegaRunner.controllerRunner.controller, SCOPE, STREAM1);
    // Create a client to write events.
    @Cleanup ClientRunner clientRunner = new ClientRunner(pravegaRunner.controllerRunner);
    // Create a writer
    @Cleanup TransactionalEventStreamWriter<Long> writer = clientRunner.clientFactory.createTransactionalEventWriter("writer1", STREAM1, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(TRANSACTION_TIMEOUT.toMillis()).build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    CompletableFuture<Void> writerFuture = writeTxEvents(writer, stopFlag);
    // scale the stream several times so that we get complex positions
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    Stream streamObj = Stream.of(SCOPE, STREAM1);
    scale(pravegaRunner.controllerRunner.controller, streamObj, config);
    // get watermarks
    LinkedBlockingQueue<Watermark> watermarks = getWatermarks(pravegaRunner, stopFlag, writerFuture);
    // Shut down the controller
    pravegaRunner.controllerRunner.close();
    // Flush DurableLog to Long Term Storage
    flushToStorage(pravegaRunner.segmentStoreRunner.serviceBuilder);
    // Shutdown SegmentStore
    pravegaRunner.segmentStoreRunner.close();
    // Shutdown BookKeeper & ZooKeeper
    pravegaRunner.bookKeeperRunner.close();
    log.info("SegmentStore, BookKeeper & ZooKeeper shutdown");
    // Get the long term storage from the running pravega instance
    @Cleanup Storage storage = new AsyncStorageWrapper(new RollingStorage(this.storageFactory.createSyncStorage(), new SegmentRollingPolicy(DEFAULT_ROLLING_SIZE)), executorService());
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).join();
    // start a new BookKeeper and ZooKeeper.
    pravegaRunner.bookKeeperRunner = new BookKeeperRunner(instanceId++, bookieCount);
    createBookKeeperLogFactory();
    log.info("Started a new BookKeeper and ZooKeeper.");
    // Recover segments
    runRecovery(containerCount, storage, backUpMetadataSegments);
    // Start a new segment store and controller
    pravegaRunner.restartControllerAndSegmentStore(this.storageFactory, this.dataLogFactory);
    log.info("Started segment store and controller again.");
    // Create the client with new controller.
    @Cleanup ClientRunner newClientRunner = new ClientRunner(pravegaRunner.controllerRunner);
    // read events and verify
    readVerifyEventsWithWatermarks(readerGroup, newClientRunner, streamObj, watermarks);
}

Example 3 with RollingStorage

use of io.pravega.segmentstore.storage.rolling.RollingStorage in project pravega by pravega.

the class RestoreBackUpDataRecoveryTest method testRecovery.

/**
 * Performs a data recovery test(with all segments flushed to the long-term storage beforehand) using the given parameters.
 * @param containerCount    The number of containers to be used in the pravega instance and the number of debug segment
 *                          containers that will be started.
 * @param withTransaction   A boolean to indicate weather to write events in the form of transactions or not.
 * @throws Exception        In case of an exception occurred while execution.
 */
private void testRecovery(int containerCount, int bookieCount, boolean withTransaction) throws Exception {
    int instanceId = 0;
    // Creating a long term storage only once here.
    this.storageFactory = new InMemoryStorageFactory(executorService());
    log.info("Created a long term storage.");
    // Start a new BK & ZK, segment store and controller
    @Cleanup PravegaRunner pravegaRunner = new PravegaRunner(instanceId++, bookieCount, containerCount, this.storageFactory);
    // Create a stream for writing data
    createScopeStream(pravegaRunner.controllerRunner.controller, SCOPE, STREAM1);
    log.info("Created stream '{}'.", STREAM1);
    // Create a client to write events.
    try (val clientRunner = new ClientRunner(pravegaRunner.controllerRunner)) {
        // Write events.
        writeEventsToStream(clientRunner.clientFactory, withTransaction);
    }
    // Shut down the controller
    pravegaRunner.controllerRunner.close();
    // Flush DurableLog to Long Term Storage
    flushToStorage(pravegaRunner.segmentStoreRunner.serviceBuilder);
    // Shutdown SegmentStore
    pravegaRunner.segmentStoreRunner.close();
    // Shutdown BookKeeper & ZooKeeper
    pravegaRunner.bookKeeperRunner.close();
    log.info("SegmentStore, BookKeeper & ZooKeeper shutdown");
    // Get the long term storage from the running pravega instance
    @Cleanup Storage storage = new AsyncStorageWrapper(new RollingStorage(this.storageFactory.createSyncStorage(), new SegmentRollingPolicy(DEFAULT_ROLLING_SIZE)), executorService());
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // start a new BookKeeper and ZooKeeper.
    pravegaRunner.bookKeeperRunner = new BookKeeperRunner(instanceId++, bookieCount);
    createBookKeeperLogFactory();
    log.info("Started a new BookKeeper and ZooKeeper.");
    // Recover segments
    runRecovery(containerCount, storage, backUpMetadataSegments);
    // Start a new segment store and controller
    pravegaRunner.restartControllerAndSegmentStore(this.storageFactory, this.dataLogFactory);
    log.info("Started segment store and controller again.");
    // Create the client with new controller.
    try (val clientRunner = new ClientRunner(pravegaRunner.controllerRunner)) {
        // Try reading all events again to verify that the recovery was successful.
        readEventsFromStream(clientRunner.clientFactory, clientRunner.readerGroupManager);
        log.info("Read all events again to verify that segments were recovered.");
    }
}

Example 4 with RollingStorage

use of io.pravega.segmentstore.storage.rolling.RollingStorage in project pravega by pravega.

the class RestoreBackUpDataRecoveryTest method testDurableDataLogFailRecoveryReadersPaused.

/**
 * Tests the data recovery scenario with readers stalling while reading. Readers read some events and then they are
 * stopped. Durable data log is erased and restored. It's validated that readers are able to read rest of the unread
 * events.
 *  What test does, step by step:
 *  1. Starts Pravega locally with just 4 segment containers.
 *  2. Writes {@link #TOTAL_NUM_EVENTS} events.
 *  3. Waits for all segments created to be flushed to the long term storage.
 *  4. Let a reader read N number of events.
 *  5. Shuts down the controller, segment store and bookeeper/zookeeper.
 *  6. Creates back up of container metadata segment and its attribute segment before deleting them from the Long Term Storage .
 *  7. Starts 4 debug segment containers using a new bookeeper/zookeeper and the Long Term Storage.
 *  8. Re-creates the container metadata segments in DurableLog and lets them to be flushed to the Long Term Storage.
 *  9. Updates core attributes of segments in the new container metadata segment by using details from the back up of old container metadata segment.
 *  10. Starts segment store and controller.
 *  11. Let the reader read rest of the 10-N number of events.
 * @throws Exception    In case of an exception occurred while execution.
 */
@Test(timeout = 180000)
public void testDurableDataLogFailRecoveryReadersPaused() throws Exception {
    int instanceId = 0;
    int bookieCount = 1;
    int containerCount = 4;
    int eventsReadCount = RANDOM.nextInt(TOTAL_NUM_EVENTS);
    String testReader = "readerDRIntegrationTest";
    String testReaderGroup = "readerGroupDRIntegrationTest";
    // Creating a long term storage only once here.
    this.storageFactory = new InMemoryStorageFactory(executorService());
    log.info("Created a long term storage.");
    // Start a new BK & ZK, segment store and controller
    @Cleanup PravegaRunner pravegaRunner = new PravegaRunner(instanceId++, bookieCount, containerCount, this.storageFactory);
    // Create a stream for writing data
    createScopeStream(pravegaRunner.controllerRunner.controller, SCOPE, STREAM1);
    log.info("Created stream '{}'", STREAM1);
    // Create a client to write events.
    try (val clientRunner = new ClientRunner(pravegaRunner.controllerRunner)) {
        // Write events.
        writeEventsToStream(clientRunner.clientFactory, true);
        // Create a reader for reading from the stream.
        EventStreamReader<String> reader = createReader(clientRunner.clientFactory, clientRunner.readerGroupManager, SCOPE, STREAM1, testReaderGroup, testReader);
        // Let reader read N number of events and mark its position.
        Position p = readNEvents(reader, eventsReadCount);
        ReaderGroup readerGroup = clientRunner.readerGroupManager.getReaderGroup(testReaderGroup);
        readerGroup.readerOffline(testReader, p);
    }
    // Shut down the controller
    pravegaRunner.controllerRunner.close();
    // Flush DurableLog to Long Term Storage
    flushToStorage(pravegaRunner.segmentStoreRunner.serviceBuilder);
    // Shutdown SegmentStore
    pravegaRunner.segmentStoreRunner.close();
    // Shutdown BookKeeper & ZooKeeper
    pravegaRunner.bookKeeperRunner.close();
    log.info("SegmentStore, BookKeeper & ZooKeeper shutdown");
    // Get the long term storage from the running pravega instance
    @Cleanup Storage storage = new AsyncStorageWrapper(new RollingStorage(this.storageFactory.createSyncStorage(), new SegmentRollingPolicy(DEFAULT_ROLLING_SIZE)), executorService());
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).join();
    // start a new BookKeeper and ZooKeeper.
    pravegaRunner.bookKeeperRunner = new BookKeeperRunner(instanceId++, bookieCount);
    createBookKeeperLogFactory();
    log.info("Started a new BookKeeper and ZooKeeper.");
    // Recover segments
    runRecovery(containerCount, storage, backUpMetadataSegments);
    // Start a new segment store and controller
    pravegaRunner.restartControllerAndSegmentStore(this.storageFactory, this.dataLogFactory);
    log.info("Started segment store and controller again.");
    // Create the client with new controller.
    try (val clientRunner = new ClientRunner(pravegaRunner.controllerRunner)) {
        // Get reader group.
        ReaderGroup readerGroup = clientRunner.readerGroupManager.getReaderGroup(testReaderGroup);
        assertNotNull(readerGroup);
        EventStreamReader<String> reader = clientRunner.clientFactory.createReader(testReader, testReaderGroup, new UTF8StringSerializer(), ReaderConfig.builder().build());
        // Read the remaining number of events.
        readNEvents(reader, TOTAL_NUM_EVENTS - eventsReadCount);
        // Reading next event should return null.
        assertNull(reader.readNextEvent(READ_TIMEOUT.toMillis()).getEvent());
        reader.close();
    }
}

Example 5 with RollingStorage

use of io.pravega.segmentstore.storage.rolling.RollingStorage in project pravega by pravega.

the class StorageTestBase method testListSegmentsWithDeletes.

/**
 * Tests listSegments() on deleting some segments.
 * @throws Exception if an unexpected error occurred.
 */
@Test
public void testListSegmentsWithDeletes() throws Exception {
    @Cleanup val baseStorage = new InMemoryStorage();
    @Cleanup val s = new RollingStorage(baseStorage, new SegmentRollingPolicy(1));
    s.initialize(DEFAULT_EPOCH);
    Set<String> deletedSegments = new HashSet<>();
    int expectedCount = 50;
    for (int i = 0; i < expectedCount; i++) {
        String segmentName = "segment-" + i;
        SegmentHandle handle = s.create(segmentName);
        if (rnd.nextInt(2) == 1) {
            s.delete(handle);
            deletedSegments.add(segmentName);
        }
    }
    Iterator<SegmentProperties> it = s.listSegments();
    expectedCount -= deletedSegments.size();
    Assert.assertEquals(expectedCount, Iterators.size(it));
}