Examples with RollingStorage io.pravega.segmentstore.storage.rolling.RollingStorage used on opensource projects

Example 6 with RollingStorage

use of io.pravega.segmentstore.storage.rolling.RollingStorage in project pravega by pravega.

the class DebugStreamSegmentContainerTests method testDataRecoveryStorageLevel.

/**
 * Use a storage instance to create segments. Lists the segments from the storage and and then recreates them using
 * debug segment containers. Before re-creating(or registering), the segments are mapped to their respective debug
 * segment container. Once registered, segment's properties are matched to verify if the test was successful or not.
 */
@Test
public void testDataRecoveryStorageLevel() throws Exception {
    // Segments are mapped to four different containers.
    int containerCount = 4;
    int segmentsToCreateCount = 50;
    // Create a storage.
    @Cleanup val baseStorage = new InMemoryStorage();
    @Cleanup val s = new RollingStorage(baseStorage, new SegmentRollingPolicy(1));
    s.initialize(1);
    log.info("Created a storage instance");
    // Record details(name, container Id & sealed status) of each segment to be created.
    Set<String> sealedSegments = new HashSet<>();
    byte[] data = "data".getBytes();
    SegmentToContainerMapper segToConMapper = new SegmentToContainerMapper(containerCount, true);
    Map<Integer, ArrayList<String>> segmentByContainers = new HashMap<>();
    // Create segments and get their container Ids, sealed status and names to verify.
    for (int i = 0; i < segmentsToCreateCount; i++) {
        String segmentName = "segment-" + RANDOM.nextInt();
        // Use segmentName to map to different containers.
        int containerId = segToConMapper.getContainerId(segmentName);
        ArrayList<String> segmentsList = segmentByContainers.get(containerId);
        if (segmentsList == null) {
            segmentsList = new ArrayList<>();
            segmentByContainers.put(containerId, segmentsList);
        }
        segmentByContainers.get(containerId).add(segmentName);
        // Create segments, write data and randomly seal some of them.
        val wh1 = s.create(segmentName);
        // Write data.
        s.write(wh1, 0, new ByteArrayInputStream(data), data.length);
        if (RANDOM.nextBoolean()) {
            s.seal(wh1);
            sealedSegments.add(segmentName);
        }
    }
    log.info("Created some segments using the storage.");
    @Cleanup TestContext context = createContext(executorService());
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainerMap = new HashMap<>();
    log.info("Start a debug segment container corresponding to each container id.");
    for (int containerId = 0; containerId < containerCount; containerId++) {
        MetadataCleanupContainer localContainer = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
        Services.startAsync(localContainer, executorService()).join();
        debugStreamSegmentContainerMap.put(containerId, localContainer);
    }
    log.info("Recover all segments using the storage and debug segment containers.");
    recoverAllSegments(new AsyncStorageWrapper(s, executorService()), debugStreamSegmentContainerMap, executorService(), TIMEOUT);
    // Re-create all segments which were listed.
    for (int containerId = 0; containerId < containerCount; containerId++) {
        for (String segment : segmentByContainers.get(containerId)) {
            SegmentProperties props = debugStreamSegmentContainerMap.get(containerId).getStreamSegmentInfo(segment, TIMEOUT).join();
            Assert.assertEquals("Segment length mismatch.", data.length, props.getLength());
            Assert.assertEquals("Sealed status of the segment don't match.", sealedSegments.contains(segment), props.isSealed());
        }
        debugStreamSegmentContainerMap.get(containerId).close();
    }
}

Example 7 with RollingStorage

use of io.pravega.segmentstore.storage.rolling.RollingStorage in project pravega by pravega.

the class StorageTestBase method testListSegments.

/**
 * Tests the ability to list Segments.
 * @throws Exception if an unexpected error occurred.
 */
@Test
public void testListSegments() throws Exception {
    @Cleanup val baseStorage = new InMemoryStorage();
    @Cleanup val s = new RollingStorage(baseStorage, new SegmentRollingPolicy(1));
    Set<String> sealedSegments = new HashSet<>();
    s.initialize(1);
    int expectedCount = 50;
    byte[] data = "data".getBytes();
    for (int i = 0; i < expectedCount; i++) {
        String segmentName = "segment-" + i;
        val wh1 = s.create(segmentName);
        // Write data.
        s.write(wh1, 0, new ByteArrayInputStream(data), data.length);
        if (rnd.nextInt(2) == 1) {
            s.seal(wh1);
            sealedSegments.add(segmentName);
        }
    }
    Iterator<SegmentProperties> it = s.listSegments();
    int actualCount = 0;
    while (it.hasNext()) {
        SegmentProperties curr = it.next();
        // check the length matches
        Assert.assertEquals(curr.getLength(), data.length);
        if (sealedSegments.contains(curr.getName())) {
            Assert.assertTrue(curr.isSealed());
        } else {
            Assert.assertFalse(curr.isSealed());
        }
        ++actualCount;
    }
    Assert.assertEquals(actualCount, expectedCount);
}