Examples with ChunkedSegmentStorage io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage used on opensource projects

Example 6 with ChunkedSegmentStorage

use of io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage in project pravega by pravega.

the class StreamSegmentContainer method initializeStorage.

/**
 * Initializes storage.
 *
 * @throws Exception
 */
private CompletableFuture<Void> initializeStorage() {
    log.info("{}: Initializing storage.", this.traceObjectId);
    this.storage.initialize(this.metadata.getContainerEpoch());
    if (this.storage instanceof ChunkedSegmentStorage) {
        ChunkedSegmentStorage chunkedSegmentStorage = (ChunkedSegmentStorage) this.storage;
        val snapshotInfoStore = getStorageSnapshotInfoStore();
        // Bootstrap
        StorageEventProcessor eventProcessor = new StorageEventProcessor(this.metadata.getContainerId(), this.containerEventProcessor, batch -> chunkedSegmentStorage.getGarbageCollector().processBatch(batch), chunkedSegmentStorage.getConfig().getGarbageCollectionMaxConcurrency());
        return chunkedSegmentStorage.bootstrap(snapshotInfoStore, eventProcessor);
    }
    return CompletableFuture.completedFuture(null);
}

Example 7 with ChunkedSegmentStorage

use of io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage in project pravega by pravega.

the class StreamSegmentContainer method getExtendedChunkInfo.

@SneakyThrows
@Override
public CompletableFuture<List<ExtendedChunkInfo>> getExtendedChunkInfo(String streamSegmentName, Duration timeout) {
    val chunkedSegmentStorage = (ChunkedSegmentStorage) storage;
    UtilsWrapper wrapper = new UtilsWrapper(chunkedSegmentStorage, BUFFER_SIZE, timeout);
    return wrapper.getExtendedChunkInfoList(streamSegmentName, true);
}

Example 8 with ChunkedSegmentStorage

use of io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage in project pravega by pravega.

the class StorageFactoryTests method testExtendedS3StorageFactoryCreator.

@Test
public void testExtendedS3StorageFactoryCreator() {
    StorageFactoryCreator factoryCreator = new ExtendedS3StorageFactoryCreator();
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("EXTENDEDS3").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build(), StorageFactoryInfo.builder().name("EXTENDEDS3").storageLayoutType(StorageLayoutType.ROLLING_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(2, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    val config = ExtendedS3StorageConfig.builder().with(ExtendedS3StorageConfig.CONFIGURI, "http://127.0.0.1?identity=x&secretKey=x").with(ExtendedS3StorageConfig.BUCKET, "bucket").with(ExtendedS3StorageConfig.PREFIX, "samplePrefix").build();
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, config);
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executorService());
    Assert.assertTrue(factory1 instanceof ExtendedS3SimpleStorageFactory);
    @Cleanup Storage storage1 = ((ExtendedS3SimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    Assert.assertTrue(((ChunkedSegmentStorage) storage1).getChunkStorage() instanceof ExtendedS3ChunkStorage);
    // Legacy Storage
    ConfigSetup configSetup2 = mock(ConfigSetup.class);
    when(configSetup2.getConfig(any())).thenReturn(config);
    val factory2 = factoryCreator.createFactory(expected[1], configSetup2, executorService());
    Assert.assertTrue(factory2 instanceof ExtendedS3StorageFactory);
    @Cleanup Storage storage2 = factory2.createStorageAdapter();
    Assert.assertTrue(storage2 instanceof AsyncStorageWrapper);
    @Cleanup SyncStorage syncStorage = factory2.createSyncStorage();
    Assert.assertNotNull(syncStorage);
    AssertExtensions.assertThrows("createStorageAdapter should throw UnsupportedOperationException.", () -> factory1.createStorageAdapter(), ex -> ex instanceof UnsupportedOperationException);
}

Example 9 with ChunkedSegmentStorage

use of io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage in project pravega by pravega.

the class StorageFactoryTests method testFileSystemStorageFactoryCreator.

@Test
public void testFileSystemStorageFactoryCreator() {
    StorageFactoryCreator factoryCreator = new FileSystemStorageFactoryCreator();
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("FILESYSTEM").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build(), StorageFactoryInfo.builder().name("FILESYSTEM").storageLayoutType(StorageLayoutType.ROLLING_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(2, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, FileSystemStorageConfig.builder().build());
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executorService());
    Assert.assertTrue(factory1 instanceof FileSystemSimpleStorageFactory);
    @Cleanup Storage storage1 = ((FileSystemSimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    Assert.assertTrue(((ChunkedSegmentStorage) storage1).getChunkStorage() instanceof FileSystemChunkStorage);
    // Legacy Storage
    ConfigSetup configSetup2 = mock(ConfigSetup.class);
    when(configSetup2.getConfig(any())).thenReturn(FileSystemStorageConfig.builder().build());
    val factory2 = factoryCreator.createFactory(expected[1], configSetup2, executorService());
    Assert.assertTrue(factory2 instanceof FileSystemStorageFactory);
    Storage storage2 = factory2.createStorageAdapter();
    Assert.assertTrue(storage2 instanceof AsyncStorageWrapper);
    @Cleanup SyncStorage syncStorage = factory2.createSyncStorage();
    Assert.assertNotNull(syncStorage);
    AssertExtensions.assertThrows("createStorageAdapter should throw UnsupportedOperationException.", () -> factory1.createStorageAdapter(), ex -> ex instanceof UnsupportedOperationException);
}

Example 10 with ChunkedSegmentStorage

use of io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage in project pravega by pravega.

the class InMemorySimpleStorageFactoryTests method testInMemorySimpleStorageFactoryCreator.

@Test
public void testInMemorySimpleStorageFactoryCreator() {
    StorageFactoryCreator factoryCreator = new InMemoryStorageFactoryCreator();
    @Cleanup("shutdownNow") val executor = ExecutorServiceHelpers.newScheduledThreadPool(1, "test");
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("INMEMORY").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build(), StorageFactoryInfo.builder().name("INMEMORY").storageLayoutType(StorageLayoutType.ROLLING_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(2, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG);
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executor);
    Assert.assertTrue(factory1 instanceof InMemorySimpleStorageFactory);
    @Cleanup Storage storage1 = ((InMemorySimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executor));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    // Legacy Storage
    ConfigSetup configSetup2 = mock(ConfigSetup.class);
    when(configSetup2.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG);
    val factory2 = factoryCreator.createFactory(expected[1], configSetup2, executor);
    Assert.assertTrue(factory2 instanceof InMemoryStorageFactory);
    Storage storage2 = factory2.createStorageAdapter();
    Assert.assertTrue(storage2 instanceof AsyncStorageWrapper);
    SyncStorage syncStorage = factory2.createSyncStorage();
    Assert.assertNotNull(syncStorage);
}