Search in sources :

Example 1 with StorageFactoryCreator

use of io.pravega.segmentstore.storage.StorageFactoryCreator in project pravega by pravega.

the class StorageFactoryTests method testS3StorageFactoryCreator.

private void testS3StorageFactoryCreator(S3StorageConfig config) {
    StorageFactoryCreator factoryCreator = new S3StorageFactoryCreator();
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("S3").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(1, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, config);
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executorService());
    Assert.assertTrue(factory1 instanceof S3SimpleStorageFactory);
    @Cleanup Storage storage1 = ((S3SimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    Assert.assertTrue(((ChunkedSegmentStorage) storage1).getChunkStorage() instanceof S3ChunkStorage);
    AssertExtensions.assertThrows("createStorageAdapter should throw UnsupportedOperationException.", () -> factory1.createStorageAdapter(), ex -> ex instanceof UnsupportedOperationException);
}
Also used : lombok.val(lombok.val) Storage(io.pravega.segmentstore.storage.Storage) ChunkedSegmentStorage(io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage) ExtendedS3ChunkStorage(io.pravega.storage.extendeds3.ExtendedS3ChunkStorage) HDFSChunkStorage(io.pravega.storage.hdfs.HDFSChunkStorage) S3ChunkStorage(io.pravega.storage.s3.S3ChunkStorage) SyncStorage(io.pravega.segmentstore.storage.SyncStorage) FileSystemChunkStorage(io.pravega.storage.filesystem.FileSystemChunkStorage) ExtendedS3SimpleStorageFactory(io.pravega.storage.extendeds3.ExtendedS3SimpleStorageFactory) S3SimpleStorageFactory(io.pravega.storage.s3.S3SimpleStorageFactory) ExtendedS3ChunkStorage(io.pravega.storage.extendeds3.ExtendedS3ChunkStorage) S3ChunkStorage(io.pravega.storage.s3.S3ChunkStorage) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) StorageFactoryInfo(io.pravega.segmentstore.storage.StorageFactoryInfo) ChunkedSegmentStorage(io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage) S3StorageFactoryCreator(io.pravega.storage.s3.S3StorageFactoryCreator) HDFSStorageFactoryCreator(io.pravega.storage.hdfs.HDFSStorageFactoryCreator) StorageFactoryCreator(io.pravega.segmentstore.storage.StorageFactoryCreator) ExtendedS3StorageFactoryCreator(io.pravega.storage.extendeds3.ExtendedS3StorageFactoryCreator) FileSystemStorageFactoryCreator(io.pravega.storage.filesystem.FileSystemStorageFactoryCreator) S3StorageFactoryCreator(io.pravega.storage.s3.S3StorageFactoryCreator) ExtendedS3StorageFactoryCreator(io.pravega.storage.extendeds3.ExtendedS3StorageFactoryCreator) Cleanup(lombok.Cleanup) ConfigSetup(io.pravega.segmentstore.storage.ConfigSetup)

Example 2 with StorageFactoryCreator

use of io.pravega.segmentstore.storage.StorageFactoryCreator in project pravega by pravega.

the class StorageFactoryTests method testHDFSStorageFactoryCreator.

@Test
public void testHDFSStorageFactoryCreator() {
    StorageFactoryCreator factoryCreator = new HDFSStorageFactoryCreator();
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("HDFS").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build(), StorageFactoryInfo.builder().name("HDFS").storageLayoutType(StorageLayoutType.ROLLING_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(2, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, HDFSStorageConfig.builder().build());
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executorService());
    Assert.assertTrue(factory1 instanceof HDFSSimpleStorageFactory);
    @Cleanup Storage storage1 = ((HDFSSimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    Assert.assertTrue(((ChunkedSegmentStorage) storage1).getChunkStorage() instanceof HDFSChunkStorage);
    // Legacy Storage
    ConfigSetup configSetup2 = mock(ConfigSetup.class);
    when(configSetup2.getConfig(any())).thenReturn(HDFSStorageConfig.builder().build());
    val factory2 = factoryCreator.createFactory(expected[1], configSetup2, executorService());
    Assert.assertTrue(factory2 instanceof HDFSStorageFactory);
    @Cleanup Storage storage2 = factory2.createStorageAdapter();
    Assert.assertTrue(storage2 instanceof AsyncStorageWrapper);
    SyncStorage syncStorage = factory2.createSyncStorage();
    Assert.assertNotNull(syncStorage);
    AssertExtensions.assertThrows("createStorageAdapter should throw UnsupportedOperationException.", () -> factory1.createStorageAdapter(), ex -> ex instanceof UnsupportedOperationException);
}
Also used : lombok.val(lombok.val) SyncStorage(io.pravega.segmentstore.storage.SyncStorage) HDFSChunkStorage(io.pravega.storage.hdfs.HDFSChunkStorage) HDFSStorageFactoryCreator(io.pravega.storage.hdfs.HDFSStorageFactoryCreator) StorageFactoryInfo(io.pravega.segmentstore.storage.StorageFactoryInfo) ChunkedSegmentStorage(io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage) Cleanup(lombok.Cleanup) HDFSSimpleStorageFactory(io.pravega.storage.hdfs.HDFSSimpleStorageFactory) HDFSStorageFactory(io.pravega.storage.hdfs.HDFSStorageFactory) Storage(io.pravega.segmentstore.storage.Storage) ChunkedSegmentStorage(io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage) ExtendedS3ChunkStorage(io.pravega.storage.extendeds3.ExtendedS3ChunkStorage) HDFSChunkStorage(io.pravega.storage.hdfs.HDFSChunkStorage) S3ChunkStorage(io.pravega.storage.s3.S3ChunkStorage) SyncStorage(io.pravega.segmentstore.storage.SyncStorage) FileSystemChunkStorage(io.pravega.storage.filesystem.FileSystemChunkStorage) AsyncStorageWrapper(io.pravega.segmentstore.storage.AsyncStorageWrapper) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) S3StorageFactoryCreator(io.pravega.storage.s3.S3StorageFactoryCreator) HDFSStorageFactoryCreator(io.pravega.storage.hdfs.HDFSStorageFactoryCreator) StorageFactoryCreator(io.pravega.segmentstore.storage.StorageFactoryCreator) ExtendedS3StorageFactoryCreator(io.pravega.storage.extendeds3.ExtendedS3StorageFactoryCreator) FileSystemStorageFactoryCreator(io.pravega.storage.filesystem.FileSystemStorageFactoryCreator) ConfigSetup(io.pravega.segmentstore.storage.ConfigSetup) Test(org.junit.Test)

Example 3 with StorageFactoryCreator

use of io.pravega.segmentstore.storage.StorageFactoryCreator in project pravega by pravega.

the class StorageLoader method load.

public StorageFactory load(ConfigSetup setup, String storageImplementation, StorageLayoutType storageLayoutType, ScheduledExecutorService executor) {
    ServiceLoader<StorageFactoryCreator> loader = ServiceLoader.load(StorageFactoryCreator.class);
    StorageExtraConfig noOpConfig = setup.getConfig(StorageExtraConfig::builder);
    for (StorageFactoryCreator factoryCreator : loader) {
        val factories = factoryCreator.getStorageFactories();
        for (val factoryInfo : factories) {
            log.info("Loading {}, trying {}", storageImplementation, factoryInfo);
            if (factoryInfo.getName().equals(storageImplementation) && factoryInfo.getStorageLayoutType() == storageLayoutType) {
                StorageFactory factory = factoryCreator.createFactory(factoryInfo, setup, executor);
                if (!noOpConfig.isStorageNoOpMode()) {
                    return factory;
                } else {
                    // The specified storage implementation is in No-Op mode.
                    log.warn("{} IS IN NO-OP MODE: DATA LOSS WILL HAPPEN! MAKE SURE IT IS BY FULL INTENTION FOR TESTING PURPOSE!", storageImplementation);
                    return new NoOpStorageFactory(noOpConfig, executor, factory, null);
                }
            }
        }
    }
    return null;
}
Also used : lombok.val(lombok.val) StorageFactory(io.pravega.segmentstore.storage.StorageFactory) NoOpStorageFactory(io.pravega.segmentstore.storage.noop.NoOpStorageFactory) NoOpStorageFactory(io.pravega.segmentstore.storage.noop.NoOpStorageFactory) StorageFactoryCreator(io.pravega.segmentstore.storage.StorageFactoryCreator) StorageExtraConfig(io.pravega.segmentstore.storage.noop.StorageExtraConfig)

Example 4 with StorageFactoryCreator

use of io.pravega.segmentstore.storage.StorageFactoryCreator in project pravega by pravega.

the class StorageFactoryTests method testExtendedS3StorageFactoryCreator.

@Test
public void testExtendedS3StorageFactoryCreator() {
    StorageFactoryCreator factoryCreator = new ExtendedS3StorageFactoryCreator();
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("EXTENDEDS3").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build(), StorageFactoryInfo.builder().name("EXTENDEDS3").storageLayoutType(StorageLayoutType.ROLLING_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(2, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    val config = ExtendedS3StorageConfig.builder().with(ExtendedS3StorageConfig.CONFIGURI, "http://127.0.0.1?identity=x&secretKey=x").with(ExtendedS3StorageConfig.BUCKET, "bucket").with(ExtendedS3StorageConfig.PREFIX, "samplePrefix").build();
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, config);
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executorService());
    Assert.assertTrue(factory1 instanceof ExtendedS3SimpleStorageFactory);
    @Cleanup Storage storage1 = ((ExtendedS3SimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    Assert.assertTrue(((ChunkedSegmentStorage) storage1).getChunkStorage() instanceof ExtendedS3ChunkStorage);
    // Legacy Storage
    ConfigSetup configSetup2 = mock(ConfigSetup.class);
    when(configSetup2.getConfig(any())).thenReturn(config);
    val factory2 = factoryCreator.createFactory(expected[1], configSetup2, executorService());
    Assert.assertTrue(factory2 instanceof ExtendedS3StorageFactory);
    @Cleanup Storage storage2 = factory2.createStorageAdapter();
    Assert.assertTrue(storage2 instanceof AsyncStorageWrapper);
    @Cleanup SyncStorage syncStorage = factory2.createSyncStorage();
    Assert.assertNotNull(syncStorage);
    AssertExtensions.assertThrows("createStorageAdapter should throw UnsupportedOperationException.", () -> factory1.createStorageAdapter(), ex -> ex instanceof UnsupportedOperationException);
}
Also used : lombok.val(lombok.val) SyncStorage(io.pravega.segmentstore.storage.SyncStorage) ExtendedS3ChunkStorage(io.pravega.storage.extendeds3.ExtendedS3ChunkStorage) StorageFactoryInfo(io.pravega.segmentstore.storage.StorageFactoryInfo) ChunkedSegmentStorage(io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage) Cleanup(lombok.Cleanup) Storage(io.pravega.segmentstore.storage.Storage) ChunkedSegmentStorage(io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage) ExtendedS3ChunkStorage(io.pravega.storage.extendeds3.ExtendedS3ChunkStorage) HDFSChunkStorage(io.pravega.storage.hdfs.HDFSChunkStorage) S3ChunkStorage(io.pravega.storage.s3.S3ChunkStorage) SyncStorage(io.pravega.segmentstore.storage.SyncStorage) FileSystemChunkStorage(io.pravega.storage.filesystem.FileSystemChunkStorage) ExtendedS3StorageFactory(io.pravega.storage.extendeds3.ExtendedS3StorageFactory) AsyncStorageWrapper(io.pravega.segmentstore.storage.AsyncStorageWrapper) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) S3StorageFactoryCreator(io.pravega.storage.s3.S3StorageFactoryCreator) HDFSStorageFactoryCreator(io.pravega.storage.hdfs.HDFSStorageFactoryCreator) StorageFactoryCreator(io.pravega.segmentstore.storage.StorageFactoryCreator) ExtendedS3StorageFactoryCreator(io.pravega.storage.extendeds3.ExtendedS3StorageFactoryCreator) FileSystemStorageFactoryCreator(io.pravega.storage.filesystem.FileSystemStorageFactoryCreator) ExtendedS3StorageFactoryCreator(io.pravega.storage.extendeds3.ExtendedS3StorageFactoryCreator) ExtendedS3SimpleStorageFactory(io.pravega.storage.extendeds3.ExtendedS3SimpleStorageFactory) ConfigSetup(io.pravega.segmentstore.storage.ConfigSetup) Test(org.junit.Test)

Example 5 with StorageFactoryCreator

use of io.pravega.segmentstore.storage.StorageFactoryCreator in project pravega by pravega.

the class StorageFactoryTests method testFileSystemStorageFactoryCreator.

@Test
public void testFileSystemStorageFactoryCreator() {
    StorageFactoryCreator factoryCreator = new FileSystemStorageFactoryCreator();
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("FILESYSTEM").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build(), StorageFactoryInfo.builder().name("FILESYSTEM").storageLayoutType(StorageLayoutType.ROLLING_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(2, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, FileSystemStorageConfig.builder().build());
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executorService());
    Assert.assertTrue(factory1 instanceof FileSystemSimpleStorageFactory);
    @Cleanup Storage storage1 = ((FileSystemSimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    Assert.assertTrue(((ChunkedSegmentStorage) storage1).getChunkStorage() instanceof FileSystemChunkStorage);
    // Legacy Storage
    ConfigSetup configSetup2 = mock(ConfigSetup.class);
    when(configSetup2.getConfig(any())).thenReturn(FileSystemStorageConfig.builder().build());
    val factory2 = factoryCreator.createFactory(expected[1], configSetup2, executorService());
    Assert.assertTrue(factory2 instanceof FileSystemStorageFactory);
    Storage storage2 = factory2.createStorageAdapter();
    Assert.assertTrue(storage2 instanceof AsyncStorageWrapper);
    @Cleanup SyncStorage syncStorage = factory2.createSyncStorage();
    Assert.assertNotNull(syncStorage);
    AssertExtensions.assertThrows("createStorageAdapter should throw UnsupportedOperationException.", () -> factory1.createStorageAdapter(), ex -> ex instanceof UnsupportedOperationException);
}
Also used : lombok.val(lombok.val) SyncStorage(io.pravega.segmentstore.storage.SyncStorage) FileSystemStorageFactoryCreator(io.pravega.storage.filesystem.FileSystemStorageFactoryCreator) FileSystemStorageFactory(io.pravega.storage.filesystem.FileSystemStorageFactory) StorageFactoryInfo(io.pravega.segmentstore.storage.StorageFactoryInfo) ChunkedSegmentStorage(io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage) FileSystemSimpleStorageFactory(io.pravega.storage.filesystem.FileSystemSimpleStorageFactory) Cleanup(lombok.Cleanup) Storage(io.pravega.segmentstore.storage.Storage) ChunkedSegmentStorage(io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage) ExtendedS3ChunkStorage(io.pravega.storage.extendeds3.ExtendedS3ChunkStorage) HDFSChunkStorage(io.pravega.storage.hdfs.HDFSChunkStorage) S3ChunkStorage(io.pravega.storage.s3.S3ChunkStorage) SyncStorage(io.pravega.segmentstore.storage.SyncStorage) FileSystemChunkStorage(io.pravega.storage.filesystem.FileSystemChunkStorage) AsyncStorageWrapper(io.pravega.segmentstore.storage.AsyncStorageWrapper) InMemoryMetadataStore(io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore) S3StorageFactoryCreator(io.pravega.storage.s3.S3StorageFactoryCreator) HDFSStorageFactoryCreator(io.pravega.storage.hdfs.HDFSStorageFactoryCreator) StorageFactoryCreator(io.pravega.segmentstore.storage.StorageFactoryCreator) ExtendedS3StorageFactoryCreator(io.pravega.storage.extendeds3.ExtendedS3StorageFactoryCreator) FileSystemStorageFactoryCreator(io.pravega.storage.filesystem.FileSystemStorageFactoryCreator) ConfigSetup(io.pravega.segmentstore.storage.ConfigSetup) FileSystemChunkStorage(io.pravega.storage.filesystem.FileSystemChunkStorage) Test(org.junit.Test)

Aggregations

StorageFactoryCreator (io.pravega.segmentstore.storage.StorageFactoryCreator)6 lombok.val (lombok.val)6 ConfigSetup (io.pravega.segmentstore.storage.ConfigSetup)5 Storage (io.pravega.segmentstore.storage.Storage)5 StorageFactoryInfo (io.pravega.segmentstore.storage.StorageFactoryInfo)5 SyncStorage (io.pravega.segmentstore.storage.SyncStorage)5 ChunkedSegmentStorage (io.pravega.segmentstore.storage.chunklayer.ChunkedSegmentStorage)5 Cleanup (lombok.Cleanup)5 AsyncStorageWrapper (io.pravega.segmentstore.storage.AsyncStorageWrapper)4 InMemoryMetadataStore (io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore)4 ExtendedS3ChunkStorage (io.pravega.storage.extendeds3.ExtendedS3ChunkStorage)4 ExtendedS3StorageFactoryCreator (io.pravega.storage.extendeds3.ExtendedS3StorageFactoryCreator)4 FileSystemChunkStorage (io.pravega.storage.filesystem.FileSystemChunkStorage)4 FileSystemStorageFactoryCreator (io.pravega.storage.filesystem.FileSystemStorageFactoryCreator)4 HDFSChunkStorage (io.pravega.storage.hdfs.HDFSChunkStorage)4 HDFSStorageFactoryCreator (io.pravega.storage.hdfs.HDFSStorageFactoryCreator)4 S3ChunkStorage (io.pravega.storage.s3.S3ChunkStorage)4 S3StorageFactoryCreator (io.pravega.storage.s3.S3StorageFactoryCreator)4 Test (org.junit.Test)4 ExtendedS3SimpleStorageFactory (io.pravega.storage.extendeds3.ExtendedS3SimpleStorageFactory)2