Examples with InMemoryMetadataStore io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore used on opensource projects

Example 16 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class StorageFactoryTests method testExtendedS3StorageFactoryCreator.

@Test
public void testExtendedS3StorageFactoryCreator() {
    StorageFactoryCreator factoryCreator = new ExtendedS3StorageFactoryCreator();
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("EXTENDEDS3").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build(), StorageFactoryInfo.builder().name("EXTENDEDS3").storageLayoutType(StorageLayoutType.ROLLING_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(2, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    val config = ExtendedS3StorageConfig.builder().with(ExtendedS3StorageConfig.CONFIGURI, "http://127.0.0.1?identity=x&secretKey=x").with(ExtendedS3StorageConfig.BUCKET, "bucket").with(ExtendedS3StorageConfig.PREFIX, "samplePrefix").build();
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, config);
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executorService());
    Assert.assertTrue(factory1 instanceof ExtendedS3SimpleStorageFactory);
    @Cleanup Storage storage1 = ((ExtendedS3SimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    Assert.assertTrue(((ChunkedSegmentStorage) storage1).getChunkStorage() instanceof ExtendedS3ChunkStorage);
    // Legacy Storage
    ConfigSetup configSetup2 = mock(ConfigSetup.class);
    when(configSetup2.getConfig(any())).thenReturn(config);
    val factory2 = factoryCreator.createFactory(expected[1], configSetup2, executorService());
    Assert.assertTrue(factory2 instanceof ExtendedS3StorageFactory);
    @Cleanup Storage storage2 = factory2.createStorageAdapter();
    Assert.assertTrue(storage2 instanceof AsyncStorageWrapper);
    @Cleanup SyncStorage syncStorage = factory2.createSyncStorage();
    Assert.assertNotNull(syncStorage);
    AssertExtensions.assertThrows("createStorageAdapter should throw UnsupportedOperationException.", () -> factory1.createStorageAdapter(), ex -> ex instanceof UnsupportedOperationException);
}

Example 17 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class StorageFactoryTests method testFileSystemStorageFactoryCreator.

@Test
public void testFileSystemStorageFactoryCreator() {
    StorageFactoryCreator factoryCreator = new FileSystemStorageFactoryCreator();
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("FILESYSTEM").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build(), StorageFactoryInfo.builder().name("FILESYSTEM").storageLayoutType(StorageLayoutType.ROLLING_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(2, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, FileSystemStorageConfig.builder().build());
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executorService());
    Assert.assertTrue(factory1 instanceof FileSystemSimpleStorageFactory);
    @Cleanup Storage storage1 = ((FileSystemSimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    Assert.assertTrue(((ChunkedSegmentStorage) storage1).getChunkStorage() instanceof FileSystemChunkStorage);
    // Legacy Storage
    ConfigSetup configSetup2 = mock(ConfigSetup.class);
    when(configSetup2.getConfig(any())).thenReturn(FileSystemStorageConfig.builder().build());
    val factory2 = factoryCreator.createFactory(expected[1], configSetup2, executorService());
    Assert.assertTrue(factory2 instanceof FileSystemStorageFactory);
    Storage storage2 = factory2.createStorageAdapter();
    Assert.assertTrue(storage2 instanceof AsyncStorageWrapper);
    @Cleanup SyncStorage syncStorage = factory2.createSyncStorage();
    Assert.assertNotNull(syncStorage);
    AssertExtensions.assertThrows("createStorageAdapter should throw UnsupportedOperationException.", () -> factory1.createStorageAdapter(), ex -> ex instanceof UnsupportedOperationException);
}

Example 18 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class UtilsWrapperTests method testEvictReadIndexCache.

@Test
public void testEvictReadIndexCache() throws Exception {
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG;
    // Set up
    @Cleanup ChunkStorage chunkStorage = new InMemoryChunkStorage(executorService());
    @Cleanup val metadataStore = new InMemoryMetadataStore(config, executorService());
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(42, chunkStorage, metadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(123);
    val chunkSizes = new long[] { 1 };
    // Insert metadata
    TestUtils.insertMetadata("a", 42, 10, chunkSizes, chunkSizes, true, true, chunkedSegmentStorage.getMetadataStore(), chunkedSegmentStorage);
    TestUtils.insertMetadata("b", 42, 10, chunkSizes, chunkSizes, true, true, chunkedSegmentStorage.getMetadataStore(), chunkedSegmentStorage);
    TestUtils.insertMetadata("c", 42, 10, chunkSizes, chunkSizes, true, true, chunkedSegmentStorage.getMetadataStore(), chunkedSegmentStorage);
    // Test
    UtilsWrapper wrapper = new UtilsWrapper(chunkedSegmentStorage, 128, Duration.ZERO);
    wrapper.evictMetadataCache().join();
    Assert.assertNotNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("a", 0));
    Assert.assertNotNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("b", 0));
    Assert.assertNotNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("c", 0));
    wrapper.evictReadIndexCacheForSegment("a").join();
    Assert.assertNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("a", 0));
    Assert.assertNotNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("b", 0));
    Assert.assertNotNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("c", 0));
    wrapper.evictReadIndexCacheForSegment("b").join();
    Assert.assertNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("a", 0));
    Assert.assertNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("b", 0));
    Assert.assertNotNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("c", 0));
    wrapper.evictReadIndexCache().join();
    Assert.assertNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("a", 0));
    Assert.assertNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("b", 0));
    Assert.assertNull(wrapper.getChunkedSegmentStorage().getReadIndexCache().findFloor("c", 0));
}

Example 19 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class UtilsWrapperTests method testCopy.

private void testCopy(long[] chunkLengths) throws Exception {
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG;
    val segmentName = "test";
    // Set up
    @Cleanup ChunkStorage chunkStorage = new InMemoryChunkStorage(executorService());
    @Cleanup BaseMetadataStore metadataStore = new InMemoryMetadataStore(config, executorService());
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(42, chunkStorage, metadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(123);
    TestUtils.insertMetadata(segmentName, 25, 123, chunkLengths, chunkLengths, false, false, chunkedSegmentStorage.getMetadataStore(), chunkedSegmentStorage);
    val chunkMetadataList = TestUtils.getChunkList(chunkedSegmentStorage.getMetadataStore(), segmentName);
    // Test
    UtilsWrapper wrapper = new UtilsWrapper(chunkedSegmentStorage, 128, Duration.ZERO);
    byte[][] expected = new byte[chunkMetadataList.size()][];
    int sum = 0;
    for (int i = 0; i < chunkMetadataList.size(); i++) {
        val info = chunkMetadataList.get(i);
        val length = Math.toIntExact(info.getLength());
        // create random data
        expected[i] = new byte[length];
        random.nextBytes(expected[i]);
        // Overwrite chunk
        wrapper.overwriteChunk(info.getName(), new ByteArrayInputStream(expected[i]), length).join();
        val actual = new ByteArrayOutputStream(length);
        // read the same data back
        wrapper.copyFromChunk(info.getName(), actual).join();
        Assert.assertArrayEquals(expected[i], actual.toByteArray());
        sum += length;
    }
    // Read back and validate segment
    val segmentContents = new ByteArrayOutputStream(sum);
    wrapper.copyFromSegment(segmentName, segmentContents).join();
    long startOffset = 0;
    for (int i = 0; i < expected.length; i++) {
        assertContentEquals(segmentContents.toByteArray(), startOffset, expected[i]);
        startOffset += expected[i].length;
    }
}

Example 20 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class UtilsWrapperTests method testGetExtendedChunkInfoList.

private void testGetExtendedChunkInfoList(ChunkedSegmentStorageConfig config, String segmentName, boolean shouldCheckStorage, long[] lengthsInStorage, long[] lengthsInMetadata, int[] chunksToDelete) throws Exception {
    // Set up
    @Cleanup ChunkStorage chunkStorage = new InMemoryChunkStorage(executorService());
    @Cleanup BaseMetadataStore metadataStore = new InMemoryMetadataStore(config, executorService());
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(42, chunkStorage, metadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(123);
    // Insert metadata
    TestUtils.insertMetadata(segmentName, 25, 123, lengthsInMetadata, lengthsInStorage, false, false, metadataStore, chunkedSegmentStorage);
    val chunkMetadataList = TestUtils.getChunkList(metadataStore, segmentName);
    // Delete few chunks
    HashSet<String> deletedChunkNames = new HashSet<>();
    for (int i = 0; i < chunksToDelete.length; i++) {
        val name = chunkMetadataList.get(i).getName();
        deletedChunkNames.add(name);
        chunkStorage.delete(ChunkHandle.writeHandle(name)).join();
    }
    // Test
    UtilsWrapper wrapper = new UtilsWrapper(chunkedSegmentStorage, 128, Duration.ZERO);
    // check the output
    checkExtendedChunkInfoList(wrapper, segmentName, shouldCheckStorage, lengthsInStorage, deletedChunkNames, chunkMetadataList);
}