Examples with BaseMetadataStore io.pravega.segmentstore.storage.metadata.BaseMetadataStore used on opensource projects

Example 6 with BaseMetadataStore

use of io.pravega.segmentstore.storage.metadata.BaseMetadataStore in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testIOExceptionDuringWrite.

@Test
public void testIOExceptionDuringWrite() throws Exception {
    String testSegmentName = "test";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    ((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    chunkedSegmentStorage.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    // Step 2: Inject fault.
    Exception exceptionToThrow = new ChunkStorageException("test", "Test Exception", new IOException("Test Exception"));
    val clazz = ChunkStorageException.class;
    doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
    AssertExtensions.assertFutureThrows("write succeeded when exception was expected.", chunkedSegmentStorage.write(h1, 10, new ByteArrayInputStream(new byte[10]), 10, null), ex -> clazz.equals(ex.getClass()));
}

Example 7 with BaseMetadataStore

use of io.pravega.segmentstore.storage.metadata.BaseMetadataStore in project pravega by pravega.

the class UtilsWrapperTests method testCopy.

private void testCopy(long[] chunkLengths) throws Exception {
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG;
    val segmentName = "test";
    // Set up
    @Cleanup ChunkStorage chunkStorage = new InMemoryChunkStorage(executorService());
    @Cleanup BaseMetadataStore metadataStore = new InMemoryMetadataStore(config, executorService());
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(42, chunkStorage, metadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(123);
    TestUtils.insertMetadata(segmentName, 25, 123, chunkLengths, chunkLengths, false, false, chunkedSegmentStorage.getMetadataStore(), chunkedSegmentStorage);
    val chunkMetadataList = TestUtils.getChunkList(chunkedSegmentStorage.getMetadataStore(), segmentName);
    // Test
    UtilsWrapper wrapper = new UtilsWrapper(chunkedSegmentStorage, 128, Duration.ZERO);
    byte[][] expected = new byte[chunkMetadataList.size()][];
    int sum = 0;
    for (int i = 0; i < chunkMetadataList.size(); i++) {
        val info = chunkMetadataList.get(i);
        val length = Math.toIntExact(info.getLength());
        // create random data
        expected[i] = new byte[length];
        random.nextBytes(expected[i]);
        // Overwrite chunk
        wrapper.overwriteChunk(info.getName(), new ByteArrayInputStream(expected[i]), length).join();
        val actual = new ByteArrayOutputStream(length);
        // read the same data back
        wrapper.copyFromChunk(info.getName(), actual).join();
        Assert.assertArrayEquals(expected[i], actual.toByteArray());
        sum += length;
    }
    // Read back and validate segment
    val segmentContents = new ByteArrayOutputStream(sum);
    wrapper.copyFromSegment(segmentName, segmentContents).join();
    long startOffset = 0;
    for (int i = 0; i < expected.length; i++) {
        assertContentEquals(segmentContents.toByteArray(), startOffset, expected[i]);
        startOffset += expected[i].length;
    }
}

Example 8 with BaseMetadataStore

use of io.pravega.segmentstore.storage.metadata.BaseMetadataStore in project pravega by pravega.

the class UtilsWrapperTests method testGetExtendedChunkInfoList.

private void testGetExtendedChunkInfoList(ChunkedSegmentStorageConfig config, String segmentName, boolean shouldCheckStorage, long[] lengthsInStorage, long[] lengthsInMetadata, int[] chunksToDelete) throws Exception {
    // Set up
    @Cleanup ChunkStorage chunkStorage = new InMemoryChunkStorage(executorService());
    @Cleanup BaseMetadataStore metadataStore = new InMemoryMetadataStore(config, executorService());
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(42, chunkStorage, metadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(123);
    // Insert metadata
    TestUtils.insertMetadata(segmentName, 25, 123, lengthsInMetadata, lengthsInStorage, false, false, metadataStore, chunkedSegmentStorage);
    val chunkMetadataList = TestUtils.getChunkList(metadataStore, segmentName);
    // Delete few chunks
    HashSet<String> deletedChunkNames = new HashSet<>();
    for (int i = 0; i < chunksToDelete.length; i++) {
        val name = chunkMetadataList.get(i).getName();
        deletedChunkNames.add(name);
        chunkStorage.delete(ChunkHandle.writeHandle(name)).join();
    }
    // Test
    UtilsWrapper wrapper = new UtilsWrapper(chunkedSegmentStorage, 128, Duration.ZERO);
    // check the output
    checkExtendedChunkInfoList(wrapper, segmentName, shouldCheckStorage, lengthsInStorage, deletedChunkNames, chunkMetadataList);
}

Example 9 with BaseMetadataStore

use of io.pravega.segmentstore.storage.metadata.BaseMetadataStore in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testReport.

@Test
public void testReport() throws Exception {
    MetricsConfig metricsConfig = MetricsConfig.builder().with(MetricsConfig.ENABLE_STATISTICS, true).build();
    MetricsProvider.initialize(metricsConfig);
    @Cleanup StatsProvider statsProvider = MetricsProvider.getMetricsProvider();
    statsProvider.startWithoutExporting();
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().safeStorageSizeCheckFrequencyInSeconds(100).maxSafeStorageSize(1000).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(config, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    when(spyChunkStorage.doGetUsedSpace(any())).thenReturn(123L);
    chunkedSegmentStorage.updateStorageStats().get();
    chunkedSegmentStorage.report();
    // Not possible to mock any other reporter except metadata store.
    verify(spyMetadataStore).report();
    Assert.assertEquals(123, MetricRegistryUtils.getGauge(SLTS_STORAGE_USED_BYTES).value(), 0);
    Assert.assertEquals(12.3, MetricRegistryUtils.getGauge(SLTS_STORAGE_USED_PERCENTAGE).value(), 0);
}

Example 10 with BaseMetadataStore

use of io.pravega.segmentstore.storage.metadata.BaseMetadataStore in project pravega by pravega.

the class ChunkedSegmentStorageMockTests method testIOExceptionDuringTruncate.

@Test
public void testIOExceptionDuringTruncate() throws Exception {
    String testSegmentName = "test";
    SegmentRollingPolicy policy = new SegmentRollingPolicy(100);
    val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().relocateOnTruncateEnabled(true).minSizeForTruncateRelocationInbytes(1).minPercentForTruncateRelocation(50).storageMetadataRollingPolicy(policy).build();
    @Cleanup BaseMetadataStore spyMetadataStore = spy(new InMemoryMetadataStore(config, executorService()));
    @Cleanup BaseChunkStorage spyChunkStorage = spy(new NoOpChunkStorage(executorService()));
    ((NoOpChunkStorage) spyChunkStorage).setShouldSupportConcat(false);
    @Cleanup ChunkedSegmentStorage chunkedSegmentStorage = new ChunkedSegmentStorage(CONTAINER_ID, spyChunkStorage, spyMetadataStore, executorService(), config);
    chunkedSegmentStorage.initialize(1);
    val taskQueueManager = new InMemoryTaskQueueManager();
    chunkedSegmentStorage.getGarbageCollector().initialize(taskQueueManager).join();
    // Step 1: Create segment and write some data.
    val h1 = chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h1.getSegmentName(), testSegmentName);
    Assert.assertFalse(h1.isReadOnly());
    chunkedSegmentStorage.write(h1, 0, new ByteArrayInputStream(new byte[10]), 10, null).get();
    val chunksListBefore = TestUtils.getChunkNameList(spyMetadataStore, testSegmentName);
    Assert.assertEquals(1, chunksListBefore.size());
    // Step 2: Inject fault.
    Exception exceptionToThrow = new ChunkStorageException("test", "Test Exception", new IOException("Test Exception"));
    val clazz = ChunkStorageException.class;
    doThrow(exceptionToThrow).when(spyChunkStorage).doWrite(any(), anyLong(), anyInt(), any());
    AssertExtensions.assertFutureThrows("truncate succeeded when exception was expected.", chunkedSegmentStorage.truncate(h1, 8, null), ex -> clazz.equals(ex.getClass()));
    TestUtils.checkSegmentLayout(spyMetadataStore, testSegmentName, new long[] { 10 });
    val chunksListAfter = TestUtils.getChunkNameList(spyMetadataStore, testSegmentName);
    Assert.assertEquals(1, chunksListAfter.size());
    Assert.assertTrue(chunksListAfter.containsAll(chunksListBefore));
    Assert.assertEquals(2, chunkedSegmentStorage.getGarbageCollector().getQueueSize().get());
    val list = taskQueueManager.drain(chunkedSegmentStorage.getGarbageCollector().getTaskQueueName(), 2);
    Assert.assertTrue(chunksListBefore.contains(list.get(0).getName()));
    val nameTemplate = String.format("%s.E-%d-O-%d.", testSegmentName, chunkedSegmentStorage.getEpoch(), 8);
    Assert.assertTrue("New first chunk should be added to GC queue.", list.get(1).getName().startsWith(nameTemplate));
}