Examples with InMemoryMetadataStore io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore used on opensource projects

Example 1 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class StorageFactoryTests method testS3StorageFactoryCreator.

private void testS3StorageFactoryCreator(S3StorageConfig config) {
    StorageFactoryCreator factoryCreator = new S3StorageFactoryCreator();
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("S3").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(1, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, config);
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executorService());
    Assert.assertTrue(factory1 instanceof S3SimpleStorageFactory);
    @Cleanup Storage storage1 = ((S3SimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    Assert.assertTrue(((ChunkedSegmentStorage) storage1).getChunkStorage() instanceof S3ChunkStorage);
    AssertExtensions.assertThrows("createStorageAdapter should throw UnsupportedOperationException.", () -> factory1.createStorageAdapter(), ex -> ex instanceof UnsupportedOperationException);
}

Example 2 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class StorageFactoryTests method testHDFSStorageFactoryCreator.

@Test
public void testHDFSStorageFactoryCreator() {
    StorageFactoryCreator factoryCreator = new HDFSStorageFactoryCreator();
    val expected = new StorageFactoryInfo[] { StorageFactoryInfo.builder().name("HDFS").storageLayoutType(StorageLayoutType.CHUNKED_STORAGE).build(), StorageFactoryInfo.builder().name("HDFS").storageLayoutType(StorageLayoutType.ROLLING_STORAGE).build() };
    val factoryInfoList = factoryCreator.getStorageFactories();
    Assert.assertEquals(2, factoryInfoList.length);
    Assert.assertArrayEquals(expected, factoryInfoList);
    // Simple Storage
    ConfigSetup configSetup1 = mock(ConfigSetup.class);
    when(configSetup1.getConfig(any())).thenReturn(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, HDFSStorageConfig.builder().build());
    val factory1 = factoryCreator.createFactory(expected[0], configSetup1, executorService());
    Assert.assertTrue(factory1 instanceof HDFSSimpleStorageFactory);
    @Cleanup Storage storage1 = ((HDFSSimpleStorageFactory) factory1).createStorageAdapter(42, new InMemoryMetadataStore(ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService()));
    Assert.assertTrue(storage1 instanceof ChunkedSegmentStorage);
    Assert.assertTrue(((ChunkedSegmentStorage) storage1).getChunkStorage() instanceof HDFSChunkStorage);
    // Legacy Storage
    ConfigSetup configSetup2 = mock(ConfigSetup.class);
    when(configSetup2.getConfig(any())).thenReturn(HDFSStorageConfig.builder().build());
    val factory2 = factoryCreator.createFactory(expected[1], configSetup2, executorService());
    Assert.assertTrue(factory2 instanceof HDFSStorageFactory);
    @Cleanup Storage storage2 = factory2.createStorageAdapter();
    Assert.assertTrue(storage2 instanceof AsyncStorageWrapper);
    SyncStorage syncStorage = factory2.createSyncStorage();
    Assert.assertNotNull(syncStorage);
    AssertExtensions.assertThrows("createStorageAdapter should throw UnsupportedOperationException.", () -> factory1.createStorageAdapter(), ex -> ex instanceof UnsupportedOperationException);
}

Example 3 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class ChunkedSegmentStorageTests method testParallelReadRequestsOnSingleSegmentWithReentry.

private void testParallelReadRequestsOnSingleSegmentWithReentry(int numberOfRequests, int threadPoolSize, boolean shouldBlock) throws Exception {
    String testSegmentName = "testSegment";
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    @Cleanup TestContext testContext = getTestContext(ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().indexBlockSize(3).build());
    if (!(testContext.metadataStore instanceof InMemoryMetadataStore)) {
        return;
    }
    CompletableFuture<Void> futureToWaitOn = shouldBlock ? new CompletableFuture<Void>() : CompletableFuture.completedFuture(null);
    // Step 1: Populate dummy system segment segment.
    // Write some data to system segment so that we can read it back in call back.
    val systemSegment = "SystemSegment";
    val hSystem = testContext.chunkedSegmentStorage.create(systemSegment, policy, null).get();
    testContext.chunkedSegmentStorage.write(hSystem, 0, new ByteArrayInputStream(new byte[1]), 1, null).join();
    // Step 2: Create test segment.
    val h = testContext.chunkedSegmentStorage.create(testSegmentName, policy, null).get();
    Assert.assertEquals(h.getSegmentName(), testSegmentName);
    Assert.assertFalse(h.isReadOnly());
    // Step 3: Write some data to test segment.
    long writeAt = 0;
    for (int i = 1; i < 5; i++) {
        testContext.chunkedSegmentStorage.write(h, writeAt, new ByteArrayInputStream(new byte[i]), i, null).join();
        writeAt += i;
    }
    TestUtils.checkSegmentLayout(testContext.metadataStore, testSegmentName, 2, 5);
    TestUtils.checkSegmentBounds(testContext.metadataStore, testSegmentName, 0, 10);
    TestUtils.checkReadIndexEntries(testContext.chunkedSegmentStorage, testContext.metadataStore, testSegmentName, 0, 10, true);
    TestUtils.checkChunksExistInStorage(testContext.chunkStorage, testContext.metadataStore, testSegmentName);
    // Step 4: Setup call backs that read system segment on each read.
    // Set up a call back which will be invoked during get call.
    ((InMemoryMetadataStore) testContext.metadataStore).setReadCallback(transactionData -> {
        // Make sure we don't invoke read for system segment itself.
        if (!transactionData.getKey().equals(systemSegment)) {
            return futureToWaitOn.thenComposeAsync(v -> checkDataReadAsync(systemSegment, testContext, 0, 1, executorService()), executorService()).thenApplyAsync(v -> null, executorService());
        }
        return CompletableFuture.completedFuture(null);
    });
    // Step 5: Read back data concurrently.
    @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize);
    CompletableFuture[] futures = new CompletableFuture[numberOfRequests];
    for (int i = 0; i < numberOfRequests; i++) {
        CompletableFuture<Void> f = testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).thenComposeAsync(info -> {
            Assert.assertFalse(info.isSealed());
            Assert.assertFalse(info.isDeleted());
            Assert.assertEquals(info.getName(), testSegmentName);
            Assert.assertEquals(info.getLength(), 10);
            Assert.assertEquals(info.getStartOffset(), 0);
            return checkDataReadAsync(testSegmentName, testContext, 0, 10, executor);
        }, executor);
        futures[i] = f;
    }
    if (shouldBlock) {
        futureToWaitOn.complete(null);
    }
    CompletableFuture.allOf(futures).join();
}

Example 4 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class ChunkedSegmentStorageTests method testParallelSegmentOperationsWithReentry.

private void testParallelSegmentOperationsWithReentry(int numberOfRequests, int threadPoolSize, boolean shouldBlock) throws Exception {
    // Force rollover after every 2 byte.
    SegmentRollingPolicy policy = new SegmentRollingPolicy(2);
    @Cleanup TestContext testContext = getTestContext();
    if (!(testContext.metadataStore instanceof InMemoryMetadataStore)) {
        return;
    }
    CompletableFuture<Void> futureToWaitOn = shouldBlock ? new CompletableFuture<Void>() : CompletableFuture.completedFuture(null);
    // Step 1: Populate system segment.
    // Write some data to system segment so that we can read it back in call back.
    val systemSegment = "SystemSegment";
    val h = testContext.chunkedSegmentStorage.create(systemSegment, policy, null).get();
    testContext.chunkedSegmentStorage.write(h, 0, new ByteArrayInputStream(new byte[1]), 1, null).join();
    // Step 2: Setup call backs.
    // Set up a call back which will be invoked during get call.
    ((InMemoryMetadataStore) testContext.metadataStore).setReadCallback(transactionData -> {
        // Make sure we don't invoke read for system segment itself.
        if (!transactionData.getKey().equals(systemSegment)) {
            return futureToWaitOn.thenComposeAsync(v -> checkDataReadAsync(systemSegment, testContext, 0, 1, executorService()), executorService()).thenApplyAsync(v -> null, executorService());
        }
        return CompletableFuture.completedFuture(null);
    });
    // Set up a call back which will be invoked during writeAll call.
    ((InMemoryMetadataStore) testContext.metadataStore).setWriteCallback(transactionDataList -> {
        // Make sure we don't invoke read for system segment itself.
        if (transactionDataList.stream().filter(t -> !t.getKey().equals(systemSegment)).findAny().isPresent()) {
            return futureToWaitOn.thenComposeAsync(v -> checkDataReadAsync(systemSegment, testContext, 0, 1, executorService()), executorService()).thenApplyAsync(v -> null, executorService());
        }
        return CompletableFuture.completedFuture(null);
    });
    // Step 3: Perform operations on multiple segments.
    @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize);
    CompletableFuture[] futures = new CompletableFuture[numberOfRequests];
    for (int i = 0; i < numberOfRequests; i++) {
        String testSegmentName = "test" + i;
        val f = testSimpleScenarioAsync(testSegmentName, policy, testContext, executor);
        futures[i] = f;
    }
    if (shouldBlock) {
        futureToWaitOn.complete(null);
    }
    CompletableFuture.allOf(futures).join();
}

Example 5 with InMemoryMetadataStore

use of io.pravega.segmentstore.storage.mocks.InMemoryMetadataStore in project pravega by pravega.

the class ChunkedSegmentStorageTests method testWritesWithFlakyMetadataStore.

public void testWritesWithFlakyMetadataStore(int failFrequency, int length) throws Exception {
    String testSegmentName = "foo";
    @Cleanup TestContext testContext = getTestContext(ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().lazyCommitEnabled(false).build());
    val invocationCount = new AtomicInteger(0);
    val testMetadataStore = (InMemoryMetadataStore) testContext.metadataStore;
    testMetadataStore.setMaxEntriesInTxnBuffer(1);
    testMetadataStore.setWriteCallback(dummy -> {
        if (invocationCount.incrementAndGet() % failFrequency == 0) {
            return CompletableFuture.failedFuture(new IntentionalException("Intentional"));
        }
        return CompletableFuture.completedFuture(null);
    });
    val h = testContext.chunkedSegmentStorage.create(testSegmentName, null).get();
    byte[] data = populate(100);
    int currentOffset = 0;
    SegmentMetadata expectedSegmentMetadata = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
    ChunkMetadata expectedChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, expectedSegmentMetadata.getLastChunk());
    testMetadataStore.evictAllEligibleEntriesFromBuffer();
    testMetadataStore.evictFromCache();
    while (currentOffset < data.length) {
        try {
            int toWrite = Math.min(length, data.length - currentOffset);
            expectedSegmentMetadata = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
            expectedChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, expectedSegmentMetadata.getLastChunk());
            testContext.chunkedSegmentStorage.write(h, currentOffset, new ByteArrayInputStream(data, currentOffset, toWrite), toWrite, null).get();
            currentOffset += toWrite;
        } catch (Exception e) {
            if (!(Exceptions.unwrap(e) instanceof IntentionalException)) {
                throw e;
            }
            val actual = TestUtils.getSegmentMetadata(testContext.metadataStore, testSegmentName);
            val actualChunkMetadata = TestUtils.getChunkMetadata(testContext.metadataStore, expectedSegmentMetadata.getLastChunk());
            Assert.assertEquals(expectedSegmentMetadata, actual);
            Assert.assertEquals(expectedChunkMetadata, actualChunkMetadata);
        } finally {
            val info = testContext.chunkedSegmentStorage.getStreamSegmentInfo(testSegmentName, null).get();
            Assert.assertEquals(info.getLength(), currentOffset);
        }
    }
    testMetadataStore.setWriteCallback(null);
    checkDataRead(testSegmentName, testContext, 0, data.length, data);
}