use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class SystemJournalTests method testSystemSegmentConcurrency.
/**
* Test concurrent writes to storage system segments by simulating concurrent writes.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSystemSegmentConcurrency() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
@Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
int containerId = 42;
int maxLength = 8;
long epoch = 1;
val policy = new SegmentRollingPolicy(maxLength);
val config = getDefaultConfigBuilder(policy).build();
val snapshotData = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> snapshotData.setSnapshotId(containerId, snapshotId), () -> snapshotData.getSnapshotId(containerId));
// Start container with epoch 1
@Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreBeforeCrash, executorService(), config);
segmentStorage1.initialize(epoch);
// Bootstrap
segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
segmentStorage1.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage1);
checkSystemSegmentsLayout(segmentStorage1);
// Simulate some writes to system segment, this should cause some new chunks being added.
val writeSize = 10;
val numWrites = 10;
val numOfStorageSystemSegments = SystemJournal.getChunkStorageSystemSegments(containerId).length;
val data = new byte[numOfStorageSystemSegments][writeSize * numWrites];
var futures = new ArrayList<CompletableFuture<Void>>();
val rnd = new Random(0);
for (int i = 0; i < numOfStorageSystemSegments; i++) {
final int k = i;
futures.add(CompletableFuture.runAsync(() -> {
rnd.nextBytes(data[k]);
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[k];
val h = segmentStorage1.openWrite(systemSegmentName).join();
// Init
long offset = 0;
for (int j = 0; j < numWrites; j++) {
segmentStorage1.write(h, offset, new ByteArrayInputStream(data[k], writeSize * j, writeSize), writeSize, null).join();
offset += writeSize;
}
val info = segmentStorage1.getStreamSegmentInfo(systemSegmentName, null).join();
Assert.assertEquals(writeSize * numWrites, info.getLength());
byte[] out = new byte[writeSize * numWrites];
val hr = segmentStorage1.openRead(systemSegmentName).join();
segmentStorage1.read(hr, 0, out, 0, writeSize * numWrites, null).join();
Assert.assertArrayEquals(data[k], out);
}, executorService()));
}
Futures.allOf(futures).join();
// Step 2
// Start container with epoch 2
epoch++;
@Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
segmentStorage2.initialize(epoch);
// Bootstrap
segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
segmentStorage2.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage2);
checkSystemSegmentsLayout(segmentStorage2);
// Validate
for (int i = 0; i < numOfStorageSystemSegments; i++) {
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[i];
val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
Assert.assertEquals(writeSize * numWrites, info.getLength());
byte[] out = new byte[writeSize * numWrites];
val hr = segmentStorage2.openRead(systemSegmentName).join();
segmentStorage2.read(hr, 0, out, 0, writeSize * numWrites, null).join();
Assert.assertArrayEquals(data[i], out);
}
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class SystemJournalTests method testCommitInvalidArgs.
@Test
public void testCommitInvalidArgs() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = 42;
int maxLength = 8;
long epoch = 1;
@Cleanup val garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector.initialize(new InMemoryTaskQueueManager()).join();
val policy = new SegmentRollingPolicy(maxLength);
val config = getDefaultConfigBuilder(policy).build();
val journal = new SystemJournal(containerId, chunkStorage, metadataStore, garbageCollector, config, executorService());
AssertExtensions.assertThrows("commitRecords() should throw", () -> journal.commitRecord(null), ex -> ex instanceof IllegalArgumentException);
AssertExtensions.assertThrows("commitRecords() should throw", () -> journal.commitRecords(null), ex -> ex instanceof IllegalArgumentException);
AssertExtensions.assertThrows("commitRecords() should throw", () -> journal.commitRecords(new ArrayList<>()), ex -> ex instanceof IllegalArgumentException);
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class SystemJournalTests method testSimpleBootstrapWithTwoTruncates.
/**
* Tests a scenario when there are two fail overs.
* The test adds a few chunks to the system segments and then fails over.
* It also truncates system segments.
* The new instance should read the journal log file and recreate the layout of system segments.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSimpleBootstrapWithTwoTruncates() throws Exception {
@Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
@Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
int containerId = 42;
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
long epoch = 1;
val policy = new SegmentRollingPolicy(8);
val config = getDefaultConfigBuilder(policy).build();
long offset = 0;
val data = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
// Epoch 1
@Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreBeforeCrash, executorService(), config);
segmentStorage1.initialize(epoch);
segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
// Bootstrap
segmentStorage1.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage1);
checkSystemSegmentsLayout(segmentStorage1);
// Simulate some writes to system segment, this should cause some new chunks being added.
val h = segmentStorage1.openWrite(systemSegmentName).join();
segmentStorage1.write(h, offset, new ByteArrayInputStream("JUNKJUNKJUNK".getBytes()), 12, null).join();
offset += 12;
val b1 = "Hello".getBytes();
segmentStorage1.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
offset += b1.length;
segmentStorage1.truncate(h, 6, null).join();
// Epoch 2
epoch++;
@Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
segmentStorage2.initialize(epoch);
segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
segmentStorage2.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage2);
checkSystemSegmentsLayout(segmentStorage2);
val h2 = segmentStorage2.openWrite(systemSegmentName).join();
// Write Junk data to both system segment and journal file.
segmentStorage1.write(h, offset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
val b2 = " World".getBytes();
segmentStorage2.write(h2, offset, new ByteArrayInputStream(b2), b2.length, null).join();
offset += b2.length;
segmentStorage2.truncate(h, 12, null).join();
val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
Assert.assertEquals(b1.length + b2.length + 12, info.getLength());
Assert.assertEquals(12, info.getStartOffset());
byte[] out = new byte[b1.length + b2.length];
val hr = segmentStorage2.openRead(systemSegmentName).join();
segmentStorage2.read(hr, 12, out, 0, b1.length + b2.length, null).join();
Assert.assertEquals("Hello World", new String(out));
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method testActiveSegment.
/**
* Test for segment that is marked active and added as garbage.
*/
@Test
public void testActiveSegment() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
int dataSize = 1;
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
val config = ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, config, executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
insertSegment(metadataStore, chunkStorage, config, "testSegment", 10, 1, new long[] { 1, 2, 3, 4 }, false, 1);
val chunkNames = TestUtils.getChunkNameList(metadataStore, "testSegment");
// Add some garbage
garbageCollector.addSegmentToGarbage(TXN_ID, "testSegment").join();
// Validate state before
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
Assert.assertEquals("testSegment", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
Assert.assertNotNull(getSegmentMetadata(metadataStore, "testSegment"));
chunkNames.stream().forEach(chunkName -> Assert.assertTrue(chunkStorage.exists(chunkName).join()));
}
use of io.pravega.segmentstore.storage.mocks.InMemoryTaskQueueManager in project pravega by pravega.
the class GarbageCollectorTests method testChunkNotFound.
/**
* Test for ChunkNotFound exception.
*/
@Test
public void testChunkNotFound() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
int dataSize = 1;
insertChunkMetadata(metadataStore, "missingChunk", dataSize, 0);
Assert.assertFalse(chunkStorage.exists("missingChunk").get());
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
// Add some garbage
garbageCollector.addChunksToGarbage(TXN_ID, Arrays.asList("missingChunk")).join();
// Validate state before
assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "missingChunk" });
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
}
Aggregations