use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.
the class SystemJournalTests method testSimpleOperationSequence.
/**
* Test simple series of operations.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSimpleOperationSequence() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
int containerId = 42;
val data = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
@Cleanup val garbageCollector1 = new GarbageCollector(containerId, chunkStorage, metadataStoreBeforeCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector1.initialize(new InMemoryTaskQueueManager()).join();
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
long epoch = 1;
val policy = new SegmentRollingPolicy(2);
val config = getDefaultConfigBuilder(policy).build();
// Inital set of additions
SystemJournal systemJournalBefore = new SystemJournal(containerId, chunkStorage, metadataStoreBeforeCrash, garbageCollector1, config, executorService());
systemJournalBefore.bootstrap(epoch, snapshotInfoStore).join();
String lastChunk = null;
long totalBytesWritten = 0;
for (int i = 0; i < 10; i++) {
String newChunk = "chunk" + i;
val h = chunkStorage.createWithContent(newChunk, Math.toIntExact(policy.getMaxLength()), new ByteArrayInputStream(new byte[Math.toIntExact(policy.getMaxLength())])).get();
totalBytesWritten += policy.getMaxLength();
systemJournalBefore.commitRecord(SystemJournal.ChunkAddedRecord.builder().segmentName(systemSegmentName).offset(policy.getMaxLength() * i).newChunkName(newChunk).oldChunkName(lastChunk).build()).join();
lastChunk = newChunk;
}
Assert.assertEquals(policy.getMaxLength() * 10, totalBytesWritten);
// Failover
@Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
@Cleanup val garbageCollector2 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector2.initialize(new InMemoryTaskQueueManager()).join();
SystemJournal systemJournalAfter = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash, garbageCollector2, config, executorService());
systemJournalAfter.bootstrap(epoch + 1, snapshotInfoStore).join();
TestUtils.checkSegmentLayout(metadataStoreAfterCrash, systemSegmentName, policy.getMaxLength(), 10);
TestUtils.checkSegmentBounds(metadataStoreAfterCrash, systemSegmentName, 0, totalBytesWritten);
@Cleanup ChunkMetadataStore metadataStoreAfterCrash2 = getMetadataStore();
@Cleanup val garbageCollector3 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash2, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector3.initialize(new InMemoryTaskQueueManager()).join();
SystemJournal systemJournalAfter2 = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash2, garbageCollector3, config, executorService());
systemJournalAfter2.bootstrap(epoch + 2, snapshotInfoStore).join();
TestUtils.checkSegmentLayout(metadataStoreAfterCrash2, systemSegmentName, policy.getMaxLength(), 10);
}
use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.
the class SystemJournalTests method testSimpleBootstrapWithMultipleFailoversWithTruncate.
/**
* Tests a scenario when there are multiple fail overs overs.
* The test adds a few chunks to the system segments and then fails over.
* After fail over the zombie instances continue to write junk data to both system segment and journal file.
* The new instance should read the journal log file and recreate the layout of system segments.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSimpleBootstrapWithMultipleFailoversWithTruncate() throws Exception {
@Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
int containerId = 42;
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
long epoch = 0;
val policy = new SegmentRollingPolicy(1024);
val config = getDefaultConfigBuilder(policy).build();
val data = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
long offset = 0;
long offsetToTruncateAt = 0;
ChunkedSegmentStorage oldChunkedSegmentStorage = null;
SegmentHandle oldHandle = null;
long oldOffset = 0;
for (int i = 1; i < 10; i++) {
// Epoch 2
epoch++;
ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
cleanupHelper.add(metadataStoreAfterCrash);
ChunkedSegmentStorage segmentStorageInLoop = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
cleanupHelper.add(segmentStorageInLoop);
segmentStorageInLoop.initialize(epoch);
segmentStorageInLoop.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
segmentStorageInLoop.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorageInLoop);
checkSystemSegmentsLayout(segmentStorageInLoop);
val h = segmentStorageInLoop.openWrite(systemSegmentName).join();
if (null != oldChunkedSegmentStorage) {
// Add some junk to previous instance after failover
oldChunkedSegmentStorage.write(oldHandle, oldOffset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
} else {
// Only first time.
for (int j = 1; j < 10; j++) {
val b0 = "JUNK".getBytes();
segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b0), b0.length, null).join();
offset += b0.length;
}
}
offsetToTruncateAt += 4;
val b1 = "Test".getBytes();
segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
offset += b1.length;
val b2 = Integer.toString(i).getBytes();
segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b2), b2.length, null).join();
offset += b2.length;
segmentStorageInLoop.truncate(h, offsetToTruncateAt, null).join();
TestUtils.checkSegmentBounds(metadataStoreAfterCrash, h.getSegmentName(), offsetToTruncateAt, offset);
val length = Math.toIntExact(offset - offsetToTruncateAt);
byte[] readBytes = new byte[length];
val bytesRead = segmentStorageInLoop.read(h, offsetToTruncateAt, readBytes, 0, length, null).get();
Assert.assertEquals(length, readBytes.length);
String s = new String(readBytes);
// Add some garbage
if (null != oldChunkedSegmentStorage) {
oldChunkedSegmentStorage.write(oldHandle, oldOffset + 4, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
}
// Save these instances so that you can write some junk after bootstrap.
oldChunkedSegmentStorage = segmentStorageInLoop;
oldHandle = h;
oldOffset = offset;
}
epoch++;
@Cleanup ChunkMetadataStore metadataStoreFinal = getMetadataStore();
ChunkedSegmentStorage segmentStorageFinal = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreFinal, executorService(), config);
cleanupHelper.add(segmentStorageFinal);
segmentStorageFinal.initialize(epoch);
segmentStorageFinal.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
segmentStorageFinal.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorageFinal);
checkSystemSegmentsLayout(segmentStorageFinal);
val info = segmentStorageFinal.getStreamSegmentInfo(systemSegmentName, null).join();
Assert.assertEquals(offset, info.getLength());
Assert.assertEquals(offsetToTruncateAt, info.getStartOffset());
byte[] out = new byte[Math.toIntExact(offset - offsetToTruncateAt)];
val hr = segmentStorageFinal.openRead(systemSegmentName).join();
segmentStorageFinal.read(hr, offsetToTruncateAt, out, 0, Math.toIntExact(offset - offsetToTruncateAt), null).join();
val expected = "Test1Test2Test3Test4Test5Test6Test7Test8Test9";
val actual = new String(out);
Assert.assertEquals(expected, actual);
}
use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.
the class GarbageCollectorTests method testMixedChunk.
/**
* Test for a mix bag of chunks.
*/
@Test
public void testMixedChunk() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
int dataSize = 1;
insertChunk(chunkStorage, "deletedChunk", dataSize);
insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
insertChunk(chunkStorage, "activeChunk", dataSize);
insertChunkMetadata(metadataStore, "activeChunk", dataSize, 1);
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
// Add some garbage
garbageCollector.addChunksToGarbage(TXN_ID, Arrays.asList("activeChunk", "nonExistingChunk", "deletedChunk")).join();
// Validate state before
assertQueueEquals(garbageCollector.getTaskQueueName(), testTaskQueue, new String[] { "activeChunk", "nonExistingChunk", "deletedChunk" });
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 3);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(3, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
Assert.assertFalse(chunkStorage.exists("deletedChunk").get());
Assert.assertTrue(chunkStorage.exists("activeChunk").get());
}
use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.
the class GarbageCollectorTests method testDeletedChunk.
/**
* Test for chunk that is marked inactive and added as garbage.
*/
@Test
public void testDeletedChunk() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
int dataSize = 1;
insertChunk(chunkStorage, "deletedChunk", dataSize);
insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
// Add some garbage
garbageCollector.addChunksToGarbage(TXN_ID, Collections.singleton("deletedChunk")).join();
// Validate state before
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
Assert.assertEquals("deletedChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
Assert.assertFalse(chunkStorage.exists("deletedChunk").get());
Assert.assertNull(getChunkMetadata(metadataStore, "deletedChunk"));
}
use of io.pravega.segmentstore.storage.metadata.ChunkMetadataStore in project pravega by pravega.
the class GarbageCollectorTests method testDeletedChunkMissingFromStorage.
/**
* Test for chunk that is marked inactive, added as garbage but missing from storage.
*/
@Test
public void testDeletedChunkMissingFromStorage() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStore = getMetadataStore();
int containerId = CONTAINER_ID;
int dataSize = 1;
insertChunkMetadata(metadataStore, "deletedChunk", dataSize, 0);
Function<Duration, CompletableFuture<Void>> noDelay = d -> CompletableFuture.completedFuture(null);
val testTaskQueue = new InMemoryTaskQueueManager();
@Cleanup GarbageCollector garbageCollector = new GarbageCollector(containerId, chunkStorage, metadataStore, ChunkedSegmentStorageConfig.DEFAULT_CONFIG.toBuilder().garbageCollectionDelay(Duration.ofMillis(1)).garbageCollectionSleep(Duration.ofMillis(1)).build(), executorService(), System::currentTimeMillis, noDelay);
// Now actually start run
garbageCollector.initialize(testTaskQueue).join();
Assert.assertNotNull(garbageCollector.getTaskQueue());
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
Assert.assertFalse(chunkStorage.exists("deletedChunk").join());
// Add some garbage
garbageCollector.addChunksToGarbage(TXN_ID, Collections.singleton("deletedChunk")).join();
// Validate state before
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
Assert.assertEquals("deletedChunk", testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).peek().getName());
val list = testTaskQueue.drain(garbageCollector.getTaskQueueName(), 1);
Assert.assertEquals(0, testTaskQueue.getTaskQueueMap().get(garbageCollector.getTaskQueueName()).size());
Assert.assertEquals(1, garbageCollector.getQueueSize().get());
garbageCollector.processBatch(list).join();
// Validate state after
Assert.assertEquals(0, garbageCollector.getQueueSize().get());
Assert.assertFalse(chunkStorage.exists("deletedChunk").get());
Assert.assertNull(getChunkMetadata(metadataStore, "deletedChunk"));
}
Aggregations