use of io.pravega.segmentstore.storage.mocks.InMemorySnapshotInfoStore in project pravega by pravega.
the class SystemJournalTests method testSimpleTruncation.
/**
* Test simple chunk truncation.
* We failover two times to test correct interaction between snapshot and system logs.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSimpleTruncation() throws Exception {
@Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
int containerId = 42;
val data = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
@Cleanup val garbageCollector1 = new GarbageCollector(containerId, chunkStorage, metadataStoreBeforeCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector1.initialize(new InMemoryTaskQueueManager()).join();
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
long epoch = 1;
val policy = new SegmentRollingPolicy(2);
val config = getDefaultConfigBuilder(policy).build();
// Step 1: Initial set of additions
SystemJournal systemJournalBefore = new SystemJournal(containerId, chunkStorage, metadataStoreBeforeCrash, garbageCollector1, config, executorService());
systemJournalBefore.bootstrap(epoch, snapshotInfoStore).join();
String lastChunk = null;
long totalBytesWritten = 0;
for (int i = 0; i < 10; i++) {
String newChunk = "chunk" + i;
val h = chunkStorage.createWithContent(newChunk, Math.toIntExact(policy.getMaxLength()), new ByteArrayInputStream(new byte[Math.toIntExact(policy.getMaxLength())])).get();
totalBytesWritten += policy.getMaxLength();
systemJournalBefore.commitRecord(SystemJournal.ChunkAddedRecord.builder().segmentName(systemSegmentName).offset(policy.getMaxLength() * i).newChunkName(newChunk).oldChunkName(lastChunk).build()).join();
lastChunk = newChunk;
}
Assert.assertEquals(policy.getMaxLength() * 10, totalBytesWritten);
// Step 2: First failover, truncate first 5 chunks.
@Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
@Cleanup val garbageCollector2 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector2.initialize(new InMemoryTaskQueueManager()).join();
SystemJournal systemJournalAfter = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash, garbageCollector2, config, executorService());
systemJournalAfter.bootstrap(2, snapshotInfoStore).join();
TestUtils.checkSegmentLayout(metadataStoreAfterCrash, systemSegmentName, policy.getMaxLength(), 10);
TestUtils.checkSegmentBounds(metadataStoreAfterCrash, systemSegmentName, 0, totalBytesWritten);
// Truncate first five chunks
for (int i = 0; i <= 10; i++) {
val firstChunkIndex = i / policy.getMaxLength();
systemJournalAfter.commitRecord(SystemJournal.TruncationRecord.builder().segmentName(systemSegmentName).offset(i).firstChunkName("chunk" + firstChunkIndex).startOffset(policy.getMaxLength() * firstChunkIndex).build()).join();
}
// Step 3: Second failover, truncate last 5 chunks.
@Cleanup ChunkMetadataStore metadataStoreAfterCrash2 = getMetadataStore();
@Cleanup val garbageCollector3 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash2, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector3.initialize(new InMemoryTaskQueueManager()).join();
SystemJournal systemJournalAfter2 = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash2, garbageCollector3, config, executorService());
systemJournalAfter2.bootstrap(3, snapshotInfoStore).join();
TestUtils.checkSegmentLayout(metadataStoreAfterCrash2, systemSegmentName, policy.getMaxLength(), 5);
TestUtils.checkSegmentBounds(metadataStoreAfterCrash2, systemSegmentName, 10, 20);
// Truncate last five chunks
for (int i = 10; i <= 20; i++) {
val firstChunkIndex = i / policy.getMaxLength();
systemJournalAfter2.commitRecord(SystemJournal.TruncationRecord.builder().segmentName(systemSegmentName).offset(i).firstChunkName("chunk" + firstChunkIndex).startOffset(policy.getMaxLength() * firstChunkIndex).build()).join();
}
// Step 4: third failover validate.
@Cleanup ChunkMetadataStore metadataStoreAfterCrash3 = getMetadataStore();
@Cleanup val garbageCollector4 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash2, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector4.initialize(new InMemoryTaskQueueManager()).join();
SystemJournal systemJournalAfter3 = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash3, garbageCollector4, config, executorService());
systemJournalAfter3.bootstrap(4, snapshotInfoStore).join();
TestUtils.checkSegmentBounds(metadataStoreAfterCrash3, systemSegmentName, 20, 20);
}
use of io.pravega.segmentstore.storage.mocks.InMemorySnapshotInfoStore in project pravega by pravega.
the class SystemJournalTests method testBootstrapWithTruncate.
/**
* Tests two failures with truncates and zombie stores writing junk data.
*
* @param initialGarbageThatIsTruncated Garbage to write that is later truncated away.
* @param garbageAfterFailure Garbage to write from the first zombie instance.
* @param validWriteBeforeFailure First part of the valid data written before the failure.
* @param validWriteAfterFailure Second part of the valid data written after the failure.
* @param maxLength Max length for the segment policy.
* @throws Exception Throws exception in case of any error.
*/
private void testBootstrapWithTruncate(String initialGarbageThatIsTruncated, String garbageAfterFailure, String validWriteBeforeFailure, String validWriteAfterFailure, int maxLength) throws Exception {
@Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
@Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
int containerId = 42;
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
long epoch = 1;
val policy = new SegmentRollingPolicy(maxLength);
val config = getDefaultConfigBuilder(policy).build();
val data = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
long offset = 0;
// Epoch 1
@Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreBeforeCrash, executorService(), config);
segmentStorage1.initialize(epoch);
segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
// Bootstrap
segmentStorage1.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage1);
checkSystemSegmentsLayout(segmentStorage1);
// Simulate some writes to system segment, this should cause some new chunks being added.
val h = segmentStorage1.openWrite(systemSegmentName).join();
val b1 = validWriteBeforeFailure.getBytes();
byte[] garbage1 = initialGarbageThatIsTruncated.getBytes();
int garbage1Length = garbage1.length;
segmentStorage1.write(h, offset, new ByteArrayInputStream(garbage1), garbage1Length, null).join();
offset += garbage1Length;
segmentStorage1.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
offset += b1.length;
segmentStorage1.truncate(h, garbage1Length, null).join();
// Epoch 2
epoch++;
@Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
segmentStorage2.initialize(epoch);
segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
segmentStorage2.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage2);
checkSystemSegmentsLayout(segmentStorage2);
val h2 = segmentStorage2.openWrite(systemSegmentName).join();
// Write Junk Data to both system segment and journal file.
val innerGarbageBytes = garbageAfterFailure.getBytes();
segmentStorage1.write(h, offset, new ByteArrayInputStream(innerGarbageBytes), innerGarbageBytes.length, null).join();
val b2 = validWriteAfterFailure.getBytes();
segmentStorage2.write(h2, offset, new ByteArrayInputStream(b2), b2.length, null).join();
offset += b2.length;
val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
Assert.assertEquals(b1.length + b2.length + garbage1Length, info.getLength());
Assert.assertEquals(garbage1Length, info.getStartOffset());
byte[] out = new byte[b1.length + b2.length];
val hr = segmentStorage2.openRead(systemSegmentName).join();
segmentStorage2.read(hr, garbage1Length, out, 0, b1.length + b2.length, null).join();
Assert.assertEquals(validWriteBeforeFailure + validWriteAfterFailure, new String(out));
}
use of io.pravega.segmentstore.storage.mocks.InMemorySnapshotInfoStore in project pravega by pravega.
the class SystemJournalTests method testSimpleBootstrapWithOneFailover.
/**
* Tests a scenario when there is only one fail over.
* The test adds a few chunks to the system segments and then fails over.
* The new instance should read the journal log file and recreate the layout of system segments.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSimpleBootstrapWithOneFailover() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
@Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
int containerId = 42;
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
int maxLength = 8;
long epoch = 1;
val policy = new SegmentRollingPolicy(maxLength);
val config = getDefaultConfigBuilder(policy).build();
val data = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
// Init
long offset = 0;
// Start container with epoch 1
@Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreBeforeCrash, executorService(), config);
segmentStorage1.initialize(epoch);
segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
// Bootstrap
segmentStorage1.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage1);
checkSystemSegmentsLayout(segmentStorage1);
// Simulate some writes to system segment, this should cause some new chunks being added.
val h = segmentStorage1.openWrite(systemSegmentName).join();
val b1 = "Hello".getBytes();
segmentStorage1.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
offset += b1.length;
val b2 = " World".getBytes();
segmentStorage1.write(h, offset, new ByteArrayInputStream(b2), b2.length, null).join();
offset += b2.length;
// Step 2
// Start container with epoch 2
epoch++;
@Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
segmentStorage2.initialize(epoch);
segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
// Bootstrap
segmentStorage2.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage2);
checkSystemSegmentsLayout(segmentStorage2);
// Validate
val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
Assert.assertEquals(b1.length + b2.length, info.getLength());
byte[] out = new byte[b1.length + b2.length];
val hr = segmentStorage2.openRead(systemSegmentName).join();
segmentStorage2.read(hr, 0, out, 0, b1.length + b2.length, null).join();
Assert.assertEquals("Hello World", new String(out));
}
use of io.pravega.segmentstore.storage.mocks.InMemorySnapshotInfoStore in project pravega by pravega.
the class SystemJournalTests method testSimpleChunkAddition.
/**
* Test simple chunk addition.
* We failover two times to test correct interaction between snapshot and system logs.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSimpleChunkAddition() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
int containerId = 42;
val data = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
@Cleanup val garbageCollector1 = new GarbageCollector(containerId, chunkStorage, metadataStoreBeforeCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector1.initialize(new InMemoryTaskQueueManager()).join();
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
long epoch = 1;
val policy = new SegmentRollingPolicy(2);
val config = getDefaultConfigBuilder(policy).build();
// Inital set of additions
SystemJournal systemJournalBefore = new SystemJournal(containerId, chunkStorage, metadataStoreBeforeCrash, garbageCollector1, config, executorService());
systemJournalBefore.bootstrap(epoch, snapshotInfoStore).join();
String lastChunk = null;
long totalBytesWritten = 0;
for (int i = 0; i < 10; i++) {
String newChunk = "chunk" + i;
val h = chunkStorage.createWithContent(newChunk, Math.toIntExact(policy.getMaxLength()), new ByteArrayInputStream(new byte[Math.toIntExact(policy.getMaxLength())])).get();
totalBytesWritten += policy.getMaxLength();
systemJournalBefore.commitRecord(SystemJournal.ChunkAddedRecord.builder().segmentName(systemSegmentName).offset(policy.getMaxLength() * i).newChunkName(newChunk).oldChunkName(lastChunk).build()).join();
lastChunk = newChunk;
}
Assert.assertEquals(policy.getMaxLength() * 10, totalBytesWritten);
// Failover
@Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
@Cleanup val garbageCollector2 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector2.initialize(new InMemoryTaskQueueManager()).join();
SystemJournal systemJournalAfter = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash, garbageCollector2, config, executorService());
systemJournalAfter.bootstrap(epoch + 1, snapshotInfoStore).join();
TestUtils.checkSegmentLayout(metadataStoreAfterCrash, systemSegmentName, policy.getMaxLength(), 10);
TestUtils.checkSegmentBounds(metadataStoreAfterCrash, systemSegmentName, 0, totalBytesWritten);
@Cleanup ChunkMetadataStore metadataStoreAfterCrash2 = getMetadataStore();
@Cleanup val garbageCollector3 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash2, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector3.initialize(new InMemoryTaskQueueManager()).join();
SystemJournal systemJournalAfter2 = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash2, garbageCollector3, config, executorService());
systemJournalAfter2.bootstrap(epoch + 2, snapshotInfoStore).join();
TestUtils.checkSegmentLayout(metadataStoreAfterCrash2, systemSegmentName, policy.getMaxLength(), 10);
}
use of io.pravega.segmentstore.storage.mocks.InMemorySnapshotInfoStore in project pravega by pravega.
the class SystemJournalTests method testSimpleOperationSequence.
/**
* Test simple series of operations.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSimpleOperationSequence() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
int containerId = 42;
val data = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
@Cleanup val garbageCollector1 = new GarbageCollector(containerId, chunkStorage, metadataStoreBeforeCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector1.initialize(new InMemoryTaskQueueManager()).join();
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
long epoch = 1;
val policy = new SegmentRollingPolicy(2);
val config = getDefaultConfigBuilder(policy).build();
// Inital set of additions
SystemJournal systemJournalBefore = new SystemJournal(containerId, chunkStorage, metadataStoreBeforeCrash, garbageCollector1, config, executorService());
systemJournalBefore.bootstrap(epoch, snapshotInfoStore).join();
String lastChunk = null;
long totalBytesWritten = 0;
for (int i = 0; i < 10; i++) {
String newChunk = "chunk" + i;
val h = chunkStorage.createWithContent(newChunk, Math.toIntExact(policy.getMaxLength()), new ByteArrayInputStream(new byte[Math.toIntExact(policy.getMaxLength())])).get();
totalBytesWritten += policy.getMaxLength();
systemJournalBefore.commitRecord(SystemJournal.ChunkAddedRecord.builder().segmentName(systemSegmentName).offset(policy.getMaxLength() * i).newChunkName(newChunk).oldChunkName(lastChunk).build()).join();
lastChunk = newChunk;
}
Assert.assertEquals(policy.getMaxLength() * 10, totalBytesWritten);
// Failover
@Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
@Cleanup val garbageCollector2 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector2.initialize(new InMemoryTaskQueueManager()).join();
SystemJournal systemJournalAfter = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash, garbageCollector2, config, executorService());
systemJournalAfter.bootstrap(epoch + 1, snapshotInfoStore).join();
TestUtils.checkSegmentLayout(metadataStoreAfterCrash, systemSegmentName, policy.getMaxLength(), 10);
TestUtils.checkSegmentBounds(metadataStoreAfterCrash, systemSegmentName, 0, totalBytesWritten);
@Cleanup ChunkMetadataStore metadataStoreAfterCrash2 = getMetadataStore();
@Cleanup val garbageCollector3 = new GarbageCollector(containerId, chunkStorage, metadataStoreAfterCrash2, ChunkedSegmentStorageConfig.DEFAULT_CONFIG, executorService());
garbageCollector3.initialize(new InMemoryTaskQueueManager()).join();
SystemJournal systemJournalAfter2 = new SystemJournal(containerId, chunkStorage, metadataStoreAfterCrash2, garbageCollector3, config, executorService());
systemJournalAfter2.bootstrap(epoch + 2, snapshotInfoStore).join();
TestUtils.checkSegmentLayout(metadataStoreAfterCrash2, systemSegmentName, policy.getMaxLength(), 10);
}
Aggregations