use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.
the class SystemJournalTests method testSimpleBootstrapWithTwoTruncates.
/**
* Tests a scenario when there are two fail overs.
* The test adds a few chunks to the system segments and then fails over.
* It also truncates system segments.
* The new instance should read the journal log file and recreate the layout of system segments.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSimpleBootstrapWithTwoTruncates() throws Exception {
@Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
@Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
int containerId = 42;
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
long epoch = 1;
val policy = new SegmentRollingPolicy(8);
val config = getDefaultConfigBuilder(policy).build();
long offset = 0;
val data = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
// Epoch 1
@Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreBeforeCrash, executorService(), config);
segmentStorage1.initialize(epoch);
segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
// Bootstrap
segmentStorage1.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage1);
checkSystemSegmentsLayout(segmentStorage1);
// Simulate some writes to system segment, this should cause some new chunks being added.
val h = segmentStorage1.openWrite(systemSegmentName).join();
segmentStorage1.write(h, offset, new ByteArrayInputStream("JUNKJUNKJUNK".getBytes()), 12, null).join();
offset += 12;
val b1 = "Hello".getBytes();
segmentStorage1.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
offset += b1.length;
segmentStorage1.truncate(h, 6, null).join();
// Epoch 2
epoch++;
@Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
segmentStorage2.initialize(epoch);
segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
segmentStorage2.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage2);
checkSystemSegmentsLayout(segmentStorage2);
val h2 = segmentStorage2.openWrite(systemSegmentName).join();
// Write Junk data to both system segment and journal file.
segmentStorage1.write(h, offset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
val b2 = " World".getBytes();
segmentStorage2.write(h2, offset, new ByteArrayInputStream(b2), b2.length, null).join();
offset += b2.length;
segmentStorage2.truncate(h, 12, null).join();
val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
Assert.assertEquals(b1.length + b2.length + 12, info.getLength());
Assert.assertEquals(12, info.getStartOffset());
byte[] out = new byte[b1.length + b2.length];
val hr = segmentStorage2.openRead(systemSegmentName).join();
segmentStorage2.read(hr, 12, out, 0, b1.length + b2.length, null).join();
Assert.assertEquals("Hello World", new String(out));
}
use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.
the class SystemJournalTests method testSimpleBootstrapWithTwoFailovers.
/**
* Tests a scenario when there are two fail overs.
* The test adds a few chunks to the system segments and then fails over.
* After fail over the zombie instance continues to write junk data to both system segment and journal file.
* The new instance should read the journal log file and recreate the layout of system segments.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSimpleBootstrapWithTwoFailovers() throws Exception {
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
@Cleanup ChunkMetadataStore metadataStoreBeforeCrash = getMetadataStore();
@Cleanup ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
int containerId = 42;
String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
long epoch = 1;
val policy = new SegmentRollingPolicy(8);
val config = getDefaultConfigBuilder(policy).build();
val data = new InMemorySnapshotInfoStore();
val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
long offset = 0;
// Epoch 1
@Cleanup ChunkedSegmentStorage segmentStorage1 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreBeforeCrash, executorService(), config);
segmentStorage1.initialize(epoch);
segmentStorage1.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
// Bootstrap
segmentStorage1.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage1);
checkSystemSegmentsLayout(segmentStorage1);
// Simulate some writes to system segment, this should cause some new chunks being added.
val h = segmentStorage1.openWrite(systemSegmentName).join();
val b1 = "Hello".getBytes();
segmentStorage1.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
offset += b1.length;
checkSystemSegmentsLayout(segmentStorage1);
// Epoch 2
epoch++;
@Cleanup ChunkedSegmentStorage segmentStorage2 = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
segmentStorage2.initialize(epoch);
segmentStorage2.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
// Bootstrap
segmentStorage2.bootstrap(snapshotInfoStore, null).join();
deleteGarbage(segmentStorage2);
checkSystemSegmentsLayout(segmentStorage2);
val h2 = segmentStorage2.openWrite(systemSegmentName).join();
// Write Junk Data to from first instance.
segmentStorage1.write(h, offset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
val b2 = " World".getBytes();
segmentStorage2.write(h2, offset, new ByteArrayInputStream(b2), b2.length, null).join();
offset += b2.length;
checkSystemSegmentsLayout(segmentStorage2);
val info = segmentStorage2.getStreamSegmentInfo(systemSegmentName, null).join();
Assert.assertEquals(b1.length + b2.length, info.getLength());
byte[] out = new byte[b1.length + b2.length];
val hr = segmentStorage2.openRead(systemSegmentName).join();
segmentStorage2.read(hr, 0, out, 0, b1.length + b2.length, null).join();
Assert.assertEquals("Hello World", new String(out));
}
use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.
the class SystemJournalTests method testSimpleBootstrapWithMissingSnapshot.
@Test
public void testSimpleBootstrapWithMissingSnapshot() throws Exception {
val containerId = 42;
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
val policy = new SegmentRollingPolicy(100);
val config = getDefaultConfigBuilder(policy).selfCheckEnabled(true).build();
try {
testSimpleBootstrapWithMultipleFailovers(containerId, chunkStorage, config, epoch -> {
val snapShotFile = NameUtils.getSystemJournalSnapshotFileName(containerId, epoch, 1);
chunkStorage.delete(ChunkHandle.writeHandle(snapShotFile)).join();
});
} catch (Exception e) {
val ex = Exceptions.unwrap(e);
Assert.assertTrue(Exceptions.unwrap(e) instanceof IllegalStateException && ex.getMessage().contains("Chunk pointed by SnapshotInfo must exist"));
}
}
use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.
the class RollingStorageTestBase method testWriteOnRollOverBoundary.
@Test
public void testWriteOnRollOverBoundary() throws Exception {
final String segmentName = "Segment";
// Really small rolling length.
final int maxLength = 3;
val seq1 = "01234";
val seq2 = "56789";
val totalWriteLength = seq1.length() + seq2.length();
@Cleanup val s = createStorage();
s.initialize(1);
val writeHandle = s.create(segmentName, new SegmentRollingPolicy(maxLength), TIMEOUT).thenCompose(v -> s.openWrite(segmentName)).join();
val byteInputStream1 = new ByteArrayInputStream(seq1.getBytes());
val byteInputStream2 = new ByteArrayInputStream(seq2.getBytes());
val sequenceInputStream = new SequenceInputStream(byteInputStream1, byteInputStream2);
// This write should cause 3 rollovers.
s.write(writeHandle, 0, sequenceInputStream, totalWriteLength, TIMEOUT).join();
// Check rollover actually happened as expected.
if (useOldLayout) {
RollingSegmentHandle checkHandle = (RollingSegmentHandle) s.openWrite(segmentName).join();
val chunks = checkHandle.chunks();
int numberOfRollovers = totalWriteLength / maxLength;
Assert.assertEquals(numberOfRollovers + 1, chunks.size());
for (int i = 0; i < numberOfRollovers; i++) {
Assert.assertEquals(maxLength * i, chunks.get(i).getStartOffset());
Assert.assertEquals(maxLength, chunks.get(i).getLength());
}
// Last chunk has index == numberOfRollovers, as list is 0 based.
Assert.assertEquals(numberOfRollovers * maxLength, chunks.get(numberOfRollovers).getStartOffset());
Assert.assertEquals(1, chunks.get(numberOfRollovers).getLength());
// Now validate the contents written.
val readHandle = s.openRead(segmentName).join();
byte[] output = new byte[totalWriteLength];
s.read(readHandle, 0, output, 0, totalWriteLength, TIMEOUT).join();
Assert.assertEquals(seq1 + seq2, new String(output));
}
}
use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.
the class ChunkedSegmentStorage method create.
@Override
public CompletableFuture<SegmentHandle> create(String streamSegmentName, SegmentRollingPolicy rollingPolicy, Duration timeout) {
checkInitialized();
return executeSerialized(() -> {
val traceId = LoggerHelpers.traceEnter(log, "create", streamSegmentName, rollingPolicy);
val timer = new Timer();
log.debug("{} create - started segment={}, rollingPolicy={}.", logPrefix, streamSegmentName, rollingPolicy);
return tryWith(metadataStore.beginTransaction(false, streamSegmentName), txn -> {
// Retrieve metadata and make sure it does not exist.
return txn.get(streamSegmentName).thenComposeAsync(storageMetadata -> {
val oldSegmentMetadata = (SegmentMetadata) storageMetadata;
if (null != oldSegmentMetadata) {
throw new CompletionException(new StreamSegmentExistsException(streamSegmentName));
}
// Create a new record.
val newSegmentMetadata = SegmentMetadata.builder().name(streamSegmentName).maxRollinglength(rollingPolicy.getMaxLength() == 0 ? SegmentRollingPolicy.NO_ROLLING.getMaxLength() : rollingPolicy.getMaxLength()).ownerEpoch(this.epoch).build();
newSegmentMetadata.setActive(true);
txn.create(newSegmentMetadata);
// commit.
return txn.commit().thenApplyAsync(v -> {
val retValue = SegmentStorageHandle.writeHandle(streamSegmentName);
Duration elapsed = timer.getElapsed();
SLTS_CREATE_LATENCY.reportSuccessEvent(elapsed);
SLTS_CREATE_COUNT.inc();
log.debug("{} create - finished segment={}, rollingPolicy={}, latency={}.", logPrefix, streamSegmentName, rollingPolicy, elapsed.toMillis());
LoggerHelpers.traceLeave(log, "create", traceId, retValue);
return retValue;
}, executor);
}, executor);
}, executor).handleAsync((v, e) -> {
if (null != e) {
log.debug("{} create - exception segment={}, rollingPolicy={}, latency={}.", logPrefix, streamSegmentName, rollingPolicy, timer.getElapsedMillis(), e);
handleException(streamSegmentName, e);
}
return v;
}, executor);
}, streamSegmentName);
}
Aggregations