use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.
the class StreamSegmentMapperTests method testCreateNewStreamSegment.
/**
* Tests the ability of the StreamSegmentMapper to create a new StreamSegment.
*/
@Test
public void testCreateNewStreamSegment() {
final int segmentCount = 10;
final int transactionsPerSegment = 5;
@Cleanup TestContext context = new TestContext();
HashMap<String, SegmentRollingPolicy> storageSegments = new HashMap<>();
HashMap<String, Integer> expectedRolloverSizes = new HashMap<>();
setupStorageCreateHandler(context, storageSegments);
setupStorageGetHandler(context, storageSegments.keySet(), segmentName -> StreamSegmentInformation.builder().name(segmentName).build());
// Create some Segments and Transaction and verify they are properly created and registered.
for (int i = 0; i < segmentCount; i++) {
String segmentName = getName(i);
val segmentAttributes = createAttributes(ATTRIBUTE_COUNT);
if (i % 2 == 0) {
segmentAttributes.add(new AttributeUpdate(Attributes.ROLLOVER_SIZE, AttributeUpdateType.Replace, i + 100));
expectedRolloverSizes.put(segmentName, i + 100);
}
context.mapper.createNewStreamSegment(segmentName, segmentAttributes, TIMEOUT).join();
assertSegmentCreated(segmentName, segmentAttributes, context);
for (int j = 0; j < transactionsPerSegment; j++) {
val transactionAttributes = createAttributes(ATTRIBUTE_COUNT);
int rolloverSize = j % 2 == 0 ? j + 100 : -1;
if (rolloverSize >= 0) {
transactionAttributes.add(new AttributeUpdate(Attributes.ROLLOVER_SIZE, AttributeUpdateType.Replace, rolloverSize));
}
String transactionName = context.mapper.createNewTransactionStreamSegment(segmentName, UUID.randomUUID(), transactionAttributes, TIMEOUT).join();
if (rolloverSize >= 0) {
expectedRolloverSizes.put(transactionName, rolloverSize);
}
assertSegmentCreated(transactionName, transactionAttributes, context);
}
}
for (val e : storageSegments.entrySet()) {
if (e.getValue() == null) {
Assert.assertFalse("Segment was expected to have a rollover policy defined.", expectedRolloverSizes.containsKey(e.getKey()));
} else {
long expectedValue = expectedRolloverSizes.containsKey(e.getKey()) ? expectedRolloverSizes.get(e.getKey()) : SegmentRollingPolicy.NO_ROLLING.getMaxLength();
Assert.assertEquals("Unexpected rollover policy set.", expectedValue, e.getValue().getMaxLength());
}
}
}
use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.
the class RestoreBackUpDataRecoveryTest method testDurableDataLogFailRecoveryWatermarking.
/**
* Tests the data recovery scenario with watermarking events.
* What test does, step by step:
* 1. Starts Pravega locally with just 4 segment containers.
* 2. Writes {@link #TOTAL_NUM_EVENTS} events to a segment with watermarks.
* 3. Waits for all segments created to be flushed to the long term storage.
* 4. Shuts down the controller, segment store and bookeeper/zookeeper.
* 5. Creates back up of container metadata segment and its attribute segment before deleting them from the Long Term Storage .
* 6. Starts 4 debug segment containers using a new bookeeper/zookeeper and the Long Term Storage.
* 7. Re-creates the container metadata segments in DurableLog and lets them to be flushed to the Long Term Storage.
* 8. Starts segment store and controller.
* 9. Read all events and verify that all events are below the bounds.
* @throws Exception In case of an exception occurred while execution.
*/
@Test(timeout = 180000)
public void testDurableDataLogFailRecoveryWatermarking() throws Exception {
int instanceId = 0;
int bookieCount = 1;
int containerCount = 4;
String readerGroup = "rgTx";
// Creating a long term storage only once here.
this.storageFactory = new InMemoryStorageFactory(executorService());
log.info("Created a long term storage.");
// Start a new BK & ZK, segment store and controller
@Cleanup PravegaRunner pravegaRunner = new PravegaRunner(instanceId++, bookieCount, containerCount, this.storageFactory);
// Create a scope and a stream
createScopeStream(pravegaRunner.controllerRunner.controller, SCOPE, STREAM1);
// Create a client to write events.
@Cleanup ClientRunner clientRunner = new ClientRunner(pravegaRunner.controllerRunner);
// Create a writer
@Cleanup TransactionalEventStreamWriter<Long> writer = clientRunner.clientFactory.createTransactionalEventWriter("writer1", STREAM1, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(TRANSACTION_TIMEOUT.toMillis()).build());
AtomicBoolean stopFlag = new AtomicBoolean(false);
// write events
CompletableFuture<Void> writerFuture = writeTxEvents(writer, stopFlag);
// scale the stream several times so that we get complex positions
StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
Stream streamObj = Stream.of(SCOPE, STREAM1);
scale(pravegaRunner.controllerRunner.controller, streamObj, config);
// get watermarks
LinkedBlockingQueue<Watermark> watermarks = getWatermarks(pravegaRunner, stopFlag, writerFuture);
// Shut down the controller
pravegaRunner.controllerRunner.close();
// Flush DurableLog to Long Term Storage
flushToStorage(pravegaRunner.segmentStoreRunner.serviceBuilder);
// Shutdown SegmentStore
pravegaRunner.segmentStoreRunner.close();
// Shutdown BookKeeper & ZooKeeper
pravegaRunner.bookKeeperRunner.close();
log.info("SegmentStore, BookKeeper & ZooKeeper shutdown");
// Get the long term storage from the running pravega instance
@Cleanup Storage storage = new AsyncStorageWrapper(new RollingStorage(this.storageFactory.createSyncStorage(), new SegmentRollingPolicy(DEFAULT_ROLLING_SIZE)), executorService());
Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).join();
// start a new BookKeeper and ZooKeeper.
pravegaRunner.bookKeeperRunner = new BookKeeperRunner(instanceId++, bookieCount);
createBookKeeperLogFactory();
log.info("Started a new BookKeeper and ZooKeeper.");
// Recover segments
runRecovery(containerCount, storage, backUpMetadataSegments);
// Start a new segment store and controller
pravegaRunner.restartControllerAndSegmentStore(this.storageFactory, this.dataLogFactory);
log.info("Started segment store and controller again.");
// Create the client with new controller.
@Cleanup ClientRunner newClientRunner = new ClientRunner(pravegaRunner.controllerRunner);
// read events and verify
readVerifyEventsWithWatermarks(readerGroup, newClientRunner, streamObj, watermarks);
}
use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.
the class RestoreBackUpDataRecoveryTest method testRecovery.
/**
* Performs a data recovery test(with all segments flushed to the long-term storage beforehand) using the given parameters.
* @param containerCount The number of containers to be used in the pravega instance and the number of debug segment
* containers that will be started.
* @param withTransaction A boolean to indicate weather to write events in the form of transactions or not.
* @throws Exception In case of an exception occurred while execution.
*/
private void testRecovery(int containerCount, int bookieCount, boolean withTransaction) throws Exception {
int instanceId = 0;
// Creating a long term storage only once here.
this.storageFactory = new InMemoryStorageFactory(executorService());
log.info("Created a long term storage.");
// Start a new BK & ZK, segment store and controller
@Cleanup PravegaRunner pravegaRunner = new PravegaRunner(instanceId++, bookieCount, containerCount, this.storageFactory);
// Create a stream for writing data
createScopeStream(pravegaRunner.controllerRunner.controller, SCOPE, STREAM1);
log.info("Created stream '{}'.", STREAM1);
// Create a client to write events.
try (val clientRunner = new ClientRunner(pravegaRunner.controllerRunner)) {
// Write events.
writeEventsToStream(clientRunner.clientFactory, withTransaction);
}
// Shut down the controller
pravegaRunner.controllerRunner.close();
// Flush DurableLog to Long Term Storage
flushToStorage(pravegaRunner.segmentStoreRunner.serviceBuilder);
// Shutdown SegmentStore
pravegaRunner.segmentStoreRunner.close();
// Shutdown BookKeeper & ZooKeeper
pravegaRunner.bookKeeperRunner.close();
log.info("SegmentStore, BookKeeper & ZooKeeper shutdown");
// Get the long term storage from the running pravega instance
@Cleanup Storage storage = new AsyncStorageWrapper(new RollingStorage(this.storageFactory.createSyncStorage(), new SegmentRollingPolicy(DEFAULT_ROLLING_SIZE)), executorService());
Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
// start a new BookKeeper and ZooKeeper.
pravegaRunner.bookKeeperRunner = new BookKeeperRunner(instanceId++, bookieCount);
createBookKeeperLogFactory();
log.info("Started a new BookKeeper and ZooKeeper.");
// Recover segments
runRecovery(containerCount, storage, backUpMetadataSegments);
// Start a new segment store and controller
pravegaRunner.restartControllerAndSegmentStore(this.storageFactory, this.dataLogFactory);
log.info("Started segment store and controller again.");
// Create the client with new controller.
try (val clientRunner = new ClientRunner(pravegaRunner.controllerRunner)) {
// Try reading all events again to verify that the recovery was successful.
readEventsFromStream(clientRunner.clientFactory, clientRunner.readerGroupManager);
log.info("Read all events again to verify that segments were recovered.");
}
}
use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.
the class RestoreBackUpDataRecoveryTest method testDurableDataLogFailRecoveryReadersPaused.
/**
* Tests the data recovery scenario with readers stalling while reading. Readers read some events and then they are
* stopped. Durable data log is erased and restored. It's validated that readers are able to read rest of the unread
* events.
* What test does, step by step:
* 1. Starts Pravega locally with just 4 segment containers.
* 2. Writes {@link #TOTAL_NUM_EVENTS} events.
* 3. Waits for all segments created to be flushed to the long term storage.
* 4. Let a reader read N number of events.
* 5. Shuts down the controller, segment store and bookeeper/zookeeper.
* 6. Creates back up of container metadata segment and its attribute segment before deleting them from the Long Term Storage .
* 7. Starts 4 debug segment containers using a new bookeeper/zookeeper and the Long Term Storage.
* 8. Re-creates the container metadata segments in DurableLog and lets them to be flushed to the Long Term Storage.
* 9. Updates core attributes of segments in the new container metadata segment by using details from the back up of old container metadata segment.
* 10. Starts segment store and controller.
* 11. Let the reader read rest of the 10-N number of events.
* @throws Exception In case of an exception occurred while execution.
*/
@Test(timeout = 180000)
public void testDurableDataLogFailRecoveryReadersPaused() throws Exception {
int instanceId = 0;
int bookieCount = 1;
int containerCount = 4;
int eventsReadCount = RANDOM.nextInt(TOTAL_NUM_EVENTS);
String testReader = "readerDRIntegrationTest";
String testReaderGroup = "readerGroupDRIntegrationTest";
// Creating a long term storage only once here.
this.storageFactory = new InMemoryStorageFactory(executorService());
log.info("Created a long term storage.");
// Start a new BK & ZK, segment store and controller
@Cleanup PravegaRunner pravegaRunner = new PravegaRunner(instanceId++, bookieCount, containerCount, this.storageFactory);
// Create a stream for writing data
createScopeStream(pravegaRunner.controllerRunner.controller, SCOPE, STREAM1);
log.info("Created stream '{}'", STREAM1);
// Create a client to write events.
try (val clientRunner = new ClientRunner(pravegaRunner.controllerRunner)) {
// Write events.
writeEventsToStream(clientRunner.clientFactory, true);
// Create a reader for reading from the stream.
EventStreamReader<String> reader = createReader(clientRunner.clientFactory, clientRunner.readerGroupManager, SCOPE, STREAM1, testReaderGroup, testReader);
// Let reader read N number of events and mark its position.
Position p = readNEvents(reader, eventsReadCount);
ReaderGroup readerGroup = clientRunner.readerGroupManager.getReaderGroup(testReaderGroup);
readerGroup.readerOffline(testReader, p);
}
// Shut down the controller
pravegaRunner.controllerRunner.close();
// Flush DurableLog to Long Term Storage
flushToStorage(pravegaRunner.segmentStoreRunner.serviceBuilder);
// Shutdown SegmentStore
pravegaRunner.segmentStoreRunner.close();
// Shutdown BookKeeper & ZooKeeper
pravegaRunner.bookKeeperRunner.close();
log.info("SegmentStore, BookKeeper & ZooKeeper shutdown");
// Get the long term storage from the running pravega instance
@Cleanup Storage storage = new AsyncStorageWrapper(new RollingStorage(this.storageFactory.createSyncStorage(), new SegmentRollingPolicy(DEFAULT_ROLLING_SIZE)), executorService());
Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).join();
// start a new BookKeeper and ZooKeeper.
pravegaRunner.bookKeeperRunner = new BookKeeperRunner(instanceId++, bookieCount);
createBookKeeperLogFactory();
log.info("Started a new BookKeeper and ZooKeeper.");
// Recover segments
runRecovery(containerCount, storage, backUpMetadataSegments);
// Start a new segment store and controller
pravegaRunner.restartControllerAndSegmentStore(this.storageFactory, this.dataLogFactory);
log.info("Started segment store and controller again.");
// Create the client with new controller.
try (val clientRunner = new ClientRunner(pravegaRunner.controllerRunner)) {
// Get reader group.
ReaderGroup readerGroup = clientRunner.readerGroupManager.getReaderGroup(testReaderGroup);
assertNotNull(readerGroup);
EventStreamReader<String> reader = clientRunner.clientFactory.createReader(testReader, testReaderGroup, new UTF8StringSerializer(), ReaderConfig.builder().build());
// Read the remaining number of events.
readNEvents(reader, TOTAL_NUM_EVENTS - eventsReadCount);
// Reading next event should return null.
assertNull(reader.readNextEvent(READ_TIMEOUT.toMillis()).getEvent());
reader.close();
}
}
use of io.pravega.segmentstore.storage.SegmentRollingPolicy in project pravega by pravega.
the class SystemJournalTests method testSimpleBootstrapWithMultipleFailovers.
/**
* Tests a scenario when there are multiple fail overs overs.
* The test adds a few chunks to the system segments and then fails over.
* After fail over the zombie instances continue to write junk data to both system segment and journal file.
* The new instance should read the journal log file and recreate the layout of system segments.
*
* @throws Exception Throws exception in case of any error.
*/
@Test
public void testSimpleBootstrapWithMultipleFailovers() throws Exception {
val containerId = 42;
@Cleanup ChunkStorage chunkStorage = getChunkStorage();
val policy = new SegmentRollingPolicy(100);
val config = getDefaultConfigBuilder(policy).selfCheckEnabled(true).build();
testSimpleBootstrapWithMultipleFailovers(containerId, chunkStorage, config, null);
}
Aggregations