Examples with UpdateableContainerMetadata io.pravega.segmentstore.server.UpdateableContainerMetadata used on opensource projects

Example 31 with UpdateableContainerMetadata

use of io.pravega.segmentstore.server.UpdateableContainerMetadata in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testAcceptStreamSegmentSeal.

/**
 * Tests the accept method with StreamSegmentSeal operations.
 */
@Test
public void testAcceptStreamSegmentSeal() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    // Set some attributes.
    val segmentAttributes = createAttributes();
    segmentAttributes.put(Attributes.CREATION_TIME, 1L);
    UpdateableSegmentMetadata segmentMetadata = metadata.getStreamSegmentMetadata(SEGMENT_ID);
    segmentMetadata.updateAttributes(segmentAttributes);
    val txn = createUpdateTransaction(metadata);
    StreamSegmentSealOperation sealOp = createSeal();
    // When no pre-process has happened.
    AssertExtensions.assertThrows("Unexpected behavior from acceptOperation() when no pre-processing was made.", () -> txn.acceptOperation(sealOp), ex -> ex instanceof MetadataUpdateException);
    Assert.assertFalse("acceptOperation updated the transaction even if it threw an exception.", txn.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    Assert.assertFalse("acceptOperation updated the metadata.", metadata.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    // When all is good.
    txn.preProcessOperation(sealOp);
    txn.acceptOperation(sealOp);
    Assert.assertTrue("acceptOperation did not update the transaction.", txn.getStreamSegmentMetadata(SEGMENT_ID).isSealed());
    Assert.assertFalse("acceptOperation updated the metadata.", segmentMetadata.isSealed());
    txn.commit(metadata);
    // Check attributes.
    // All dynamic attributes should be removed.
    segmentAttributes.keySet().removeIf(Attributes::isDynamic);
    SegmentMetadataComparer.assertSameAttributes("Unexpected set of attributes after commit.", segmentAttributes, segmentMetadata);
}

Example 32 with UpdateableContainerMetadata

use of io.pravega.segmentstore.server.UpdateableContainerMetadata in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessTransactionMap.

/**
 * Tests the processOperation and acceptOperation methods with TransactionMap operations.
 */
@Test
public void testPreProcessTransactionMap() throws Exception {
    UpdateableContainerMetadata metadata = createBlankMetadata();
    val txn1 = createUpdateTransaction(metadata);
    // Parent does not exist.
    AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when attempting to map a Transaction StreamSegment to an inexistent parent.", () -> txn1.preProcessOperation(createTransactionMap(12345)), ex -> ex instanceof MetadataUpdateException);
    // Brand new Transaction (and parent).
    StreamSegmentMapOperation mapParent = createMap();
    // Create parent.
    txn1.preProcessOperation(mapParent);
    txn1.acceptOperation(mapParent);
    txn1.commit(metadata);
    // Part 1: recovery mode.
    metadata.enterRecoveryMode();
    StreamSegmentMapOperation mapOp = createTransactionMap(mapParent.getStreamSegmentId());
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(mapOp);
    Assert.assertEquals("preProcessOperation changed the StreamSegmentId on the operation in recovery mode.", ContainerMetadata.NO_STREAM_SEGMENT_ID, mapOp.getStreamSegmentId());
    // Part 2: non-recovery mode.
    metadata.exitRecoveryMode();
    val txn3 = createUpdateTransaction(metadata);
    txn3.preProcessOperation(mapOp);
    Assert.assertNotEquals("preProcessOperation did not set the StreamSegmentId on the operation.", ContainerMetadata.NO_STREAM_SEGMENT_ID, mapOp.getStreamSegmentId());
    Assert.assertNull("preProcessOperation modified the current transaction.", txn3.getStreamSegmentMetadata(mapOp.getStreamSegmentId()));
    Assert.assertNull("preProcessOperation modified the underlying metadata.", metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId()));
    txn3.acceptOperation(mapOp);
    val updaterMetadata = txn3.getStreamSegmentMetadata(mapOp.getStreamSegmentId());
    Assert.assertEquals("Unexpected StorageLength after call to processMetadataOperation (in transaction).", mapOp.getLength(), updaterMetadata.getStorageLength());
    Assert.assertEquals("Unexpected Length after call to processMetadataOperation (in transaction).", mapOp.getLength(), updaterMetadata.getLength());
    Assert.assertEquals("Unexpected value for isSealed after call to processMetadataOperation (in transaction).", mapOp.isSealed(), updaterMetadata.isSealed());
    Assert.assertEquals("Unexpected value for StartOffset after call to processMetadataOperation (in transaction).", 0, updaterMetadata.getStartOffset());
    Assert.assertNull("processMetadataOperation modified the underlying metadata.", metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId()));
    // Transaction StreamSegmentName exists (transaction).
    AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when a TransactionStreamSegment with the same Name already exists (in transaction).", () -> txn3.preProcessOperation(createTransactionMap(mapParent.getStreamSegmentId(), mapOp.getStreamSegmentName())), ex -> ex instanceof MetadataUpdateException);
    // Make changes permanent.
    txn3.commit(metadata);
    val segmentMetadata = metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId());
    AssertExtensions.assertMapEquals("Unexpected attributes in SegmentMetadata after call to commit().", mapOp.getAttributes(), segmentMetadata.getAttributes());
    // Transaction StreamSegmentName exists (metadata).
    AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when a TransactionStreamSegment with the same Name already exists (in metadata).", () -> txn3.preProcessOperation(createTransactionMap(mapParent.getStreamSegmentId(), mapOp.getStreamSegmentName())), ex -> ex instanceof MetadataUpdateException);
    // StreamSegmentName already exists and we try to map with the same id. Verify that we are able to update its
    // StorageLength (if different).
    val updateMap = new StreamSegmentMapOperation(mapOp.getParentStreamSegmentId(), StreamSegmentInformation.builder().name(mapOp.getStreamSegmentName()).startOffset(// Purposefully setting this wrong to see if it is auto-corrected.
    1).length(mapOp.getLength() + 1).sealed(true).attributes(createAttributes()).build());
    updateMap.setStreamSegmentId(mapOp.getStreamSegmentId());
    txn3.preProcessOperation(updateMap);
    txn3.acceptOperation(updateMap);
    Assert.assertEquals("Unexpected StorageLength after call to acceptOperation with remap (in transaction).", updateMap.getLength(), txn3.getStreamSegmentMetadata(mapOp.getStreamSegmentId()).getLength());
    txn3.commit(metadata);
    Assert.assertEquals("Unexpected StorageLength after call to acceptOperation with remap (post-commit).", updateMap.getLength(), metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId()).getLength());
    Assert.assertEquals("Unexpected StartOffset after call to acceptOperation with remap (post-commit).", 0, metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId()).getStartOffset());
}

Example 33 with UpdateableContainerMetadata

use of io.pravega.segmentstore.server.UpdateableContainerMetadata in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method getCheckpointedMetadata.

private UpdateableContainerMetadata getCheckpointedMetadata(MetadataCheckpointOperation operation) throws Exception {
    UpdateableContainerMetadata metadata = createBlankMetadata();
    metadata.enterRecoveryMode();
    val txn = createUpdateTransaction(metadata);
    processOperation(operation, txn, () -> 1L);
    txn.commit(metadata);
    return metadata;
}

Example 34 with UpdateableContainerMetadata

use of io.pravega.segmentstore.server.UpdateableContainerMetadata in project pravega by pravega.

the class DurableLogTests method testTruncateWithRecovery.

/**
 * Tests the truncate() method while performing recovery.
 */
@Test
public void testTruncateWithRecovery() {
    int streamSegmentCount = 50;
    int appendsPerStreamSegment = 20;
    // Setup a DurableLog and start it.
    AtomicReference<TestDurableDataLog> dataLog = new AtomicReference<>();
    AtomicReference<Boolean> truncationOccurred = new AtomicReference<>();
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), dataLog::set);
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    UpdateableContainerMetadata metadata = new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup InMemoryCacheFactory cacheFactory = new InMemoryCacheFactory();
    @Cleanup CacheManager cacheManager = new CacheManager(DEFAULT_READ_INDEX_CONFIG.getCachePolicy(), executorService());
    @Cleanup ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata, cacheFactory, storage, cacheManager, executorService());
    HashSet<Long> streamSegmentIds;
    List<OperationWithCompletion> completionFutures;
    List<Operation> originalOperations;
    // First DurableLog. We use this for generating data.
    try (DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Generate some test data (we need to do this after we started the DurableLog because in the process of
        // recovery, it wipes away all existing metadata).
        streamSegmentIds = createStreamSegmentsWithOperations(streamSegmentCount, metadata, durableLog, storage);
        List<Operation> queuedOperations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
        completionFutures = processOperations(queuedOperations, durableLog);
        OperationWithCompletion.allOf(completionFutures).join();
        // Get a list of all the operations, before any truncation.
        originalOperations = readUpToSequenceNumber(durableLog, metadata.getOperationSequenceNumber());
        // Stop the processor.
        durableLog.stopAsync().awaitTerminated();
    }
    // Truncate up to each MetadataCheckpointOperation and:
    // * If the DataLog was truncated:
    // ** Shut down DurableLog, re-start it (recovery) and verify the operations are as they should.
    // At the end, verify all operations and all entries in the DataLog were truncated.
    DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService());
    try {
        durableLog.startAsync().awaitRunning();
        dataLog.get().setTruncateCallback(seqNo -> truncationOccurred.set(true));
        for (int i = 0; i < originalOperations.size(); i++) {
            Operation currentOperation = originalOperations.get(i);
            if (!(currentOperation instanceof MetadataCheckpointOperation)) {
                // We can only truncate on MetadataCheckpointOperations.
                continue;
            }
            truncationOccurred.set(false);
            durableLog.truncate(currentOperation.getSequenceNumber(), TIMEOUT).join();
            if (truncationOccurred.get()) {
                // Close current DurableLog and start a brand new one, forcing recovery.
                durableLog.close();
                durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata, dataLogFactory, readIndex, executorService());
                durableLog.startAsync().awaitRunning();
                dataLog.get().setTruncateCallback(seqNo -> truncationOccurred.set(true));
                // Verify all operations up to, and including this one have been removed.
                Iterator<Operation> reader = durableLog.read(-1, 2, TIMEOUT).join();
                Assert.assertTrue("Not expecting an empty log after truncating an operation (a MetadataCheckpoint must always exist).", reader.hasNext());
                verifyFirstItemIsMetadataCheckpoint(reader);
                if (i < originalOperations.size() - 1) {
                    Operation firstOp = reader.next();
                    OperationComparer.DEFAULT.assertEquals(String.format("Unexpected first operation after truncating SeqNo %d.", currentOperation.getSequenceNumber()), originalOperations.get(i + 1), firstOp);
                }
            }
        }
    } finally {
        // This closes whatever current instance this variable refers to, not necessarily the first one.
        durableLog.close();
    }
}

Example 35 with UpdateableContainerMetadata

use of io.pravega.segmentstore.server.UpdateableContainerMetadata in project pravega by pravega.

the class MemoryStateUpdaterTests method testRecoveryMode.

/**
 * Tests the ability of the MemoryStateUpdater to delegate Enter/Exit recovery mode to the read index.
 */
@Test
public void testRecoveryMode() throws Exception {
    // Check it's properly delegated to Read index.
    SequencedItemList<Operation> opLog = new SequencedItemList<>();
    ArrayList<TestReadIndex.MethodInvocation> methodInvocations = new ArrayList<>();
    TestReadIndex readIndex = new TestReadIndex(methodInvocations::add);
    MemoryStateUpdater updater = new MemoryStateUpdater(opLog, readIndex, Runnables.doNothing());
    UpdateableContainerMetadata metadata1 = new MetadataBuilder(1).build();
    updater.enterRecoveryMode(metadata1);
    updater.exitRecoveryMode(true);
    Assert.assertEquals("Unexpected number of method invocations.", 2, methodInvocations.size());
    TestReadIndex.MethodInvocation enterRecovery = methodInvocations.get(0);
    Assert.assertEquals("ReadIndex.enterRecoveryMode was not called when expected.", TestReadIndex.ENTER_RECOVERY_MODE, enterRecovery.methodName);
    Assert.assertEquals("ReadIndex.enterRecoveryMode was called with the wrong arguments.", metadata1, enterRecovery.args.get("recoveryMetadataSource"));
    TestReadIndex.MethodInvocation exitRecovery = methodInvocations.get(1);
    Assert.assertEquals("ReadIndex.exitRecoveryMode was not called when expected.", TestReadIndex.EXIT_RECOVERY_MODE, exitRecovery.methodName);
    Assert.assertEquals("ReadIndex.exitRecoveryMode was called with the wrong arguments.", true, exitRecovery.args.get("successfulRecovery"));
}