Examples with StreamSegmentMapOperation io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation used on opensource projects

Example 11 with StreamSegmentMapOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testSegmentMapMax.

/**
 * Tests the ability to reject new StreamSegment/Transaction map operations that would exceed the max allowed counts.
 */
@Test
public void testSegmentMapMax() throws Exception {
    UpdateableContainerMetadata metadata = new MetadataBuilder(CONTAINER_ID).withMaxActiveSegmentCount(3).build();
    metadata.mapStreamSegmentId("a", SEGMENT_ID);
    metadata.mapStreamSegmentId("a_txn1", 123457, SEGMENT_ID);
    // Non-recovery mode.
    val txn1 = createUpdateTransaction(metadata);
    // Map one segment, which should fill up the quota.
    StreamSegmentMapOperation acceptedMap = createMap();
    txn1.preProcessOperation(acceptedMap);
    txn1.acceptOperation(acceptedMap);
    // Verify non-recovery mode.
    AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when attempting to map a StreamSegment that would exceed the active segment quota.", () -> txn1.preProcessOperation(createMap("foo")), ex -> ex instanceof TooManyActiveSegmentsException);
    AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when attempting to map a StreamSegment that would exceed the active segment quota.", () -> txn1.preProcessOperation(createTransactionMap(SEGMENT_ID, "foo")), ex -> ex instanceof TooManyActiveSegmentsException);
    // Verify recovery mode.
    metadata.enterRecoveryMode();
    val txn2 = createUpdateTransaction(metadata);
    // updater.setOperationSequenceNumber(10000);
    StreamSegmentMapOperation secondMap = createMap("c");
    secondMap.setStreamSegmentId(1234);
    txn2.preProcessOperation(secondMap);
    txn2.acceptOperation(secondMap);
    StreamSegmentMapOperation secondTxnMap = createTransactionMap(SEGMENT_ID, "a_txn2");
    secondTxnMap.setStreamSegmentId(1235);
    txn2.preProcessOperation(secondTxnMap);
    txn2.acceptOperation(secondTxnMap);
    txn2.commit(metadata);
    metadata.exitRecoveryMode();
    Assert.assertNotNull("Updater did not create metadata for new segment in recovery mode even if quota is exceeded.", metadata.getStreamSegmentMetadata(secondMap.getStreamSegmentId()));
    Assert.assertNotNull("Updater did not create metadata for new transaction in recovery mode even if quota is exceeded.", metadata.getStreamSegmentMetadata(secondTxnMap.getStreamSegmentId()));
}

Example 12 with StreamSegmentMapOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testPreProcessTransactionMap.

/**
 * Tests the processOperation and acceptOperation methods with TransactionMap operations.
 */
@Test
public void testPreProcessTransactionMap() throws Exception {
    UpdateableContainerMetadata metadata = createBlankMetadata();
    val txn1 = createUpdateTransaction(metadata);
    // Parent does not exist.
    AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when attempting to map a Transaction StreamSegment to an inexistent parent.", () -> txn1.preProcessOperation(createTransactionMap(12345)), ex -> ex instanceof MetadataUpdateException);
    // Brand new Transaction (and parent).
    StreamSegmentMapOperation mapParent = createMap();
    // Create parent.
    txn1.preProcessOperation(mapParent);
    txn1.acceptOperation(mapParent);
    txn1.commit(metadata);
    // Part 1: recovery mode.
    metadata.enterRecoveryMode();
    StreamSegmentMapOperation mapOp = createTransactionMap(mapParent.getStreamSegmentId());
    val txn2 = createUpdateTransaction(metadata);
    txn2.preProcessOperation(mapOp);
    Assert.assertEquals("preProcessOperation changed the StreamSegmentId on the operation in recovery mode.", ContainerMetadata.NO_STREAM_SEGMENT_ID, mapOp.getStreamSegmentId());
    // Part 2: non-recovery mode.
    metadata.exitRecoveryMode();
    val txn3 = createUpdateTransaction(metadata);
    txn3.preProcessOperation(mapOp);
    Assert.assertNotEquals("preProcessOperation did not set the StreamSegmentId on the operation.", ContainerMetadata.NO_STREAM_SEGMENT_ID, mapOp.getStreamSegmentId());
    Assert.assertNull("preProcessOperation modified the current transaction.", txn3.getStreamSegmentMetadata(mapOp.getStreamSegmentId()));
    Assert.assertNull("preProcessOperation modified the underlying metadata.", metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId()));
    txn3.acceptOperation(mapOp);
    val updaterMetadata = txn3.getStreamSegmentMetadata(mapOp.getStreamSegmentId());
    Assert.assertEquals("Unexpected StorageLength after call to processMetadataOperation (in transaction).", mapOp.getLength(), updaterMetadata.getStorageLength());
    Assert.assertEquals("Unexpected Length after call to processMetadataOperation (in transaction).", mapOp.getLength(), updaterMetadata.getLength());
    Assert.assertEquals("Unexpected value for isSealed after call to processMetadataOperation (in transaction).", mapOp.isSealed(), updaterMetadata.isSealed());
    Assert.assertEquals("Unexpected value for StartOffset after call to processMetadataOperation (in transaction).", 0, updaterMetadata.getStartOffset());
    Assert.assertNull("processMetadataOperation modified the underlying metadata.", metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId()));
    // Transaction StreamSegmentName exists (transaction).
    AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when a TransactionStreamSegment with the same Name already exists (in transaction).", () -> txn3.preProcessOperation(createTransactionMap(mapParent.getStreamSegmentId(), mapOp.getStreamSegmentName())), ex -> ex instanceof MetadataUpdateException);
    // Make changes permanent.
    txn3.commit(metadata);
    val segmentMetadata = metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId());
    AssertExtensions.assertMapEquals("Unexpected attributes in SegmentMetadata after call to commit().", mapOp.getAttributes(), segmentMetadata.getAttributes());
    // Transaction StreamSegmentName exists (metadata).
    AssertExtensions.assertThrows("Unexpected behavior from preProcessOperation when a TransactionStreamSegment with the same Name already exists (in metadata).", () -> txn3.preProcessOperation(createTransactionMap(mapParent.getStreamSegmentId(), mapOp.getStreamSegmentName())), ex -> ex instanceof MetadataUpdateException);
    // StreamSegmentName already exists and we try to map with the same id. Verify that we are able to update its
    // StorageLength (if different).
    val updateMap = new StreamSegmentMapOperation(mapOp.getParentStreamSegmentId(), StreamSegmentInformation.builder().name(mapOp.getStreamSegmentName()).startOffset(// Purposefully setting this wrong to see if it is auto-corrected.
    1).length(mapOp.getLength() + 1).sealed(true).attributes(createAttributes()).build());
    updateMap.setStreamSegmentId(mapOp.getStreamSegmentId());
    txn3.preProcessOperation(updateMap);
    txn3.acceptOperation(updateMap);
    Assert.assertEquals("Unexpected StorageLength after call to acceptOperation with remap (in transaction).", updateMap.getLength(), txn3.getStreamSegmentMetadata(mapOp.getStreamSegmentId()).getLength());
    txn3.commit(metadata);
    Assert.assertEquals("Unexpected StorageLength after call to acceptOperation with remap (post-commit).", updateMap.getLength(), metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId()).getLength());
    Assert.assertEquals("Unexpected StartOffset after call to acceptOperation with remap (post-commit).", 0, metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId()).getStartOffset());
}

Example 13 with StreamSegmentMapOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation in project pravega by pravega.

the class DurableLogTests method testRecoveryWithMetadataCleanup.

/**
 * Tests the following recovery scenario:
 * 1. A Segment is created and recorded in the metadata with some optional operations executing on it.
 * 2. The segment is evicted from the metadata.
 * 3. The segment is reactivated (with a new metadata mapping) - possibly due to an append. No truncation since #2.
 * 4. Recovery.
 */
@Test
public void testRecoveryWithMetadataCleanup() throws Exception {
    final long truncatedSeqNo = Integer.MAX_VALUE;
    // Setup a DurableLog and start it.
    @Cleanup TestDurableDataLogFactory dataLogFactory = new TestDurableDataLogFactory(new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()));
    @Cleanup Storage storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    long segmentId;
    // First DurableLog. We use this for generating data.
    val metadata1 = (StreamSegmentContainerMetadata) new MetadataBuilder(CONTAINER_ID).build();
    @Cleanup InMemoryCacheFactory cacheFactory = new InMemoryCacheFactory();
    @Cleanup CacheManager cacheManager = new CacheManager(DEFAULT_READ_INDEX_CONFIG.getCachePolicy(), executorService());
    SegmentProperties originalSegmentInfo;
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata1, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata1, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Create the segment.
        val segmentIds = createStreamSegmentsWithOperations(1, metadata1, durableLog, storage);
        segmentId = segmentIds.stream().findFirst().orElse(-1L);
        // Evict the segment.
        val sm1 = metadata1.getStreamSegmentMetadata(segmentId);
        originalSegmentInfo = sm1.getSnapshot();
        // Simulate a truncation. This is needed in order to trigger a cleanup.
        metadata1.removeTruncationMarkers(truncatedSeqNo);
        val cleanedUpSegments = metadata1.cleanup(Collections.singleton(sm1), truncatedSeqNo);
        Assert.assertEquals("Unexpected number of segments evicted.", 1, cleanedUpSegments.size());
        // Map the segment again.
        val reMapOp = new StreamSegmentMapOperation(originalSegmentInfo);
        reMapOp.setStreamSegmentId(segmentId);
        durableLog.add(reMapOp, TIMEOUT).join();
        // Stop.
        durableLog.stopAsync().awaitTerminated();
    }
    // Recovery #1. This should work well.
    val metadata2 = (StreamSegmentContainerMetadata) new MetadataBuilder(CONTAINER_ID).build();
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata2, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata2, dataLogFactory, readIndex, executorService())) {
        durableLog.startAsync().awaitRunning();
        // Get segment info
        val recoveredSegmentInfo = metadata1.getStreamSegmentMetadata(segmentId).getSnapshot();
        Assert.assertEquals("Unexpected length from recovered segment.", originalSegmentInfo.getLength(), recoveredSegmentInfo.getLength());
        // Now evict the segment again ...
        val sm = metadata2.getStreamSegmentMetadata(segmentId);
        // Simulate a truncation. This is needed in order to trigger a cleanup.
        metadata2.removeTruncationMarkers(truncatedSeqNo);
        val cleanedUpSegments = metadata2.cleanup(Collections.singleton(sm), truncatedSeqNo);
        Assert.assertEquals("Unexpected number of segments evicted.", 1, cleanedUpSegments.size());
        // ... and re-map it with a new Id. This is a perfectly valid operation, and we can't prevent it.
        durableLog.add(new StreamSegmentMapOperation(originalSegmentInfo), TIMEOUT).join();
        // Stop.
        durableLog.stopAsync().awaitTerminated();
    }
    // Recovery #2. This should fail due to the same segment mapped multiple times with different ids.
    val metadata3 = (StreamSegmentContainerMetadata) new MetadataBuilder(CONTAINER_ID).build();
    try (ReadIndex readIndex = new ContainerReadIndex(DEFAULT_READ_INDEX_CONFIG, metadata3, cacheFactory, storage, cacheManager, executorService());
        DurableLog durableLog = new DurableLog(ContainerSetup.defaultDurableLogConfig(), metadata3, dataLogFactory, readIndex, executorService())) {
        AssertExtensions.assertThrows("Recovery did not fail with the expected exception in case of multi-mapping", () -> durableLog.startAsync().awaitRunning(), ex -> ex instanceof IllegalStateException && ex.getCause() instanceof DataCorruptionException && ex.getCause().getCause() instanceof MetadataUpdateException);
    }
}

Example 14 with StreamSegmentMapOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation in project pravega by pravega.

the class StorageWriterTests method createTransactions.

private HashMap<Long, ArrayList<Long>> createTransactions(Collection<Long> segmentIds, TestContext context) {
    // Create the Transactions.
    HashMap<Long, ArrayList<Long>> transactions = new HashMap<>();
    long transactionId = Integer.MAX_VALUE;
    for (long parentId : segmentIds) {
        ArrayList<Long> segmentTransactions = new ArrayList<>();
        transactions.put(parentId, segmentTransactions);
        SegmentMetadata parentMetadata = context.metadata.getStreamSegmentMetadata(parentId);
        for (int i = 0; i < TRANSACTIONS_PER_SEGMENT; i++) {
            String transactionName = StreamSegmentNameUtils.getTransactionNameFromId(parentMetadata.getName(), UUID.randomUUID());
            context.metadata.mapStreamSegmentId(transactionName, transactionId, parentId);
            initializeSegment(transactionId, context);
            segmentTransactions.add(transactionId);
            // Add the operation to the log.
            StreamSegmentMapOperation mapOp = new StreamSegmentMapOperation(parentId, context.storage.getStreamSegmentInfo(transactionName, TIMEOUT).join());
            mapOp.setStreamSegmentId(transactionId);
            context.dataSource.add(mapOp);
            transactionId++;
        }
    }
    return transactions;
}

Example 15 with StreamSegmentMapOperation

use of io.pravega.segmentstore.server.logs.operations.StreamSegmentMapOperation in project pravega by pravega.

the class StreamSegmentMapper method submitToOperationLog.

/**
 * Submits a StreamSegmentMapOperation to the OperationLog. Upon completion, this operation
 * will have mapped the given Segment to a new internal Segment Id if none was provided in the given SegmentInfo.
 * If the given SegmentInfo already has a SegmentId set, then all efforts will be made to map that Segment with the
 * requested Segment Id.
 *
 * @param segmentInfo           The SegmentInfo for the StreamSegment to generate and persist.
 * @param parentStreamSegmentId If different from ContainerMetadata.NO_STREAM_SEGMENT_ID, the given streamSegmentInfo
 *                              will be mapped as a transaction. Otherwise, this will be registered as a standalone StreamSegment.
 * @param timeout               Timeout for the operation.
 * @return A CompletableFuture that, when completed, will contain the internal SegmentId that was assigned (or the
 * one supplied via SegmentInfo, if any). If the operation failed, then this Future will complete with that exception.
 */
private CompletableFuture<Long> submitToOperationLog(SegmentInfo segmentInfo, long parentStreamSegmentId, Duration timeout) {
    SegmentProperties properties = segmentInfo.getProperties();
    if (properties.isDeleted()) {
        // Stream does not exist. Fail the request with the appropriate exception.
        failAssignment(properties.getName(), new StreamSegmentNotExistsException("StreamSegment does not exist."));
        return Futures.failedFuture(new StreamSegmentNotExistsException(properties.getName()));
    }
    long existingSegmentId = this.containerMetadata.getStreamSegmentId(properties.getName(), true);
    if (isValidStreamSegmentId(existingSegmentId)) {
        // Looks like someone else beat us to it.
        completeAssignment(properties.getName(), existingSegmentId);
        return CompletableFuture.completedFuture(existingSegmentId);
    } else {
        StreamSegmentMapOperation op;
        if (isValidStreamSegmentId(parentStreamSegmentId)) {
            // Transaction.
            SegmentMetadata parentMetadata = this.containerMetadata.getStreamSegmentMetadata(parentStreamSegmentId);
            assert parentMetadata != null : "parentMetadata is null";
            op = new StreamSegmentMapOperation(parentStreamSegmentId, properties);
        } else {
            // Standalone StreamSegment.
            op = new StreamSegmentMapOperation(properties);
        }
        if (segmentInfo.getSegmentId() != ContainerMetadata.NO_STREAM_SEGMENT_ID) {
            op.setStreamSegmentId(segmentInfo.getSegmentId());
        }
        return this.durableLog.add(op, timeout).thenApply(seqNo -> completeAssignment(properties.getName(), op.getStreamSegmentId()));
    }
}