Examples with AttributeUpdate io.pravega.segmentstore.contracts.AttributeUpdate used on opensource projects

Example 41 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testProcessMetadataCheckpoint.

/**
 * Tests the processMetadataOperation method with MetadataCheckpoint operations.
 */
@Test
public void testProcessMetadataCheckpoint() throws Exception {
    // When encountering MetadataCheckpoint in non-Recovery Mode, the ContainerMetadataUpdateTransaction serializes a snapshot
    // of the current metadata inside the Operation.
    // When encountering MetadataCheckpoint in Recovery Mode, the ContainerMetadataUpdateTransaction deserializes the snapshot-ted
    // metadata in it and applies it to the container metadata (inside the transaction). All existing metadata updates
    // are cleared.
    String newSegmentName = "NewSegmentId";
    AtomicLong seqNo = new AtomicLong();
    // Create a non-empty metadata, and in addition, seal a segment and truncate it.
    this.timeProvider.setElapsedMillis(1234);
    UpdateableContainerMetadata metadata = createMetadata();
    metadata.getStreamSegmentMetadata(SEGMENT_ID).markSealed();
    metadata.getStreamSegmentMetadata(SEGMENT_ID).setStartOffset(SEGMENT_LENGTH / 2);
    val txn = createUpdateTransaction(metadata);
    // Checkpoint 1: original metadata.
    // Checkpoint 2: Checkpoint 1 + 1 StreamSegment and 1 Transaction + 1 Append
    MetadataCheckpointOperation checkpoint1 = createMetadataCheckpoint();
    MetadataCheckpointOperation checkpoint2 = createMetadataCheckpoint();
    // Checkpoint 1 Should have original metadata.
    val checkpoint1Contents = processCheckpointOperation(checkpoint1, txn, seqNo::incrementAndGet);
    Assert.assertNull("Expecting checkpoint contents to be cleared after processing.", checkpoint1.getContents());
    UpdateableContainerMetadata checkpointedMetadata = getCheckpointedMetadata(checkpoint1Contents);
    assertMetadataSame("Unexpected metadata before any operation.", metadata, checkpointedMetadata);
    // Map another StreamSegment, and add an append
    StreamSegmentMapOperation mapOp = new StreamSegmentMapOperation(StreamSegmentInformation.builder().name(newSegmentName).length(SEGMENT_LENGTH).build());
    processOperation(mapOp, txn, seqNo::incrementAndGet);
    // Add a few Extended Attributes.
    val extendedAttributeUpdates = createAttributeUpdates();
    processOperation(new StreamSegmentAppendOperation(mapOp.getStreamSegmentId(), DEFAULT_APPEND_DATA, extendedAttributeUpdates), txn, seqNo::incrementAndGet);
    // Add a Core Attribute.
    val coreAttributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(Attributes.EVENT_COUNT, AttributeUpdateType.Replace, 1));
    processOperation(new StreamSegmentAppendOperation(mapOp.getStreamSegmentId(), DEFAULT_APPEND_DATA, coreAttributeUpdates), txn, seqNo::incrementAndGet);
    val checkpoint2Contents = processCheckpointOperation(checkpoint2, txn, seqNo::incrementAndGet);
    Assert.assertNull("Expecting checkpoint contents to be cleared after processing.", checkpoint2.getContents());
    // Checkpoint 2 should have Checkpoint 1 + New StreamSegment + Append.
    txn.commit(metadata);
    checkpointedMetadata = getCheckpointedMetadata(checkpoint2Contents);
    // Checkpointing will remove all Extended Attributes. In order to facilitate the comparison, we should remove those
    // too, from the original metadata (after we've serialized the checkpoint).
    val expectedCleared = extendedAttributeUpdates.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, au -> Attributes.NULL_ATTRIBUTE_VALUE));
    metadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId()).updateAttributes(expectedCleared);
    // Now actually check the checkpointed metadata.
    val checkpointedSm = checkpointedMetadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId());
    for (val e : extendedAttributeUpdates) {
        Assert.assertFalse("Extended attribute was serialized.", checkpointedSm.getAttributes().containsKey(e.getAttributeId()));
    }
    checkpointedMetadata.getStreamSegmentMetadata(mapOp.getStreamSegmentId()).updateAttributes(expectedCleared);
    assertMetadataSame("Unexpected metadata after deserializing checkpoint.", metadata, checkpointedMetadata);
}

Example 42 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testTransientSegmentExtendedAttributeLimit.

/**
 * Tests that a Transient Segment may only have {@link SegmentMetadataUpdateTransaction#TRANSIENT_ATTRIBUTE_LIMIT}
 * or fewer Extended Attributes.
 */
@Test
public void testTransientSegmentExtendedAttributeLimit() throws ContainerException, StreamSegmentException {
    // Create base metadata with one Transient Segment.
    UpdateableContainerMetadata metadata = createMetadataTransient();
    int expected = 1;
    Assert.assertEquals("Unexpected initial Active Segment Count for base metadata.", expected, metadata.getActiveSegmentCount());
    // Create an UpdateTransaction containing updates for Extended Attributes on the Transient Segment -- it should succeed.
    val txn1 = new ContainerMetadataUpdateTransaction(metadata, metadata, 0);
    Assert.assertEquals("Unexpected Active Segment Count for first transaction.", expected, txn1.getActiveSegmentCount());
    // Subtract one from remaining due to Segment Type Attribute.
    long id = SegmentMetadataUpdateTransaction.TRANSIENT_ATTRIBUTE_LIMIT - (TRANSIENT_ATTRIBUTE_REMAINING - 1);
    AttributeUpdateCollection attributes = AttributeUpdateCollection.from(// The new entry.
    new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, id), AttributeUpdateType.None, id), // Update two old entries.
    new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, 0), AttributeUpdateType.Replace, (long) 1), new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, 1), AttributeUpdateType.Replace, (long) 2));
    val map1 = createTransientAppend(TRANSIENT_SEGMENT_ID, attributes);
    txn1.preProcessOperation(map1);
    map1.setSequenceNumber(metadata.nextOperationSequenceNumber());
    txn1.acceptOperation(map1);
    int expectedExtendedAttributes = SegmentMetadataUpdateTransaction.TRANSIENT_ATTRIBUTE_LIMIT - (TRANSIENT_ATTRIBUTE_REMAINING - 1);
    SegmentMetadata segmentMetadata = txn1.getStreamSegmentMetadata(TRANSIENT_SEGMENT_ID);
    Assert.assertEquals("Unexpected Extended Attribute count after first transaction.", expectedExtendedAttributes, segmentMetadata.getAttributes().size());
    val txn2 = new ContainerMetadataUpdateTransaction(txn1, metadata, 1);
    // Add two new Extended Attributes which should exceed the set limit.
    attributes = AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, ++id), AttributeUpdateType.None, (long) 0), new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, ++id), AttributeUpdateType.None, (long) 0));
    // Should not fail as there was space before the operation, so accept any new additions.
    val map2 = createTransientAppend(TRANSIENT_SEGMENT_ID, attributes);
    txn2.preProcessOperation(map2);
    txn2.acceptOperation(map2);
    // Since we are now over the limit, enforce that no new Extended Attributes may be added.
    val txn3 = new ContainerMetadataUpdateTransaction(txn2, metadata, 2);
    // Expect another Attribute addition to fail as the limit has been exceeded.
    val map3 = createTransientAppend(TRANSIENT_SEGMENT_ID, AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, ++id), AttributeUpdateType.None, (long) 0)));
    AssertExtensions.assertThrows("Exception was not thrown when too many Extended Attributes were registered.", () -> txn3.preProcessOperation(map3), ex -> ex instanceof MetadataUpdateException);
}

Example 43 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testUpdateAttributesSealedSegment.

/**
 * Tests the ability of the ContainerMetadataUpdateTransaction to handle UpdateAttributeOperations after the segment
 * has been sealed.
 */
@Test
public void testUpdateAttributesSealedSegment() throws Exception {
    final AttributeId coreAttribute = Attributes.ATTRIBUTE_SEGMENT_ROOT_POINTER;
    final AttributeId extAttribute = AttributeId.randomUUID();
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    // Seal the segment.
    val sealOp = createSeal();
    txn.preProcessOperation(sealOp);
    txn.acceptOperation(sealOp);
    // 1. Core attribute, but not internal.
    val coreUpdate = new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(new AttributeUpdate(coreAttribute, AttributeUpdateType.Replace, 1L)));
    AssertExtensions.assertThrows("Non-internal update of core attribute succeeded.", () -> txn.preProcessOperation(coreUpdate), ex -> ex instanceof StreamSegmentSealedException);
    // 2. Core attribute, internal update.
    coreUpdate.setInternal(true);
    txn.preProcessOperation(coreUpdate);
    txn.acceptOperation(coreUpdate);
    Assert.assertEquals("Core attribute was not updated.", 1L, (long) txn.getStreamSegmentMetadata(SEGMENT_ID).getAttributes().get(coreAttribute));
    // 3. Extended attributes.
    val extUpdate1 = new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(new AttributeUpdate(extAttribute, AttributeUpdateType.Replace, 1L)));
    extUpdate1.setInternal(true);
    AssertExtensions.assertThrows("Extended attribute update succeeded.", () -> txn.preProcessOperation(extUpdate1), ex -> ex instanceof StreamSegmentSealedException);
    val extUpdate2 = new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(new AttributeUpdate(coreAttribute, AttributeUpdateType.Replace, 2L), new AttributeUpdate(extAttribute, AttributeUpdateType.Replace, 3L)));
    extUpdate1.setInternal(true);
    AssertExtensions.assertThrows("Mixed attribute update succeeded.", () -> txn.preProcessOperation(extUpdate2), ex -> ex instanceof StreamSegmentSealedException);
}

Example 44 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributeIterators.

/**
 * Tests the ability to run attribute iterators over all or a subset of attributes in a segment.
 */
@Test
public void testAttributeIterators() throws Exception {
    final List<AttributeId> sortedAttributes = IntStream.range(0, 100).mapToObj(i -> AttributeId.uuid(i, i)).sorted().collect(Collectors.toList());
    final Map<AttributeId, Long> expectedValues = sortedAttributes.stream().collect(Collectors.toMap(id -> id, id -> id.getBitGroup(0)));
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT));
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the Segment.
    String segmentName = getSegmentName(0);
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val segment1 = localContainer.forSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 2. Set some initial attribute values and verify in-memory iterator.
    AttributeUpdateCollection attributeUpdates = sortedAttributes.stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Replace, expectedValues.get(attributeId))).collect(Collectors.toCollection(AttributeUpdateCollection::new));
    segment1.updateAttributes(attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment1, sortedAttributes, expectedValues);
    // 3. Force these segments out of memory and verify out-of-memory iterator.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val segment2 = localContainer.forSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment2, sortedAttributes, expectedValues);
    // 4. Update some values, and verify mixed iterator.
    attributeUpdates.clear();
    for (int i = 0; i < sortedAttributes.size(); i += 3) {
        AttributeId attributeId = sortedAttributes.get(i);
        expectedValues.put(attributeId, expectedValues.get(attributeId) + 1);
        attributeUpdates.add(new AttributeUpdate(attributeId, AttributeUpdateType.Replace, expectedValues.get(attributeId)));
    }
    segment2.updateAttributes(attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment2, sortedAttributes, expectedValues);
    localContainer.stopAsync().awaitTerminated();
}

Example 45 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class MetadataStoreTestBase method testCreateSegmentAlreadyExists.

/**
 * Tests the ability of the MetadataStore to create a new Segment if the Segment already exists.
 */
@Test
public void testCreateSegmentAlreadyExists() {
    final String segmentName = "NewSegment";
    final Map<AttributeId, Long> originalAttributes = ImmutableMap.of(AttributeId.randomUUID(), 123L, Attributes.EVENT_COUNT, 1L);
    final Map<AttributeId, Long> expectedAttributes = getExpectedCoreAttributes(originalAttributes, SEGMENT_TYPE);
    final Collection<AttributeUpdate> correctAttributeUpdates = originalAttributes.entrySet().stream().map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())).collect(Collectors.toList());
    final Map<AttributeId, Long> badAttributes = Collections.singletonMap(AttributeId.randomUUID(), 456L);
    final Collection<AttributeUpdate> badAttributeUpdates = badAttributes.entrySet().stream().map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())).collect(Collectors.toList());
    @Cleanup TestContext context = createTestContext();
    // Create a segment.
    context.getMetadataStore().createSegment(segmentName, SEGMENT_TYPE, correctAttributeUpdates, TIMEOUT).join();
    // Try to create it again.
    AssertExtensions.assertSuppliedFutureThrows("createSegment did not fail when Segment already exists.", () -> context.getMetadataStore().createSegment(segmentName, SEGMENT_TYPE, badAttributeUpdates, TIMEOUT), ex -> ex instanceof StreamSegmentExistsException);
    val si = context.getMetadataStore().getSegmentInfo(segmentName, TIMEOUT).join();
    AssertExtensions.assertMapEquals("Unexpected attributes after failed attempt to recreate correctly created segment", expectedAttributes, si.getAttributes());
}