Examples with AttributeUpdateCollection io.pravega.segmentstore.contracts.AttributeUpdateCollection used on opensource projects

Example 16 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testTransientSegmentExtendedAttributeLimit.

/**
 * Tests that a Transient Segment may only have {@link SegmentMetadataUpdateTransaction#TRANSIENT_ATTRIBUTE_LIMIT}
 * or fewer Extended Attributes.
 */
@Test
public void testTransientSegmentExtendedAttributeLimit() throws ContainerException, StreamSegmentException {
    // Create base metadata with one Transient Segment.
    UpdateableContainerMetadata metadata = createMetadataTransient();
    int expected = 1;
    Assert.assertEquals("Unexpected initial Active Segment Count for base metadata.", expected, metadata.getActiveSegmentCount());
    // Create an UpdateTransaction containing updates for Extended Attributes on the Transient Segment -- it should succeed.
    val txn1 = new ContainerMetadataUpdateTransaction(metadata, metadata, 0);
    Assert.assertEquals("Unexpected Active Segment Count for first transaction.", expected, txn1.getActiveSegmentCount());
    // Subtract one from remaining due to Segment Type Attribute.
    long id = SegmentMetadataUpdateTransaction.TRANSIENT_ATTRIBUTE_LIMIT - (TRANSIENT_ATTRIBUTE_REMAINING - 1);
    AttributeUpdateCollection attributes = AttributeUpdateCollection.from(// The new entry.
    new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, id), AttributeUpdateType.None, id), // Update two old entries.
    new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, 0), AttributeUpdateType.Replace, (long) 1), new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, 1), AttributeUpdateType.Replace, (long) 2));
    val map1 = createTransientAppend(TRANSIENT_SEGMENT_ID, attributes);
    txn1.preProcessOperation(map1);
    map1.setSequenceNumber(metadata.nextOperationSequenceNumber());
    txn1.acceptOperation(map1);
    int expectedExtendedAttributes = SegmentMetadataUpdateTransaction.TRANSIENT_ATTRIBUTE_LIMIT - (TRANSIENT_ATTRIBUTE_REMAINING - 1);
    SegmentMetadata segmentMetadata = txn1.getStreamSegmentMetadata(TRANSIENT_SEGMENT_ID);
    Assert.assertEquals("Unexpected Extended Attribute count after first transaction.", expectedExtendedAttributes, segmentMetadata.getAttributes().size());
    val txn2 = new ContainerMetadataUpdateTransaction(txn1, metadata, 1);
    // Add two new Extended Attributes which should exceed the set limit.
    attributes = AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, ++id), AttributeUpdateType.None, (long) 0), new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, ++id), AttributeUpdateType.None, (long) 0));
    // Should not fail as there was space before the operation, so accept any new additions.
    val map2 = createTransientAppend(TRANSIENT_SEGMENT_ID, attributes);
    txn2.preProcessOperation(map2);
    txn2.acceptOperation(map2);
    // Since we are now over the limit, enforce that no new Extended Attributes may be added.
    val txn3 = new ContainerMetadataUpdateTransaction(txn2, metadata, 2);
    // Expect another Attribute addition to fail as the limit has been exceeded.
    val map3 = createTransientAppend(TRANSIENT_SEGMENT_ID, AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, ++id), AttributeUpdateType.None, (long) 0)));
    AssertExtensions.assertThrows("Exception was not thrown when too many Extended Attributes were registered.", () -> txn3.preProcessOperation(map3), ex -> ex instanceof MetadataUpdateException);
}

Example 17 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributeIterators.

/**
 * Tests the ability to run attribute iterators over all or a subset of attributes in a segment.
 */
@Test
public void testAttributeIterators() throws Exception {
    final List<AttributeId> sortedAttributes = IntStream.range(0, 100).mapToObj(i -> AttributeId.uuid(i, i)).sorted().collect(Collectors.toList());
    final Map<AttributeId, Long> expectedValues = sortedAttributes.stream().collect(Collectors.toMap(id -> id, id -> id.getBitGroup(0)));
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT));
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the Segment.
    String segmentName = getSegmentName(0);
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val segment1 = localContainer.forSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 2. Set some initial attribute values and verify in-memory iterator.
    AttributeUpdateCollection attributeUpdates = sortedAttributes.stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Replace, expectedValues.get(attributeId))).collect(Collectors.toCollection(AttributeUpdateCollection::new));
    segment1.updateAttributes(attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment1, sortedAttributes, expectedValues);
    // 3. Force these segments out of memory and verify out-of-memory iterator.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val segment2 = localContainer.forSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment2, sortedAttributes, expectedValues);
    // 4. Update some values, and verify mixed iterator.
    attributeUpdates.clear();
    for (int i = 0; i < sortedAttributes.size(); i += 3) {
        AttributeId attributeId = sortedAttributes.get(i);
        expectedValues.put(attributeId, expectedValues.get(attributeId) + 1);
        attributeUpdates.add(new AttributeUpdate(attributeId, AttributeUpdateType.Replace, expectedValues.get(attributeId)));
    }
    segment2.updateAttributes(attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment2, sortedAttributes, expectedValues);
    localContainer.stopAsync().awaitTerminated();
}

Example 18 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class ReadOnlySegmentContainerTests method testUnsupportedOperations.

/**
 * Verifies that non-read operations are not supported.
 */
@Test
public void testUnsupportedOperations() {
    @Cleanup val context = new TestContext();
    context.container.startAsync().awaitRunning();
    assertUnsupported("createStreamSegment", () -> context.container.createStreamSegment(SEGMENT_NAME, SegmentType.STREAM_SEGMENT, null, TIMEOUT));
    assertUnsupported("append", () -> context.container.append(SEGMENT_NAME, new ByteArraySegment(new byte[1]), null, TIMEOUT));
    assertUnsupported("append-offset", () -> context.container.append(SEGMENT_NAME, 0, new ByteArraySegment(new byte[1]), null, TIMEOUT));
    assertUnsupported("sealStreamSegment", () -> context.container.sealStreamSegment(SEGMENT_NAME, TIMEOUT));
    assertUnsupported("truncateStreamSegment", () -> context.container.truncateStreamSegment(SEGMENT_NAME, 0, TIMEOUT));
    assertUnsupported("deleteStreamSegment", () -> context.container.deleteStreamSegment(SEGMENT_NAME, TIMEOUT));
    assertUnsupported("mergeTransaction", () -> context.container.mergeStreamSegment(SEGMENT_NAME, SEGMENT_NAME, TIMEOUT));
    assertUnsupported("mergeTransaction", () -> context.container.mergeStreamSegment(SEGMENT_NAME, SEGMENT_NAME, new AttributeUpdateCollection(), TIMEOUT));
    assertUnsupported("getExtendedChunkInfo", () -> context.container.getExtendedChunkInfo(SEGMENT_NAME, TIMEOUT));
}

Example 19 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class StorageWriterTests method updateAttributes.

private void updateAttributes(UpdateableSegmentMetadata segmentMetadata, TestContext context) {
    AttributeUpdateCollection attributeUpdates = generateAttributeUpdates(segmentMetadata);
    context.dataSource.add(new UpdateAttributesOperation(segmentMetadata.getId(), attributeUpdates));
}

Example 20 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class AppendProcessor method storeAppend.

private CompletableFuture<Long> storeAppend(Append append, long lastEventNumber) {
    AttributeUpdateCollection attributes = AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.fromUUID(append.getWriterId()), AttributeUpdateType.ReplaceIfEquals, append.getEventNumber(), lastEventNumber), new AttributeUpdate(EVENT_COUNT, AttributeUpdateType.Accumulate, append.getEventCount()));
    ByteBufWrapper buf = new ByteBufWrapper(append.getData());
    if (append.isConditional()) {
        return store.append(append.getSegment(), append.getExpectedLength(), buf, attributes, TIMEOUT);
    } else {
        return store.append(append.getSegment(), buf, attributes, TIMEOUT);
    }
}