Examples with DynamicAttributeUpdate io.pravega.segmentstore.contracts.DynamicAttributeUpdate used on opensource projects

Example 1 with DynamicAttributeUpdate

use of io.pravega.segmentstore.contracts.DynamicAttributeUpdate in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributes.

/**
 * Tests the ability to set attributes (via append() or updateAttributes()), then fetch them back using getAttributes(),
 * emphasizing on Extended Attributes that are dumped into Storage and cleared from memory.
 */
@Test
public void testAttributes() throws Exception {
    final AttributeId coreAttribute = Attributes.EVENT_COUNT;
    final int variableAttributeIdLength = 4;
    final List<AttributeId> extendedAttributesUUID = Arrays.asList(AttributeId.randomUUID(), AttributeId.randomUUID());
    final List<AttributeId> extendedAttributesVariable = Arrays.asList(AttributeId.random(variableAttributeIdLength), AttributeId.random(variableAttributeIdLength));
    final List<AttributeId> allAttributesWithUUID = Stream.concat(extendedAttributesUUID.stream(), Stream.of(coreAttribute)).collect(Collectors.toList());
    final List<AttributeId> allAttributesWithVariable = Stream.concat(extendedAttributesVariable.stream(), Stream.of(coreAttribute)).collect(Collectors.toList());
    final AttributeId segmentLengthAttributeUUID = AttributeId.randomUUID();
    final AttributeId segmentLengthAttributeVariable = AttributeId.random(variableAttributeIdLength);
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT));
    containerConfig.setMaxCachedExtendedAttributeCount(SEGMENT_COUNT * allAttributesWithUUID.size());
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    val segmentNames = IntStream.range(0, SEGMENT_COUNT).boxed().collect(Collectors.toMap(StreamSegmentContainerTests::getSegmentName, i -> i % 2 == 0 ? variableAttributeIdLength : 0));
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (val sn : segmentNames.entrySet()) {
        opFutures.add(localContainer.createStreamSegment(sn.getKey(), SegmentType.STREAM_SEGMENT, AttributeUpdateCollection.from(new AttributeUpdate(Attributes.ATTRIBUTE_ID_LENGTH, AttributeUpdateType.None, sn.getValue())), TIMEOUT));
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Predicate<Map.Entry<String, Integer>> isUUIDOnly = e -> e.getValue() == 0;
    // 2. Add some appends.
    for (val sn : segmentNames.entrySet()) {
        boolean isUUID = isUUIDOnly.test(sn);
        for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
            AttributeUpdateCollection attributeUpdates = (isUUID ? allAttributesWithUUID : allAttributesWithVariable).stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Accumulate, 1)).collect(Collectors.toCollection(AttributeUpdateCollection::new));
            opFutures.add(Futures.toVoid(localContainer.append(sn.getKey(), getAppendData(sn.getKey(), i), attributeUpdates, TIMEOUT)));
        }
    }
    // 2.1 Update some of the attributes.
    for (val sn : segmentNames.entrySet()) {
        boolean isUUID = isUUIDOnly.test(sn);
        for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
            AttributeUpdateCollection attributeUpdates = (isUUID ? allAttributesWithUUID : allAttributesWithVariable).stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Accumulate, 1)).collect(Collectors.toCollection(AttributeUpdateCollection::new));
            opFutures.add(localContainer.updateAttributes(sn.getKey(), attributeUpdates, TIMEOUT));
        }
        // Verify that we are not allowed to update attributes of the wrong type.
        val badUpdate = new AttributeUpdate(isUUID ? AttributeId.random(variableAttributeIdLength) : AttributeId.randomUUID(), AttributeUpdateType.Accumulate, 1);
        AssertExtensions.assertSuppliedFutureThrows("updateAttributes allowed updating attributes with wrong type and/or length.", () -> localContainer.updateAttributes(sn.getKey(), AttributeUpdateCollection.from(badUpdate), TIMEOUT), ex -> ex instanceof AttributeIdLengthMismatchException);
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 2.2 Dynamic attributes.
    for (val sn : segmentNames.entrySet()) {
        boolean isUUID = isUUIDOnly.test(sn);
        val dynamicId = isUUID ? segmentLengthAttributeUUID : segmentLengthAttributeVariable;
        val dynamicAttributes = AttributeUpdateCollection.from(new DynamicAttributeUpdate(dynamicId, AttributeUpdateType.Replace, DynamicAttributeValue.segmentLength(10)));
        val appendData = getAppendData(sn.getKey(), 1000);
        val lastOffset = localContainer.append(sn.getKey(), appendData, dynamicAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        val expectedValue = lastOffset - appendData.getLength() + 10;
        Assert.assertEquals(expectedValue, (long) localContainer.getAttributes(sn.getKey(), Collections.singleton(dynamicId), false, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).get(dynamicId));
    }
    // 3. getSegmentInfo
    for (val sn : segmentNames.entrySet()) {
        val segmentName = sn.getKey();
        val allAttributes = isUUIDOnly.test(sn) ? allAttributesWithUUID : allAttributesWithVariable;
        val allAttributeValues = localContainer.getAttributes(segmentName, allAttributes, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected number of attributes retrieved via getAttributes().", allAttributes.size(), allAttributeValues.size());
        // Verify all attribute values.
        SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        for (val attributeId : allAttributes) {
            Assert.assertEquals("Unexpected value for attribute " + attributeId + " via getInfo() for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            Assert.assertEquals("Unexpected value for attribute " + attributeId + " via getAttributes() for segment " + segmentName, expectedAttributeValue, (long) allAttributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
        }
        // Verify we can't request wrong lengths/types.
        val badId = isUUIDOnly.test(sn) ? AttributeId.random(variableAttributeIdLength) : AttributeId.randomUUID();
        AssertExtensions.assertSuppliedFutureThrows("getAttributes allowed getting attributes with wrong type and/or length.", () -> localContainer.getAttributes(segmentName, Collections.singleton(badId), true, TIMEOUT), ex -> ex instanceof IllegalArgumentException);
    }
    // Force these segments out of memory, so that we may verify that extended attributes are still recoverable.
    localContainer.triggerMetadataCleanup(segmentNames.keySet()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    for (val sn : segmentNames.entrySet()) {
        val segmentName = sn.getKey();
        val allAttributes = isUUIDOnly.test(sn) ? allAttributesWithUUID : allAttributesWithVariable;
        val allAttributeValues = localContainer.getAttributes(segmentName, allAttributes, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected number of attributes retrieved via getAttributes() after recovery for segment " + segmentName, allAttributes.size(), allAttributeValues.size());
        // Verify all attribute values. Core attributes should still be loaded in memory, while extended attributes can
        // only be fetched via their special API.
        SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        for (val attributeId : allAttributes) {
            Assert.assertEquals("Unexpected value for attribute " + attributeId + " via getAttributes() after recovery for segment " + segmentName, expectedAttributeValue, (long) allAttributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            if (Attributes.isCoreAttribute(attributeId)) {
                Assert.assertEquals("Expecting core attribute to be loaded in memory.", expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            } else {
                Assert.assertEquals("Not expecting extended attribute to be loaded in memory.", Attributes.NULL_ATTRIBUTE_VALUE, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            }
        }
        // Now instruct the Container to cache missing values (do it a few times so we make sure it's idempotent).
        // Also introduce some random new attribute to fetch. We want to make sure we can properly handle caching
        // missing attribute values.
        val missingAttributeId = isUUIDOnly.test(sn) ? AttributeId.randomUUID() : AttributeId.random(variableAttributeIdLength);
        val attributesToCache = new ArrayList<>(allAttributes);
        attributesToCache.add(missingAttributeId);
        val attributesToCacheValues = new HashMap<>(allAttributeValues);
        attributesToCacheValues.put(missingAttributeId, Attributes.NULL_ATTRIBUTE_VALUE);
        Map<AttributeId, Long> allAttributeValuesWithCache;
        for (int i = 0; i < 2; i++) {
            allAttributeValuesWithCache = localContainer.getAttributes(segmentName, attributesToCache, true, TIMEOUT).join();
            AssertExtensions.assertMapEquals("Inconsistent results from getAttributes(cache=true, attempt=" + i + ").", attributesToCacheValues, allAttributeValuesWithCache);
            sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
            for (val attributeId : allAttributes) {
                Assert.assertEquals("Expecting all attributes to be loaded in memory.", expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            }
            Assert.assertEquals("Unexpected value for missing Attribute Id", Attributes.NULL_ATTRIBUTE_VALUE, (long) sp.getAttributes().get(missingAttributeId));
        }
    }
    // 4. Make an update, then immediately seal the segment, then verify the update updated the root pointer.
    AttributeId attr = Attributes.ATTRIBUTE_SEGMENT_ROOT_POINTER;
    val oldRootPointers = new HashMap<String, Long>();
    for (val sn : segmentNames.entrySet()) {
        val segmentName = sn.getKey();
        val newAttributeId = isUUIDOnly.test(sn) ? AttributeId.randomUUID() : AttributeId.random(variableAttributeIdLength);
        // Get the old root pointer, then make a random attribute update, then immediately seal the segment.
        localContainer.getAttributes(segmentName, Collections.singleton(attr), false, TIMEOUT).thenCompose(values -> {
            oldRootPointers.put(segmentName, values.get(attr));
            return CompletableFuture.allOf(localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(newAttributeId, AttributeUpdateType.Replace, 1L)), TIMEOUT), localContainer.sealStreamSegment(segmentName, TIMEOUT));
        }).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    }
    // which indicates the StorageWriter was able to successfully record it after its final Attribute Index update.
    for (String segmentName : segmentNames.keySet()) {
        Long oldValue = oldRootPointers.get(segmentName);
        TestUtils.await(() -> {
            val newVal = localContainer.getAttributes(segmentName, Collections.singleton(attr), false, TIMEOUT).join().get(attr);
            return oldValue < newVal;
        }, 10, TIMEOUT.toMillis());
    }
    waitForSegmentsInStorage(segmentNames.keySet(), localContainer, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    localContainer.stopAsync().awaitTerminated();
}

Example 2 with DynamicAttributeUpdate

use of io.pravega.segmentstore.contracts.DynamicAttributeUpdate in project pravega by pravega.

the class FixedKeyLengthTableSegmentLayout method createIndexUpdate.

// endregion
// region Helpers
private AttributeUpdate createIndexUpdate(TableKey key, int batchOffset) {
    val attributeId = AttributeId.from(key.getKey().getCopy());
    val ref = DynamicAttributeValue.segmentLength(batchOffset);
    return key.hasVersion() ? new DynamicAttributeUpdate(attributeId, AttributeUpdateType.ReplaceIfEquals, ref, getCompareVersion(key)) : new DynamicAttributeUpdate(attributeId, AttributeUpdateType.Replace, ref);
}

Example 3 with DynamicAttributeUpdate

use of io.pravega.segmentstore.contracts.DynamicAttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testWithAttributesByReference.

private void testWithAttributesByReference(Function<AttributeUpdateCollection, Operation> createOperation) throws Exception {
    final AttributeId referenceAttributeId = AttributeId.randomUUID();
    final AttributeId attributeSegmentLength = AttributeId.randomUUID();
    final long initialAttributeValue = 1234567;
    UpdateableContainerMetadata metadata = createMetadata();
    metadata.getStreamSegmentMetadata(SEGMENT_ID).updateAttributes(ImmutableMap.of(referenceAttributeId, initialAttributeValue));
    val txn = createUpdateTransaction(metadata);
    // Update #1.
    AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(referenceAttributeId, AttributeUpdateType.Accumulate, 2), new DynamicAttributeUpdate(attributeSegmentLength, AttributeUpdateType.None, DynamicAttributeValue.segmentLength(5)));
    Map<AttributeId, Long> expectedValues = ImmutableMap.of(Attributes.ATTRIBUTE_SEGMENT_TYPE, DEFAULT_TYPE.getValue(), referenceAttributeId, initialAttributeValue + 2, attributeSegmentLength, SEGMENT_LENGTH + 5);
    Operation op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    // Verify result.
    verifyAttributeUpdates("after acceptOperation", txn, attributeUpdates, expectedValues);
    txn.commit(metadata);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in segment metadata after final commit.", expectedValues, metadata.getStreamSegmentMetadata(SEGMENT_ID));
}

Example 4 with DynamicAttributeUpdate

use of io.pravega.segmentstore.contracts.DynamicAttributeUpdate in project pravega by pravega.

the class SegmentMetadataUpdateTransaction method preProcessAttributes.

/**
 * Pre-processes a collection of attributes.
 * After this method returns, all AttributeUpdates in the given collection will have the actual (and updated) value
 * of that attribute in the Segment.
 *
 * @param attributeUpdates The Updates to process (if any).
 * @throws BadAttributeUpdateException        If any of the given AttributeUpdates is invalid given the current state
 *                                            of the segment.
 * @throws AttributeIdLengthMismatchException If the given AttributeUpdates contains extended Attributes with the
 *                                            wrong length (compared to that which is defined on the Segment).
 * @throws MetadataUpdateException            If the operation cannot not be processed because a Metadata Update
 *                                            precondition check failed.
 */
private void preProcessAttributes(AttributeUpdateCollection attributeUpdates) throws BadAttributeUpdateException, MetadataUpdateException {
    if (attributeUpdates == null) {
        return;
    }
    // Make sure that the number of existing ExtendedAttributes + incoming ExtendedAttributes does not exceed the limit.
    if (type.isTransientSegment() && this.baseAttributeValues.size() > TRANSIENT_ATTRIBUTE_LIMIT) {
        throw new MetadataUpdateException(this.containerId, String.format("A Transient Segment ('%s') may not exceed %s Extended Attributes.", this.name, TRANSIENT_ATTRIBUTE_LIMIT));
    }
    // We must ensure that we aren't trying to set/update attributes that are incompatible with this Segment.
    validateAttributeIdLengths(attributeUpdates);
    for (AttributeUpdate u : attributeUpdates) {
        if (Attributes.isUnmodifiable(u.getAttributeId())) {
            throw new MetadataUpdateException(this.containerId, String.format("Attribute Id '%s' on Segment Id %s ('%s') may not be modified.", u.getAttributeId(), this.id, this.name));
        }
        AttributeUpdateType updateType = u.getUpdateType();
        boolean hasValue = false;
        long previousValue = Attributes.NULL_ATTRIBUTE_VALUE;
        if (this.attributeUpdates.containsKey(u.getAttributeId())) {
            hasValue = true;
            previousValue = this.attributeUpdates.get(u.getAttributeId());
        } else if (this.baseAttributeValues.containsKey(u.getAttributeId())) {
            hasValue = true;
            previousValue = this.baseAttributeValues.get(u.getAttributeId());
        }
        // Perform validation, and set the AttributeUpdate.value to the updated value, if necessary.
        switch(updateType) {
            case ReplaceIfGreater:
                // Verify value against existing value, if any.
                if (hasValue && u.getValue() <= previousValue) {
                    throw new BadAttributeUpdateException(this.name, u, false, String.format("Expected greater than '%s'.", previousValue));
                }
                break;
            case ReplaceIfEquals:
                // Verify value against existing value, if any.
                if (u.getComparisonValue() != previousValue || !hasValue) {
                    throw new BadAttributeUpdateException(this.name, u, !hasValue, String.format("Expected '%s', given '%s'.", previousValue, u.getComparisonValue()));
                }
                break;
            case None:
                // Verify value is not already set.
                if (hasValue) {
                    throw new BadAttributeUpdateException(this.name, u, false, String.format("Attribute value already set (%s).", previousValue));
                }
                break;
            case Accumulate:
                if (hasValue) {
                    u.setValue(previousValue + u.getValue());
                }
                break;
            case Replace:
                break;
            default:
                throw new BadAttributeUpdateException(this.name, u, !hasValue, "Unexpected update type: " + updateType);
        }
    }
    // Evaluate and set DynamicAttributeUpdates.
    for (DynamicAttributeUpdate u : attributeUpdates.getDynamicAttributeUpdates()) {
        u.setValue(u.getValueReference().evaluate(this));
    }
}

Example 5 with DynamicAttributeUpdate

use of io.pravega.segmentstore.contracts.DynamicAttributeUpdate in project pravega by pravega.

the class FixedKeyLengthTableCompactor method generateIndexUpdates.

@Override
protected void generateIndexUpdates(Candidate c, int batchOffset, AttributeUpdateCollection indexUpdates) {
    // For each Candidate we're about to copy, create an index update that:
    // - Updates the Index Key Conditionally to point to the new location of the Key/Entry.
    // - The new location is dynamically calculated based on the append offset + offset within the append (batchOffset)
    // - The condition is that the Key we're about to copy has not changed since we read it (so we condition on the
    // index entry value pointing to its original Segment Offset.
    val au = new DynamicAttributeUpdate(AttributeId.from(c.entry.getKey().getKey().getCopy()), AttributeUpdateType.ReplaceIfEquals, DynamicAttributeValue.segmentLength(batchOffset), c.segmentOffset);
    indexUpdates.add(au);
}