Examples with AttributeId io.pravega.segmentstore.contracts.AttributeId used on opensource projects

Example 6 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class AppendProcessorTest method testAppendAfterSealThrows.

@Test(timeout = 15 * 1000)
public void testAppendAfterSealThrows() throws Exception {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).build();
    InOrder connectionVerifier = Mockito.inOrder(connection);
    @SuppressWarnings("unchecked") Map<AttributeId, Long> attributes = mock(Map.class);
    when(store.getAttributes(streamSegmentName, Collections.singleton(AttributeId.fromUUID(clientId)), true, AppendProcessor.TIMEOUT)).thenReturn(CompletableFuture.completedFuture(attributes));
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    verify(store).getAttributes(eq(streamSegmentName), any(), eq(true), eq(AppendProcessor.TIMEOUT));
    verify(connection).close();
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 7 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentContainer method getAndCacheAttributes.

/**
 * Gets the values of the given (Core and Extended) Attribute Ids for the given segment.
 *
 * @param segmentMetadata The SegmentMetadata for the Segment to retrieve attribute values for.
 * @param attributeIds    A Collection of AttributeIds to retrieve.
 * @param cache           If true, any Extended Attribute value that is not present in the SegmentMetadata cache will
 *                        be added to that (using a conditional updateAttributes() call) before completing.
 * @param timer           Timer for the operation.
 * @return A CompletableFuture that, when completed normally, will contain the desired result. If the operation failed,
 * it will be completed with the appropriate exception. If cache==true and the conditional call to updateAttributes()
 * could not be completed because of a conflicting update, it will be failed with BadAttributeUpdateException, in which
 * case a retry is warranted.
 */
private CompletableFuture<Map<AttributeId, Long>> getAndCacheAttributes(SegmentMetadata segmentMetadata, Collection<AttributeId> attributeIds, boolean cache, TimeoutTimer timer) {
    // Collect Core Attributes and Cached Extended Attributes.
    Map<AttributeId, Long> result = new HashMap<>();
    Map<AttributeId, Long> metadataAttributes = segmentMetadata.getAttributes();
    ArrayList<AttributeId> extendedAttributeIds = new ArrayList<>();
    attributeIds.forEach(attributeId -> {
        Long v = metadataAttributes.get(attributeId);
        if (v != null) {
            // This attribute is cached in the Segment Metadata, even if it has a value equal to Attributes.NULL_ATTRIBUTE_VALUE.
            result.put(attributeId, v);
        } else if (!Attributes.isCoreAttribute(attributeId)) {
            extendedAttributeIds.add(attributeId);
        }
    });
    if (extendedAttributeIds.isEmpty()) {
        // Nothing to lookup in the Attribute Index, so bail out early.
        return CompletableFuture.completedFuture(result);
    }
    // Collect remaining Extended Attributes.
    CompletableFuture<Map<AttributeId, Long>> r = this.attributeIndex.forSegment(segmentMetadata.getId(), timer.getRemaining()).thenComposeAsync(idx -> idx.get(extendedAttributeIds, timer.getRemaining()), this.executor).thenApplyAsync(extendedAttributes -> {
        if (extendedAttributeIds.size() == extendedAttributes.size()) {
            // We found a value for each Attribute Id. Nothing more to do.
            return extendedAttributes;
        }
        // Insert a NULL_ATTRIBUTE_VALUE for each missing value.
        Map<AttributeId, Long> allValues = new HashMap<>(extendedAttributes);
        extendedAttributeIds.stream().filter(id -> !extendedAttributes.containsKey(id)).forEach(id -> allValues.put(id, Attributes.NULL_ATTRIBUTE_VALUE));
        return allValues;
    }, this.executor);
    if (cache && !segmentMetadata.isSealed()) {
        // Add them to the cache if requested.
        r = r.thenComposeAsync(extendedAttributes -> {
            // Update the in-memory Segment Metadata using a special update (AttributeUpdateType.None, which should
            // complete if the attribute is not currently set). If it has some value, then a concurrent update
            // must have changed it and we cannot update anymore.
            val updates = new AttributeUpdateCollection();
            for (val e : extendedAttributes.entrySet()) {
                updates.add(new AttributeUpdate(e.getKey(), AttributeUpdateType.None, e.getValue()));
            }
            // invoke this one again.
            return addOperation(new UpdateAttributesOperation(segmentMetadata.getId(), updates), timer.getRemaining()).thenApply(v -> extendedAttributes);
        }, this.executor);
    }
    // Compile the final result.
    return r.thenApply(extendedAttributes -> {
        result.putAll(extendedAttributes);
        return result;
    });
}

Example 8 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentContainer method processAttributeUpdaterOperation.

/**
 * Processes the given AttributeUpdateOperation with exactly one retry in case it was rejected because of an attribute
 * update failure due to the attribute value missing from the in-memory cache.
 *
 * @param operation The Operation to process.
 * @param timer     Timer for the operation.
 * @param <T>       Type of the operation.
 * @return A CompletableFuture that, when completed normally, will indicate that the Operation has been successfully
 * processed. If it failed, it will be completed with an appropriate exception.
 */
private <T extends Operation & AttributeUpdaterOperation> CompletableFuture<Void> processAttributeUpdaterOperation(T operation, TimeoutTimer timer) {
    Collection<AttributeUpdate> updates = operation.getAttributeUpdates();
    if (updates == null || updates.isEmpty()) {
        // No need for extra complicated handling.
        return addOperation(operation, timer.getRemaining());
    }
    return Futures.exceptionallyCompose(addOperation(operation, timer.getRemaining()), ex -> {
        // We only retry BadAttributeUpdateExceptions if it has the PreviousValueMissing flag set.
        ex = Exceptions.unwrap(ex);
        if (ex instanceof BadAttributeUpdateException && ((BadAttributeUpdateException) ex).isPreviousValueMissing()) {
            // Get the missing attributes and load them into the cache, then retry the operation, exactly once.
            SegmentMetadata segmentMetadata = this.metadata.getStreamSegmentMetadata(operation.getStreamSegmentId());
            Collection<AttributeId> attributeIds = updates.stream().map(AttributeUpdate::getAttributeId).filter(id -> !Attributes.isCoreAttribute(id)).collect(Collectors.toList());
            if (!attributeIds.isEmpty()) {
                // This only makes sense if a core attribute was missing.
                return getAndCacheAttributes(segmentMetadata, attributeIds, true, timer).thenComposeAsync(attributes -> {
                    // Final attempt - now that we should have the attributes cached.
                    return addOperation(operation, timer.getRemaining());
                }, this.executor);
            }
        }
        // Anything else is non-retryable; rethrow.
        return Futures.failedFuture(ex);
    });
}

Example 9 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class SegmentAttributeIterator method mix.

/**
 * Mixes the Attributes from the given iterator with the ones from the {@link SegmentMetadata} passed to this class'
 * constructor.
 *
 * The mixing algorithm works along these lines:
 * - Each entry (Attribute) in the given Iterator is considered.
 * - All non-deleted Attributes from the {@link SegmentMetadata} that are smaller than the one from the current entry
 * will be added.
 * - The entry will be added only if there is no corresponding updated value for its Attribute Id in the {@link SegmentMetadata},
 * If there is, then the updated value is used.
 *
 * @param indexAttributes A List containing pairs of AttributeId to Long representing the base Attributes (i.e., from the
 *                        SegmentAttributeIndex. This iterator must return the pairs in their natural order based on
 *                        the {@link AttributeId#compareTo} comparer.
 * @return A List of Map Entries (AttributeId to Long) containing all the Attributes from the index, mixed with the appropriate
 * Attributes from the {@link SegmentMetadata} passed to this class' constructor. This will return null if both
 * indexAttributes is null and there are no more Attributes to process from the {@link SegmentMetadata}.
 */
private List<Map.Entry<AttributeId, Long>> mix(List<Map.Entry<AttributeId, Long>> indexAttributes) {
    val result = new ArrayList<Map.Entry<AttributeId, Long>>();
    if (indexAttributes == null) {
        // Nothing more in the base iterator. Add whatever is in the metadata attributes.
        synchronized (this.metadataAttributes) {
            while (!this.metadataAttributes.isEmpty()) {
                include(this.metadataAttributes.removeFirst(), result);
            }
        }
        return result.isEmpty() ? null : result;
    } else {
        // Exclude those that are deleted.
        for (val idxAttribute : indexAttributes) {
            checkIndexAttribute(idxAttribute.getKey());
            // Find all metadata attributes that are smaller or the same as the base attribute and include them all.
            // This also handles value overrides (metadata attributes, if present, always have the latest value).
            AttributeId lastMetadataAttribute = null;
            synchronized (this.metadataAttributes) {
                while (!this.metadataAttributes.isEmpty() && this.metadataAttributes.peekFirst().getKey().compareTo(idxAttribute.getKey()) <= 0) {
                    lastMetadataAttribute = include(this.metadataAttributes.removeFirst(), result);
                }
            }
            // Only add our element if it hasn't already been processed.
            if (lastMetadataAttribute == null || !lastMetadataAttribute.equals(idxAttribute.getKey())) {
                include(idxAttribute, result);
            }
        }
        return result;
    }
}

Example 10 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class SegmentAttributeIterator method checkIndexAttribute.

private void checkIndexAttribute(AttributeId attributeId) {
    AttributeId prevId = this.lastIndexAttribute.get();
    if (prevId != null) {
        Preconditions.checkArgument(prevId.compareTo(attributeId) < 0, "baseIterator did not return Attributes in order. Expected at greater than {%s}, found {%s}.", prevId, attributeId);
    }
    Preconditions.checkArgument(this.fromId.compareTo(attributeId) <= 0 && this.toId.compareTo(attributeId) >= 0, "baseIterator returned an Attribute Id that was out of range. Expected between {%s} and {%s}, found {%s}.", this.fromId, this.toId, attributeId);
    this.lastIndexAttribute.set(attributeId);
}