Examples with AttributeUpdateCollection io.pravega.segmentstore.contracts.AttributeUpdateCollection used on opensource projects

Example 6 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class StreamSegmentContainer method getAndCacheAttributes.

/**
 * Gets the values of the given (Core and Extended) Attribute Ids for the given segment.
 *
 * @param segmentMetadata The SegmentMetadata for the Segment to retrieve attribute values for.
 * @param attributeIds    A Collection of AttributeIds to retrieve.
 * @param cache           If true, any Extended Attribute value that is not present in the SegmentMetadata cache will
 *                        be added to that (using a conditional updateAttributes() call) before completing.
 * @param timer           Timer for the operation.
 * @return A CompletableFuture that, when completed normally, will contain the desired result. If the operation failed,
 * it will be completed with the appropriate exception. If cache==true and the conditional call to updateAttributes()
 * could not be completed because of a conflicting update, it will be failed with BadAttributeUpdateException, in which
 * case a retry is warranted.
 */
private CompletableFuture<Map<AttributeId, Long>> getAndCacheAttributes(SegmentMetadata segmentMetadata, Collection<AttributeId> attributeIds, boolean cache, TimeoutTimer timer) {
    // Collect Core Attributes and Cached Extended Attributes.
    Map<AttributeId, Long> result = new HashMap<>();
    Map<AttributeId, Long> metadataAttributes = segmentMetadata.getAttributes();
    ArrayList<AttributeId> extendedAttributeIds = new ArrayList<>();
    attributeIds.forEach(attributeId -> {
        Long v = metadataAttributes.get(attributeId);
        if (v != null) {
            // This attribute is cached in the Segment Metadata, even if it has a value equal to Attributes.NULL_ATTRIBUTE_VALUE.
            result.put(attributeId, v);
        } else if (!Attributes.isCoreAttribute(attributeId)) {
            extendedAttributeIds.add(attributeId);
        }
    });
    if (extendedAttributeIds.isEmpty()) {
        // Nothing to lookup in the Attribute Index, so bail out early.
        return CompletableFuture.completedFuture(result);
    }
    // Collect remaining Extended Attributes.
    CompletableFuture<Map<AttributeId, Long>> r = this.attributeIndex.forSegment(segmentMetadata.getId(), timer.getRemaining()).thenComposeAsync(idx -> idx.get(extendedAttributeIds, timer.getRemaining()), this.executor).thenApplyAsync(extendedAttributes -> {
        if (extendedAttributeIds.size() == extendedAttributes.size()) {
            // We found a value for each Attribute Id. Nothing more to do.
            return extendedAttributes;
        }
        // Insert a NULL_ATTRIBUTE_VALUE for each missing value.
        Map<AttributeId, Long> allValues = new HashMap<>(extendedAttributes);
        extendedAttributeIds.stream().filter(id -> !extendedAttributes.containsKey(id)).forEach(id -> allValues.put(id, Attributes.NULL_ATTRIBUTE_VALUE));
        return allValues;
    }, this.executor);
    if (cache && !segmentMetadata.isSealed()) {
        // Add them to the cache if requested.
        r = r.thenComposeAsync(extendedAttributes -> {
            // Update the in-memory Segment Metadata using a special update (AttributeUpdateType.None, which should
            // complete if the attribute is not currently set). If it has some value, then a concurrent update
            // must have changed it and we cannot update anymore.
            val updates = new AttributeUpdateCollection();
            for (val e : extendedAttributes.entrySet()) {
                updates.add(new AttributeUpdate(e.getKey(), AttributeUpdateType.None, e.getValue()));
            }
            // invoke this one again.
            return addOperation(new UpdateAttributesOperation(segmentMetadata.getId(), updates), timer.getRemaining()).thenApply(v -> extendedAttributes);
        }, this.executor);
    }
    // Compile the final result.
    return r.thenApply(extendedAttributes -> {
        result.putAll(extendedAttributes);
        return result;
    });
}

Example 7 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class StreamSegmentContainer method mergeStreamSegment.

private CompletableFuture<MergeStreamSegmentResult> mergeStreamSegment(long targetSegmentId, long sourceSegmentId, AttributeUpdateCollection attributeUpdates, TimeoutTimer timer) {
    // Get a reference to the source segment's metadata now, before the merge. It may not be accessible afterwards.
    SegmentMetadata sourceMetadata = this.metadata.getStreamSegmentMetadata(sourceSegmentId);
    CompletableFuture<Void> sealResult = trySealStreamSegment(sourceMetadata, timer.getRemaining());
    if (sourceMetadata.getLength() == 0) {
        // writes in the pipeline. As such, we cannot pipeline the two operations, and must wait for the seal to finish first.
        return sealResult.thenComposeAsync(v -> {
            // to and including the seal, so if there were any writes outstanding before, they should now be reflected in it.
            if (sourceMetadata.getLength() == 0) {
                // Source is still empty after sealing - OK to delete.
                log.debug("{}: Updating attributes (if any) and deleting empty source segment instead of merging {}.", this.traceObjectId, sourceMetadata.getName());
                // Execute the attribute update on the target segment only if needed.
                Supplier<CompletableFuture<Void>> updateAttributesIfNeeded = () -> attributeUpdates == null ? CompletableFuture.completedFuture(null) : updateAttributesForSegment(targetSegmentId, attributeUpdates, timer.getRemaining());
                return updateAttributesIfNeeded.get().thenCompose(v2 -> deleteStreamSegment(sourceMetadata.getName(), timer.getRemaining()).thenApply(v3 -> new MergeStreamSegmentResult(this.metadata.getStreamSegmentMetadata(targetSegmentId).getLength(), sourceMetadata.getLength(), sourceMetadata.getAttributes())));
            } else {
                // Source now has some data - we must merge the two.
                MergeSegmentOperation operation = new MergeSegmentOperation(targetSegmentId, sourceSegmentId, attributeUpdates);
                return processAttributeUpdaterOperation(operation, timer).thenApply(v2 -> new MergeStreamSegmentResult(operation.getStreamSegmentOffset() + operation.getLength(), operation.getLength(), sourceMetadata.getAttributes()));
            }
        }, this.executor);
    } else {
        // Source is not empty, so we cannot delete. Make use of the DurableLog's pipelining abilities by queueing up
        // the Merge right after the Seal.
        MergeSegmentOperation operation = new MergeSegmentOperation(targetSegmentId, sourceSegmentId, attributeUpdates);
        return CompletableFuture.allOf(sealResult, processAttributeUpdaterOperation(operation, timer)).thenApply(v2 -> new MergeStreamSegmentResult(operation.getStreamSegmentOffset() + operation.getLength(), operation.getLength(), sourceMetadata.getAttributes()));
    }
}

Example 8 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class StreamSegmentContainer method mergeStreamSegment.

@Override
public CompletableFuture<MergeStreamSegmentResult> mergeStreamSegment(String targetStreamSegment, String sourceStreamSegment, AttributeUpdateCollection attributes, Duration timeout) {
    ensureRunning();
    logRequest("mergeStreamSegment", targetStreamSegment, sourceStreamSegment);
    this.metrics.mergeSegment();
    TimeoutTimer timer = new TimeoutTimer(timeout);
    // complete the cleanup phase, but still bubble up any exceptions to the caller.
    return this.metadataStore.getOrAssignSegmentId(targetStreamSegment, timer.getRemaining(), targetSegmentId -> this.metadataStore.getOrAssignSegmentId(sourceStreamSegment, timer.getRemaining(), sourceSegmentId -> mergeStreamSegment(targetSegmentId, sourceSegmentId, attributes, timer))).handleAsync((msr, ex) -> {
        if (ex == null || Exceptions.unwrap(ex) instanceof StreamSegmentMergedException) {
            // No exception or segment was already merged. Need to clear SegmentInfo for source.
            // We can do this asynchronously and not wait on it.
            this.metadataStore.clearSegmentInfo(sourceStreamSegment, timer.getRemaining());
        }
        if (ex == null) {
            // Everything is good. Return the result.
            return msr;
        } else {
            // Re-throw the exception to the caller in this case.
            throw new CompletionException(ex);
        }
    }, this.executor);
}

Example 9 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class ContainerRecoveryUtils method updateCoreAttributes.

/**
 * Updates Core Attributes for all Segments for the given Containers.
 * This method iterates through all the back copies of Container Metadata Segments, interprets all entries as
 * Segment-SegmentInfo mappings and extracts the Core Attributes for each. These Core Attributes are then applied
 * to the same segments in the given Containers.
 * @param backUpMetadataSegments    A map of back copies of metadata segments along with their container Ids.
 * @param containersMap             A map of {@link DebugStreamSegmentContainer} instances with their container Ids.
 * @param executorService           A thread pool for execution.
 * @param timeout                   Timeout for the operation.
 * @throws InterruptedException     If the operation was interrupted while waiting.
 * @throws TimeoutException         If the timeout expired prior to being able to complete update attributes for all segments.
 * @throws ExecutionException       When execution of update attributes to all segments encountered an error.
 */
public static void updateCoreAttributes(Map<Integer, String> backUpMetadataSegments, Map<Integer, DebugStreamSegmentContainer> containersMap, ExecutorService executorService, Duration timeout) throws InterruptedException, ExecutionException, TimeoutException {
    Preconditions.checkState(backUpMetadataSegments.size() == containersMap.size(), "The number of " + "back-up metadata segments = %s and the number of containers = %s should match.", backUpMetadataSegments.size(), containersMap.size());
    val args = IteratorArgs.builder().fetchTimeout(timeout).build();
    SegmentToContainerMapper segToConMapper = new SegmentToContainerMapper(containersMap.size(), true);
    // Iterate through all back up metadata segments
    for (val backUpMetadataSegmentEntry : backUpMetadataSegments.entrySet()) {
        // Get the name of original metadata segment
        val metadataSegment = NameUtils.getMetadataSegmentName(backUpMetadataSegmentEntry.getKey());
        // Get the name of back up metadata segment
        val backUpMetadataSegment = backUpMetadataSegmentEntry.getValue();
        // Get the container for back up metadata segment
        val containerForBackUpMetadataSegment = containersMap.get(segToConMapper.getContainerId(backUpMetadataSegment));
        log.info("Back up container metadata segment name: {} and its container id: {}", backUpMetadataSegment, containerForBackUpMetadataSegment.getId());
        // Get the container for segments inside back up metadata segment
        val container = containersMap.get(backUpMetadataSegmentEntry.getKey());
        // Make sure the backup segment is registered as a table segment.
        val bmsInfo = containerForBackUpMetadataSegment.getStreamSegmentInfo(backUpMetadataSegment, timeout).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        if (bmsInfo.getAttributes().getOrDefault(TableAttributes.INDEX_OFFSET, Attributes.NULL_ATTRIBUTE_VALUE) == Attributes.NULL_ATTRIBUTE_VALUE) {
            log.info("Back up container metadata segment name: {} does not have INDEX_OFFSET set; setting to 0 (forcing reindexing).", backUpMetadataSegment);
            containerForBackUpMetadataSegment.forSegment(backUpMetadataSegment, timeout).thenCompose(s -> s.updateAttributes(AttributeUpdateCollection.from(new AttributeUpdate(TableAttributes.INDEX_OFFSET, AttributeUpdateType.Replace, 0)), timeout)).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
            refreshDerivedProperties(backUpMetadataSegment, containerForBackUpMetadataSegment);
        }
        // Get the iterator to iterate through all segments in the back up metadata segment
        val tableExtension = containerForBackUpMetadataSegment.getExtension(ContainerTableExtension.class);
        val entryIterator = tableExtension.entryIterator(backUpMetadataSegment, args).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        val futures = new ArrayList<CompletableFuture<Void>>();
        // Iterating through all segments in the back up metadata segment
        entryIterator.forEachRemaining(item -> {
            for (val entry : item.getEntries()) {
                val segmentInfo = MetadataStore.SegmentInfo.deserialize(entry.getValue());
                val properties = segmentInfo.getProperties();
                // skip, if this is original metadata segment
                if (properties.getName().equals(metadataSegment)) {
                    continue;
                }
                // Get the attributes for the current segment
                val attributeUpdates = properties.getAttributes().entrySet().stream().map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())).collect(Collectors.toCollection(AttributeUpdateCollection::new));
                log.info("Segment Name: {} Attributes Updates: {}", properties.getName(), attributeUpdates);
                // Update attributes for the current segment
                futures.add(Futures.exceptionallyExpecting(container.updateAttributes(properties.getName(), attributeUpdates, timeout).thenRun(() -> refreshDerivedProperties(properties.getName(), container)), ex -> ex instanceof StreamSegmentNotExistsException, null));
            }
        }, executorService).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        // Waiting for update attributes for all segments in each back up metadata segment.
        Futures.allOf(futures).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
    }
}

Example 10 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class MetadataStore method createTransientSegment.

/**
 * Creates a new Transient Segment with given name.
 *
 * @param segmentName The case-sensitive Segment Name.
 * @param attributes  The initial attributes for the StreamSegment, if any.
 * @param timeout     Timeout for the operation.
 * @return A CompletableFuture that, when completed normally, will indicate the TransientSegment has been created.
 */
private CompletableFuture<Void> createTransientSegment(String segmentName, Collection<AttributeUpdate> attributes, Duration timeout) {
    AttributeUpdateCollection attrs = AttributeUpdateCollection.from(attributes);
    attrs.add(new AttributeUpdate(Attributes.CREATION_EPOCH, AttributeUpdateType.None, this.connector.containerMetadata.getContainerEpoch()));
    return Futures.toVoid(submitAssignmentWithRetry(newSegment(segmentName, SegmentType.TRANSIENT_SEGMENT, attrs), timeout));
}