Examples with NameUtils.getMetadataSegmentName io.pravega.shared.NameUtils.getMetadataSegmentName used on opensource projects

Example 1 with NameUtils.getMetadataSegmentName

use of io.pravega.shared.NameUtils.getMetadataSegmentName in project pravega by pravega.

the class ContainerRecoveryUtils method updateCoreAttributes.

/**
 * Updates Core Attributes for all Segments for the given Containers.
 * This method iterates through all the back copies of Container Metadata Segments, interprets all entries as
 * Segment-SegmentInfo mappings and extracts the Core Attributes for each. These Core Attributes are then applied
 * to the same segments in the given Containers.
 * @param backUpMetadataSegments    A map of back copies of metadata segments along with their container Ids.
 * @param containersMap             A map of {@link DebugStreamSegmentContainer} instances with their container Ids.
 * @param executorService           A thread pool for execution.
 * @param timeout                   Timeout for the operation.
 * @throws InterruptedException     If the operation was interrupted while waiting.
 * @throws TimeoutException         If the timeout expired prior to being able to complete update attributes for all segments.
 * @throws ExecutionException       When execution of update attributes to all segments encountered an error.
 */
public static void updateCoreAttributes(Map<Integer, String> backUpMetadataSegments, Map<Integer, DebugStreamSegmentContainer> containersMap, ExecutorService executorService, Duration timeout) throws InterruptedException, ExecutionException, TimeoutException {
    Preconditions.checkState(backUpMetadataSegments.size() == containersMap.size(), "The number of " + "back-up metadata segments = %s and the number of containers = %s should match.", backUpMetadataSegments.size(), containersMap.size());
    val args = IteratorArgs.builder().fetchTimeout(timeout).build();
    SegmentToContainerMapper segToConMapper = new SegmentToContainerMapper(containersMap.size(), true);
    // Iterate through all back up metadata segments
    for (val backUpMetadataSegmentEntry : backUpMetadataSegments.entrySet()) {
        // Get the name of original metadata segment
        val metadataSegment = NameUtils.getMetadataSegmentName(backUpMetadataSegmentEntry.getKey());
        // Get the name of back up metadata segment
        val backUpMetadataSegment = backUpMetadataSegmentEntry.getValue();
        // Get the container for back up metadata segment
        val containerForBackUpMetadataSegment = containersMap.get(segToConMapper.getContainerId(backUpMetadataSegment));
        log.info("Back up container metadata segment name: {} and its container id: {}", backUpMetadataSegment, containerForBackUpMetadataSegment.getId());
        // Get the container for segments inside back up metadata segment
        val container = containersMap.get(backUpMetadataSegmentEntry.getKey());
        // Make sure the backup segment is registered as a table segment.
        val bmsInfo = containerForBackUpMetadataSegment.getStreamSegmentInfo(backUpMetadataSegment, timeout).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        if (bmsInfo.getAttributes().getOrDefault(TableAttributes.INDEX_OFFSET, Attributes.NULL_ATTRIBUTE_VALUE) == Attributes.NULL_ATTRIBUTE_VALUE) {
            log.info("Back up container metadata segment name: {} does not have INDEX_OFFSET set; setting to 0 (forcing reindexing).", backUpMetadataSegment);
            containerForBackUpMetadataSegment.forSegment(backUpMetadataSegment, timeout).thenCompose(s -> s.updateAttributes(AttributeUpdateCollection.from(new AttributeUpdate(TableAttributes.INDEX_OFFSET, AttributeUpdateType.Replace, 0)), timeout)).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
            refreshDerivedProperties(backUpMetadataSegment, containerForBackUpMetadataSegment);
        }
        // Get the iterator to iterate through all segments in the back up metadata segment
        val tableExtension = containerForBackUpMetadataSegment.getExtension(ContainerTableExtension.class);
        val entryIterator = tableExtension.entryIterator(backUpMetadataSegment, args).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        val futures = new ArrayList<CompletableFuture<Void>>();
        // Iterating through all segments in the back up metadata segment
        entryIterator.forEachRemaining(item -> {
            for (val entry : item.getEntries()) {
                val segmentInfo = MetadataStore.SegmentInfo.deserialize(entry.getValue());
                val properties = segmentInfo.getProperties();
                // skip, if this is original metadata segment
                if (properties.getName().equals(metadataSegment)) {
                    continue;
                }
                // Get the attributes for the current segment
                val attributeUpdates = properties.getAttributes().entrySet().stream().map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())).collect(Collectors.toCollection(AttributeUpdateCollection::new));
                log.info("Segment Name: {} Attributes Updates: {}", properties.getName(), attributeUpdates);
                // Update attributes for the current segment
                futures.add(Futures.exceptionallyExpecting(container.updateAttributes(properties.getName(), attributeUpdates, timeout).thenRun(() -> refreshDerivedProperties(properties.getName(), container)), ex -> ex instanceof StreamSegmentNotExistsException, null));
            }
        }, executorService).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        // Waiting for update attributes for all segments in each back up metadata segment.
        Futures.allOf(futures).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
    }
}

Example 2 with NameUtils.getMetadataSegmentName

use of io.pravega.shared.NameUtils.getMetadataSegmentName in project pravega by pravega.

the class ContainerRecoveryUtils method createBackUpMetadataSegments.

/**
 * This method creates a back up segment of container metadata segment and its attribute segment for each
 * container Id. The original metadata segments and its attribute segments are deleted and the back up copy
 * of original metadata segments are stored in a map and returned.
 *
 * @param storage                   A {@link Storage} instance to get the segments from.
 * @param containerCount            The number of containers for which renaming of container metadata segment and its
 *                                  attributes segment has to be performed.
 * @param executorService           A thread pool for execution.
 * @param timeout                   Timeout for the operation.
 * @return                          A CompletableFuture that, when completed normally, will have a Map of Container
 * Ids to new container metadata segment names.
 */
public static synchronized CompletableFuture<Map<Integer, String>> createBackUpMetadataSegments(Storage storage, int containerCount, ExecutorService executorService, Duration timeout) {
    String fileSuffix = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
    Map<Integer, String> backUpMetadataSegments = new HashMap<>();
    val futures = new ArrayList<CompletableFuture<Void>>();
    for (int containerId = 0; containerId < containerCount; containerId++) {
        String backUpMetadataSegment = NameUtils.getMetadataSegmentName(containerId) + fileSuffix;
        String backUpAttributeSegment = NameUtils.getAttributeSegmentName(backUpMetadataSegment);
        log.debug("Created '{}' as a back of metadata segment of container Id '{}'", backUpMetadataSegment, containerId);
        val finalContainerId = containerId;
        futures.add(Futures.exceptionallyExpecting(ContainerRecoveryUtils.backUpMetadataAndAttributeSegments(storage, containerId, backUpMetadataSegment, backUpAttributeSegment, executorService, timeout).thenCompose(x -> ContainerRecoveryUtils.deleteMetadataAndAttributeSegments(storage, finalContainerId, timeout)), ex -> Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException, null));
        backUpMetadataSegments.put(finalContainerId, backUpMetadataSegment);
    }
    return Futures.allOf(futures).thenApply(v -> backUpMetadataSegments);
}