Examples with SegmentToContainerMapper io.pravega.shared.segment.SegmentToContainerMapper used on opensource projects

Example 1 with SegmentToContainerMapper

use of io.pravega.shared.segment.SegmentToContainerMapper in project pravega by pravega.

the class StreamSegmentStoreTestBase method testSegmentRestoration.

/**
 * SegmentStore is used to create some segments, write data to them and let them flush to the storage.
 * This test only uses this storage to restore the container metadata segments in a new durable data log. Segment
 * properties are matched for verification after the restoration.
 * @throws Exception If an exception occurred.
 */
public void testSegmentRestoration() throws Exception {
    ArrayList<String> segmentNames;
    HashMap<String, ArrayList<String>> transactionsBySegment;
    HashMap<String, Long> lengths = new HashMap<>();
    ArrayList<ByteBuf> appendBuffers = new ArrayList<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    try (val builder = createBuilder(0, false)) {
        val segmentStore = builder.createStreamSegmentService();
        segmentNames = createSegments(segmentStore);
        log.info("Created Segments: {}.", String.join(", ", segmentNames));
        transactionsBySegment = createTransactions(segmentNames, segmentStore);
        log.info("Created Transactions: {}.", transactionsBySegment.values().stream().flatMap(Collection::stream).collect(Collectors.joining(", ")));
        // Add some appends and seal segments
        ArrayList<String> segmentsAndTransactions = new ArrayList<>(segmentNames);
        transactionsBySegment.values().forEach(segmentsAndTransactions::addAll);
        appendData(segmentsAndTransactions, segmentContents, lengths, appendBuffers, segmentStore).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        log.info("Finished appending data.");
        // Wait for flushing the segments to tier2
        waitForSegmentsInStorage(segmentNames, segmentStore).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        log.info("Finished waiting for segments in Storage.");
        // Get the persistent storage from readOnlySegmentStore.
        @Cleanup Storage storage = builder.createStorageFactory().createStorageAdapter();
        storage.initialize(DEFAULT_EPOCH);
        // Create the environment for DebugSegmentContainer using the given storageFactory.
        @Cleanup DebugStreamSegmentContainerTests.TestContext context = DebugStreamSegmentContainerTests.createContext(executorService());
        OperationLogFactory localDurableLogFactory = new DurableLogFactory(DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
        // Start a debug segment container corresponding to each container Id and put it in the Hashmap with the Id.
        Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainerMap = new HashMap<>();
        for (int containerId = 0; containerId < CONTAINER_COUNT; containerId++) {
            // Delete container metadata segment and attributes index segment corresponding to the container Id from the long term storage
            ContainerRecoveryUtils.deleteMetadataAndAttributeSegments(storage, containerId, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            DebugStreamSegmentContainerTests.MetadataCleanupContainer localContainer = new DebugStreamSegmentContainerTests.MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
            Services.startAsync(localContainer, executorService()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            debugStreamSegmentContainerMap.put(containerId, localContainer);
        }
        // Restore all segments from the long term storage using debug segment container.
        ContainerRecoveryUtils.recoverAllSegments(storage, debugStreamSegmentContainerMap, executorService(), TIMEOUT);
        // Verify that segment details match post restoration.
        SegmentToContainerMapper segToConMapper = new SegmentToContainerMapper(CONTAINER_COUNT, true);
        for (String segment : segmentNames) {
            int containerId = segToConMapper.getContainerId(segment);
            SegmentProperties props = debugStreamSegmentContainerMap.get(containerId).getStreamSegmentInfo(segment, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            Assert.assertEquals("Segment length mismatch.", (long) lengths.get(segment), props.getLength());
        }
        for (int containerId = 0; containerId < CONTAINER_COUNT; containerId++) {
            debugStreamSegmentContainerMap.get(containerId).close();
        }
    }
}

Example 2 with SegmentToContainerMapper

use of io.pravega.shared.segment.SegmentToContainerMapper in project pravega by pravega.

the class ContainerRecoveryUtils method updateCoreAttributes.

/**
 * Updates Core Attributes for all Segments for the given Containers.
 * This method iterates through all the back copies of Container Metadata Segments, interprets all entries as
 * Segment-SegmentInfo mappings and extracts the Core Attributes for each. These Core Attributes are then applied
 * to the same segments in the given Containers.
 * @param backUpMetadataSegments    A map of back copies of metadata segments along with their container Ids.
 * @param containersMap             A map of {@link DebugStreamSegmentContainer} instances with their container Ids.
 * @param executorService           A thread pool for execution.
 * @param timeout                   Timeout for the operation.
 * @throws InterruptedException     If the operation was interrupted while waiting.
 * @throws TimeoutException         If the timeout expired prior to being able to complete update attributes for all segments.
 * @throws ExecutionException       When execution of update attributes to all segments encountered an error.
 */
public static void updateCoreAttributes(Map<Integer, String> backUpMetadataSegments, Map<Integer, DebugStreamSegmentContainer> containersMap, ExecutorService executorService, Duration timeout) throws InterruptedException, ExecutionException, TimeoutException {
    Preconditions.checkState(backUpMetadataSegments.size() == containersMap.size(), "The number of " + "back-up metadata segments = %s and the number of containers = %s should match.", backUpMetadataSegments.size(), containersMap.size());
    val args = IteratorArgs.builder().fetchTimeout(timeout).build();
    SegmentToContainerMapper segToConMapper = new SegmentToContainerMapper(containersMap.size(), true);
    // Iterate through all back up metadata segments
    for (val backUpMetadataSegmentEntry : backUpMetadataSegments.entrySet()) {
        // Get the name of original metadata segment
        val metadataSegment = NameUtils.getMetadataSegmentName(backUpMetadataSegmentEntry.getKey());
        // Get the name of back up metadata segment
        val backUpMetadataSegment = backUpMetadataSegmentEntry.getValue();
        // Get the container for back up metadata segment
        val containerForBackUpMetadataSegment = containersMap.get(segToConMapper.getContainerId(backUpMetadataSegment));
        log.info("Back up container metadata segment name: {} and its container id: {}", backUpMetadataSegment, containerForBackUpMetadataSegment.getId());
        // Get the container for segments inside back up metadata segment
        val container = containersMap.get(backUpMetadataSegmentEntry.getKey());
        // Make sure the backup segment is registered as a table segment.
        val bmsInfo = containerForBackUpMetadataSegment.getStreamSegmentInfo(backUpMetadataSegment, timeout).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        if (bmsInfo.getAttributes().getOrDefault(TableAttributes.INDEX_OFFSET, Attributes.NULL_ATTRIBUTE_VALUE) == Attributes.NULL_ATTRIBUTE_VALUE) {
            log.info("Back up container metadata segment name: {} does not have INDEX_OFFSET set; setting to 0 (forcing reindexing).", backUpMetadataSegment);
            containerForBackUpMetadataSegment.forSegment(backUpMetadataSegment, timeout).thenCompose(s -> s.updateAttributes(AttributeUpdateCollection.from(new AttributeUpdate(TableAttributes.INDEX_OFFSET, AttributeUpdateType.Replace, 0)), timeout)).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
            refreshDerivedProperties(backUpMetadataSegment, containerForBackUpMetadataSegment);
        }
        // Get the iterator to iterate through all segments in the back up metadata segment
        val tableExtension = containerForBackUpMetadataSegment.getExtension(ContainerTableExtension.class);
        val entryIterator = tableExtension.entryIterator(backUpMetadataSegment, args).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        val futures = new ArrayList<CompletableFuture<Void>>();
        // Iterating through all segments in the back up metadata segment
        entryIterator.forEachRemaining(item -> {
            for (val entry : item.getEntries()) {
                val segmentInfo = MetadataStore.SegmentInfo.deserialize(entry.getValue());
                val properties = segmentInfo.getProperties();
                // skip, if this is original metadata segment
                if (properties.getName().equals(metadataSegment)) {
                    continue;
                }
                // Get the attributes for the current segment
                val attributeUpdates = properties.getAttributes().entrySet().stream().map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())).collect(Collectors.toCollection(AttributeUpdateCollection::new));
                log.info("Segment Name: {} Attributes Updates: {}", properties.getName(), attributeUpdates);
                // Update attributes for the current segment
                futures.add(Futures.exceptionallyExpecting(container.updateAttributes(properties.getName(), attributeUpdates, timeout).thenRun(() -> refreshDerivedProperties(properties.getName(), container)), ex -> ex instanceof StreamSegmentNotExistsException, null));
            }
        }, executorService).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        // Waiting for update attributes for all segments in each back up metadata segment.
        Futures.allOf(futures).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
    }
}

Example 3 with SegmentToContainerMapper

use of io.pravega.shared.segment.SegmentToContainerMapper in project pravega by pravega.

the class ContainerRecoveryUtils method recoverAllSegments.

/**
 * Recovers Segments from the given Storage instance. This is done by:
 * 1. Listing all Segments from the given Storage instance and partitioning them by their assigned Container Id using
 * the standard {@link SegmentToContainerMapper}.
 * 2. Filtering out all shadow Segments (such as Attribute Segments).
 * 3. Registering all remaining (external) Segments to the owning Container's {@link MetadataStore}.
 *
 * The {@link DebugStreamSegmentContainer} instance(s) that are provided to this method may have some segments already
 * present in their respective {@link MetadataStore}.
 * After the method successfully completes, the following are true:
 * - Only the segments which exist in the {@link Storage} will remain in the Container's {@link MetadataStore}.
 * - If a Segment exists both in the Container's {@link MetadataStore} and in {@link Storage}, then the information
 * that exists in {@link Storage} (length, sealed) will prevail.
 *
 * If the method fails during execution, the appropriate exception is thrown and the Containers' {@link MetadataStore}
 * may be left in an inconsistent state.
 * @param storage                           A {@link Storage} instance that will be used to list segments from.
 * @param debugStreamSegmentContainersMap   A Map of Container Ids to {@link DebugStreamSegmentContainer} instances
 *                                          representing the containers that will be recovered.
 * @param executorService                   A thread pool for execution.
 * @param timeout                           Timeout for the operation.
 * @throws Exception                        If an exception occurred. This could be one of the following:
 *                                              * TimeoutException:     If the calls for computation(used in the method)
 *                                                                      didn't complete in time.
 *                                              * IOException     :     If a general IO exception occurred.
 */
public static void recoverAllSegments(Storage storage, Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainersMap, ExecutorService executorService, Duration timeout) throws Exception {
    Preconditions.checkNotNull(storage);
    Preconditions.checkNotNull(executorService);
    Preconditions.checkNotNull(debugStreamSegmentContainersMap);
    Preconditions.checkArgument(debugStreamSegmentContainersMap.size() > 0, "There should be at least one " + "debug segment container instance.");
    int containerCount = debugStreamSegmentContainersMap.size();
    validateContainerIds(debugStreamSegmentContainersMap, containerCount);
    log.info("Recovery started for all containers...");
    // Get all segments in the metadata store for each debug segment container instance.
    Map<Integer, Set<String>> existingSegmentsMap = getExistingSegments(debugStreamSegmentContainersMap, executorService, timeout);
    SegmentToContainerMapper segToConMapper = new SegmentToContainerMapper(containerCount, true);
    Iterator<SegmentProperties> segmentIterator = storage.listSegments().join();
    Preconditions.checkNotNull(segmentIterator);
    // Iterate through all segments. Create each one of their using their respective debugSegmentContainer instance.
    ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
    while (segmentIterator.hasNext()) {
        val currentSegment = segmentIterator.next();
        int containerId = segToConMapper.getContainerId(currentSegment.getName());
        String currentSegmentName = currentSegment.getName();
        // skip recovery if the segment is an attribute segment or metadata segment.
        if (NameUtils.isAttributeSegment(currentSegmentName) || getMetadataSegmentName(containerId).equals(currentSegmentName)) {
            continue;
        }
        existingSegmentsMap.get(containerId).remove(currentSegment.getName());
        futures.add(recoverSegment(debugStreamSegmentContainersMap.get(containerId), currentSegment, timeout));
    }
    Futures.allOf(futures).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
    futures.clear();
    // Delete segments which only exist in the container metadata, not in storage.
    for (val existingSegmentsSetEntry : existingSegmentsMap.entrySet()) {
        for (String segmentName : existingSegmentsSetEntry.getValue()) {
            log.info("Deleting segment '{}' as it is not in the storage.", segmentName);
            futures.add(debugStreamSegmentContainersMap.get(existingSegmentsSetEntry.getKey()).deleteStreamSegment(segmentName, timeout));
        }
    }
    Futures.allOf(futures).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
}

Example 4 with SegmentToContainerMapper

use of io.pravega.shared.segment.SegmentToContainerMapper in project pravega by pravega.

the class DebugStreamSegmentContainerTests method testDataRecoveryStorageLevel.

/**
 * Use a storage instance to create segments. Lists the segments from the storage and and then recreates them using
 * debug segment containers. Before re-creating(or registering), the segments are mapped to their respective debug
 * segment container. Once registered, segment's properties are matched to verify if the test was successful or not.
 */
@Test
public void testDataRecoveryStorageLevel() throws Exception {
    // Segments are mapped to four different containers.
    int containerCount = 4;
    int segmentsToCreateCount = 50;
    // Create a storage.
    @Cleanup val baseStorage = new InMemoryStorage();
    @Cleanup val s = new RollingStorage(baseStorage, new SegmentRollingPolicy(1));
    s.initialize(1);
    log.info("Created a storage instance");
    // Record details(name, container Id & sealed status) of each segment to be created.
    Set<String> sealedSegments = new HashSet<>();
    byte[] data = "data".getBytes();
    SegmentToContainerMapper segToConMapper = new SegmentToContainerMapper(containerCount, true);
    Map<Integer, ArrayList<String>> segmentByContainers = new HashMap<>();
    // Create segments and get their container Ids, sealed status and names to verify.
    for (int i = 0; i < segmentsToCreateCount; i++) {
        String segmentName = "segment-" + RANDOM.nextInt();
        // Use segmentName to map to different containers.
        int containerId = segToConMapper.getContainerId(segmentName);
        ArrayList<String> segmentsList = segmentByContainers.get(containerId);
        if (segmentsList == null) {
            segmentsList = new ArrayList<>();
            segmentByContainers.put(containerId, segmentsList);
        }
        segmentByContainers.get(containerId).add(segmentName);
        // Create segments, write data and randomly seal some of them.
        val wh1 = s.create(segmentName);
        // Write data.
        s.write(wh1, 0, new ByteArrayInputStream(data), data.length);
        if (RANDOM.nextBoolean()) {
            s.seal(wh1);
            sealedSegments.add(segmentName);
        }
    }
    log.info("Created some segments using the storage.");
    @Cleanup TestContext context = createContext(executorService());
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainerMap = new HashMap<>();
    log.info("Start a debug segment container corresponding to each container id.");
    for (int containerId = 0; containerId < containerCount; containerId++) {
        MetadataCleanupContainer localContainer = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
        Services.startAsync(localContainer, executorService()).join();
        debugStreamSegmentContainerMap.put(containerId, localContainer);
    }
    log.info("Recover all segments using the storage and debug segment containers.");
    recoverAllSegments(new AsyncStorageWrapper(s, executorService()), debugStreamSegmentContainerMap, executorService(), TIMEOUT);
    // Re-create all segments which were listed.
    for (int containerId = 0; containerId < containerCount; containerId++) {
        for (String segment : segmentByContainers.get(containerId)) {
            SegmentProperties props = debugStreamSegmentContainerMap.get(containerId).getStreamSegmentInfo(segment, TIMEOUT).join();
            Assert.assertEquals("Segment length mismatch.", data.length, props.getLength());
            Assert.assertEquals("Sealed status of the segment don't match.", sealedSegments.contains(segment), props.isSealed());
        }
        debugStreamSegmentContainerMap.get(containerId).close();
    }
}