Examples with OperationLogFactory io.pravega.segmentstore.server.OperationLogFactory used on opensource projects

Example 6 with OperationLogFactory

use of io.pravega.segmentstore.server.OperationLogFactory in project pravega by pravega.

the class DebugStreamSegmentContainerTests method testRegisterExistingSegment.

/**
 * It tests the ability to register an existing segment(segment existing only in the Long-Term Storage) using debug
 * segment container. Method registerSegment in {@link DebugStreamSegmentContainer} is tested here.
 * The test starts a debug segment container and creates some segments using it and then verifies if the segments
 * were created successfully.
 */
@Test
public void testRegisterExistingSegment() {
    int maxSegmentCount = 100;
    final int createdSegmentCount = maxSegmentCount * 2;
    // Sets up dataLogFactory, readIndexFactory, attributeIndexFactory etc for the DebugSegmentContainer.
    @Cleanup TestContext context = createContext(executorService());
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    // Starts a DebugSegmentContainer.
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, CONTAINER_CONFIG, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    log.info("Started debug segment container.");
    // Record details(name, length & sealed status) of each segment to be created.
    ArrayList<String> segments = new ArrayList<>();
    ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
    long[] segmentLengths = new long[createdSegmentCount];
    boolean[] segmentSealedStatus = new boolean[createdSegmentCount];
    for (int i = 0; i < createdSegmentCount; i++) {
        segmentLengths[i] = MIN_SEGMENT_LENGTH + RANDOM.nextInt(MAX_SEGMENT_LENGTH - MIN_SEGMENT_LENGTH);
        segmentSealedStatus[i] = RANDOM.nextBoolean();
        String name = "Segment_" + i;
        segments.add(name);
        futures.add(localContainer.registerSegment(name, segmentLengths[i], segmentSealedStatus[i]));
    }
    // Creates all the segments.
    Futures.allOf(futures).join();
    log.info("Created the segments using debug segment container.");
    // Verify the Segments are still there with their length & sealed status.
    for (int i = 0; i < createdSegmentCount; i++) {
        SegmentProperties props = localContainer.getStreamSegmentInfo(segments.get(i), TIMEOUT).join();
        Assert.assertEquals("Segment length mismatch ", segmentLengths[i], props.getLength());
        Assert.assertEquals("Segment sealed status mismatch", segmentSealedStatus[i], props.isSealed());
    }
    localContainer.stopAsync().awaitTerminated();
}

Example 7 with OperationLogFactory

use of io.pravega.segmentstore.server.OperationLogFactory in project pravega by pravega.

the class DebugStreamSegmentContainerTests method testDataRecoveryContainerLevel.

/**
 * The test creates a container. Using it, some segments are created, data is written to them and their attributes are
 * updated. After Tier1 is flushed to the storage, the container is closed. From the storage, we recover the
 * segments and update their attributes. The data and attributes for each of the segment are verified to validate the
 * recovery process.
 */
@Test
public void testDataRecoveryContainerLevel() throws Exception {
    int attributesUpdatesPerSegment = 50;
    int segmentsCount = 10;
    final AttributeId attributeReplace = AttributeId.uuid(CORE_ATTRIBUTE_ID_PREFIX, RANDOM.nextInt(10));
    final long expectedAttributeValue = attributesUpdatesPerSegment;
    int containerId = 0;
    int containerCount = 1;
    // 1MB
    int maxDataSize = 1024 * 1024;
    StorageFactory storageFactory = new InMemoryStorageFactory(executorService());
    @Cleanup TestContext context = createContext(executorService());
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer container = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, storageFactory, context.getDefaultExtensions(), executorService());
    container.startAsync().awaitRunning();
    // 1. Create segments.
    ArrayList<String> segmentNames = new ArrayList<>();
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (int i = 0; i < segmentsCount; i++) {
        String segmentName = getSegmentName(i);
        segmentNames.add(segmentName);
        opFutures.add(container.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT));
    }
    // 1.1 Wait for all segments to be created prior to using them.
    Futures.allOf(opFutures).join();
    opFutures.clear();
    // 2. Write data and update some of the attributes.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArraySegment> segmentContents = new HashMap<>();
    for (String segmentName : segmentNames) {
        val dataSize = RANDOM.nextInt(maxDataSize);
        byte[] writeData = populate(dataSize);
        val appendData = new ByteArraySegment(writeData);
        val append = container.append(segmentName, appendData, null, TIMEOUT);
        opFutures.add(Futures.toVoid(append));
        lengths.put(segmentName, (long) dataSize);
        segmentContents.put(segmentName, appendData);
        for (int i = 0; i < attributesUpdatesPerSegment; i++) {
            AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            opFutures.add(container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Instead of waiting for the Writer to move data to Storage, we invoke the flushToStorage to verify that all
    // operations have been applied to Storage.
    val forceFlush = container.flushToStorage(TIMEOUT);
    forceFlush.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    container.close();
    // Get the storage instance
    @Cleanup Storage storage = storageFactory.createStorageAdapter();
    // 4. Move container metadata and its attribute segment to back up segments.
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    OperationLogFactory localDurableLogFactory2 = new DurableLogFactory(NO_TRUNCATIONS_DURABLE_LOG_CONFIG, new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), executorService());
    // Starts a DebugSegmentContainer with new Durable Log
    @Cleanup TestContext context1 = createContext(executorService());
    @Cleanup MetadataCleanupContainer container2 = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory2, context1.readIndexFactory, context1.attributeIndexFactory, context1.writerFactory, storageFactory, context1.getDefaultExtensions(), executorService());
    container2.startAsync().awaitRunning();
    Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainersMap = new HashMap<>();
    debugStreamSegmentContainersMap.put(containerId, container2);
    // 4. Recover all segments.
    recoverAllSegments(storage, debugStreamSegmentContainersMap, executorService(), TIMEOUT);
    // 5. Update core attributes using back up segments.
    updateCoreAttributes(backUpMetadataSegments, debugStreamSegmentContainersMap, executorService(), TIMEOUT);
    // 6. Verify Segment Data.
    for (val sc : segmentContents.entrySet()) {
        // Contents.
        byte[] expectedData = sc.getValue().array();
        byte[] actualData = new byte[expectedData.length];
        container2.read(sc.getKey(), 0, actualData.length, TIMEOUT).join().readRemaining(actualData, TIMEOUT);
        Assert.assertArrayEquals("Unexpected contents for " + sc.getKey(), expectedData, actualData);
        // Length.
        val si = container2.getStreamSegmentInfo(sc.getKey(), TIMEOUT).join();
        Assert.assertEquals("Unexpected length for " + sc.getKey(), expectedData.length, si.getLength());
        // Attributes.
        val attributes = container2.getAttributes(sc.getKey(), Collections.singleton(attributeReplace), false, TIMEOUT).join();
        Assert.assertEquals("Unexpected attribute for " + sc.getKey(), expectedAttributeValue, (long) attributes.get(attributeReplace));
    }
}

Example 8 with OperationLogFactory

use of io.pravega.segmentstore.server.OperationLogFactory in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributeIterators.

/**
 * Tests the ability to run attribute iterators over all or a subset of attributes in a segment.
 */
@Test
public void testAttributeIterators() throws Exception {
    final List<AttributeId> sortedAttributes = IntStream.range(0, 100).mapToObj(i -> AttributeId.uuid(i, i)).sorted().collect(Collectors.toList());
    final Map<AttributeId, Long> expectedValues = sortedAttributes.stream().collect(Collectors.toMap(id -> id, id -> id.getBitGroup(0)));
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT));
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the Segment.
    String segmentName = getSegmentName(0);
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val segment1 = localContainer.forSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 2. Set some initial attribute values and verify in-memory iterator.
    AttributeUpdateCollection attributeUpdates = sortedAttributes.stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Replace, expectedValues.get(attributeId))).collect(Collectors.toCollection(AttributeUpdateCollection::new));
    segment1.updateAttributes(attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment1, sortedAttributes, expectedValues);
    // 3. Force these segments out of memory and verify out-of-memory iterator.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val segment2 = localContainer.forSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment2, sortedAttributes, expectedValues);
    // 4. Update some values, and verify mixed iterator.
    attributeUpdates.clear();
    for (int i = 0; i < sortedAttributes.size(); i += 3) {
        AttributeId attributeId = sortedAttributes.get(i);
        expectedValues.put(attributeId, expectedValues.get(attributeId) + 1);
        attributeUpdates.add(new AttributeUpdate(attributeId, AttributeUpdateType.Replace, expectedValues.get(attributeId)));
    }
    segment2.updateAttributes(attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment2, sortedAttributes, expectedValues);
    localContainer.stopAsync().awaitTerminated();
}

Example 9 with OperationLogFactory

use of io.pravega.segmentstore.server.OperationLogFactory in project pravega by pravega.

the class StreamSegmentStoreTestBase method testSegmentRestoration.

/**
 * SegmentStore is used to create some segments, write data to them and let them flush to the storage.
 * This test only uses this storage to restore the container metadata segments in a new durable data log. Segment
 * properties are matched for verification after the restoration.
 * @throws Exception If an exception occurred.
 */
public void testSegmentRestoration() throws Exception {
    ArrayList<String> segmentNames;
    HashMap<String, ArrayList<String>> transactionsBySegment;
    HashMap<String, Long> lengths = new HashMap<>();
    ArrayList<ByteBuf> appendBuffers = new ArrayList<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    try (val builder = createBuilder(0, false)) {
        val segmentStore = builder.createStreamSegmentService();
        segmentNames = createSegments(segmentStore);
        log.info("Created Segments: {}.", String.join(", ", segmentNames));
        transactionsBySegment = createTransactions(segmentNames, segmentStore);
        log.info("Created Transactions: {}.", transactionsBySegment.values().stream().flatMap(Collection::stream).collect(Collectors.joining(", ")));
        // Add some appends and seal segments
        ArrayList<String> segmentsAndTransactions = new ArrayList<>(segmentNames);
        transactionsBySegment.values().forEach(segmentsAndTransactions::addAll);
        appendData(segmentsAndTransactions, segmentContents, lengths, appendBuffers, segmentStore).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        log.info("Finished appending data.");
        // Wait for flushing the segments to tier2
        waitForSegmentsInStorage(segmentNames, segmentStore).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        log.info("Finished waiting for segments in Storage.");
        // Get the persistent storage from readOnlySegmentStore.
        @Cleanup Storage storage = builder.createStorageFactory().createStorageAdapter();
        storage.initialize(DEFAULT_EPOCH);
        // Create the environment for DebugSegmentContainer using the given storageFactory.
        @Cleanup DebugStreamSegmentContainerTests.TestContext context = DebugStreamSegmentContainerTests.createContext(executorService());
        OperationLogFactory localDurableLogFactory = new DurableLogFactory(DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
        // Start a debug segment container corresponding to each container Id and put it in the Hashmap with the Id.
        Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainerMap = new HashMap<>();
        for (int containerId = 0; containerId < CONTAINER_COUNT; containerId++) {
            // Delete container metadata segment and attributes index segment corresponding to the container Id from the long term storage
            ContainerRecoveryUtils.deleteMetadataAndAttributeSegments(storage, containerId, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            DebugStreamSegmentContainerTests.MetadataCleanupContainer localContainer = new DebugStreamSegmentContainerTests.MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
            Services.startAsync(localContainer, executorService()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            debugStreamSegmentContainerMap.put(containerId, localContainer);
        }
        // Restore all segments from the long term storage using debug segment container.
        ContainerRecoveryUtils.recoverAllSegments(storage, debugStreamSegmentContainerMap, executorService(), TIMEOUT);
        // Verify that segment details match post restoration.
        SegmentToContainerMapper segToConMapper = new SegmentToContainerMapper(CONTAINER_COUNT, true);
        for (String segment : segmentNames) {
            int containerId = segToConMapper.getContainerId(segment);
            SegmentProperties props = debugStreamSegmentContainerMap.get(containerId).getStreamSegmentInfo(segment, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            Assert.assertEquals("Segment length mismatch.", (long) lengths.get(segment), props.getLength());
        }
        for (int containerId = 0; containerId < CONTAINER_COUNT; containerId++) {
            debugStreamSegmentContainerMap.get(containerId).close();
        }
    }
}

Example 10 with OperationLogFactory

use of io.pravega.segmentstore.server.OperationLogFactory in project pravega by pravega.

the class StreamSegmentContainerTests method testMetadataCleanup.

/**
 * Tests the ability to clean up SegmentMetadata for those segments which have not been used recently.
 * This test does the following:
 * 1. Sets up a custom SegmentContainer with a hook into the metadataCleanup task
 * 2. Creates a segment and appends something to it, each time updating attributes (and verifies they were updated correctly).
 * 3. Waits for the segment to be forgotten (evicted).
 * 4. Requests info on the segment, validates it, then makes another append, seals it, at each step verifying it was done
 * correctly (checking Metadata, Attributes and Storage).
 * 5. Deletes the segment, waits for metadata to be cleared (via forcing another log truncation), re-creates the
 * same segment and validates that the old attributes did not "bleed in".
 */
@Test
public void testMetadataCleanup() throws Exception {
    final String segmentName = "segment";
    final AttributeId[] attributes = new AttributeId[] { Attributes.CREATION_TIME, Attributes.EVENT_COUNT, AttributeId.randomUUID() };
    final ByteArraySegment appendData = new ByteArraySegment("hello".getBytes());
    Map<AttributeId, Long> expectedAttributes = new HashMap<>();
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT));
    @Cleanup TestContext context = createContext(containerConfig);
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // Create segment with initial attributes and verify they were set correctly.
    val initialAttributes = createAttributeUpdates(attributes);
    applyAttributes(initialAttributes, expectedAttributes);
    // We expect extended attributes to be dropped in this case.
    expectedAttributes = Attributes.getCoreNonNullAttributes(expectedAttributes);
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), initialAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after segment creation.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    // Add one append with some attribute changes and verify they were set correctly.
    val appendAttributes = createAttributeUpdates(attributes);
    applyAttributes(appendAttributes, expectedAttributes);
    localContainer.append(segmentName, appendData, appendAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after append.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    // Wait until the segment is forgotten.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Now get attributes again and verify them.
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // We expect extended attributes to be dropped in this case.
    expectedAttributes = Attributes.getCoreNonNullAttributes(expectedAttributes);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction & resurrection.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    // Append again, and make sure we can append at the right offset.
    val secondAppendAttributes = createAttributeUpdates(attributes);
    applyAttributes(secondAppendAttributes, expectedAttributes);
    localContainer.append(segmentName, appendData.getLength(), appendData, secondAppendAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertEquals("Unexpected length from segment after eviction & resurrection.", 2 * appendData.getLength(), sp.getLength());
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction & resurrection.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    // Seal.
    localContainer.sealStreamSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after seal.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    // Verify the segment actually made to Storage in one piece.
    waitForSegmentInStorage(sp, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val storageInfo = context.storage.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertEquals("Unexpected length in storage for segment.", sp.getLength(), storageInfo.getLength());
    // Delete segment and wait until it is forgotten again (we need to create another dummy segment so that we can
    // force a Metadata Truncation in order to facilitate that; this is the purpose of segment2).
    localContainer.deleteStreamSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Wait for the segment to be forgotten again.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Now Create the Segment again and verify the old attributes were not "remembered".
    val newAttributes = createAttributeUpdates(attributes);
    applyAttributes(newAttributes, expectedAttributes);
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), newAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // We expect extended attributes to be dropped in this case.
    expectedAttributes = Attributes.getCoreNonNullAttributes(expectedAttributes);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after deletion and re-creation.", expectedAttributes, sp, AUTO_ATTRIBUTES);
}