Examples with DurableLogConfig io.pravega.segmentstore.server.logs.DurableLogConfig used on opensource projects

Example 1 with DurableLogConfig

use of io.pravega.segmentstore.server.logs.DurableLogConfig in project pravega by pravega.

the class ServiceBuilder method createOperationLogFactory.

protected OperationLogFactory createOperationLogFactory() {
    DurableDataLogFactory dataLogFactory = getSingleton(this.dataLogFactory, this.dataLogFactoryCreator);
    DurableLogConfig durableLogConfig = this.serviceBuilderConfig.getConfig(DurableLogConfig::builder);
    return new DurableLogFactory(durableLogConfig, dataLogFactory, this.coreExecutor);
}

Example 2 with DurableLogConfig

use of io.pravega.segmentstore.server.logs.DurableLogConfig in project pravega by pravega.

the class ServiceBuilderConfigTests method testGetConfig.

/**
 * Verifies the include(ConfigBuilder) and getConfig() work properly. This test follows a bit of an unusual approach
 * in that it picks a few known config classes, populates them using their builders and reflection, then uses reflection
 * once again to compare expected output (generated using their builders) with output from the ServiceBuilderConfig.getConfig().
 * <p>
 * This verifies that the namespacing inside ServiceBuilderConfig.builder() works correctly, as well as the constructors
 * for various configs.
 */
@Test
@SuppressWarnings("unchecked")
public void testGetConfig() throws Exception {
    // Select a few classes to test dynamically.
    val testClasses = new HashMap<Class<?>, Supplier<ConfigBuilder<?>>>();
    testClasses.put(ReadIndexConfig.class, ReadIndexConfig::builder);
    testClasses.put(WriterConfig.class, WriterConfig::builder);
    testClasses.put(MetricsConfig.class, MetricsConfig::builder);
    testClasses.put(DurableLogConfig.class, DurableLogConfig::builder);
    testClasses.put(ServiceConfig.class, ServiceConfig::builder);
    // Value generator.
    val nextValue = new AtomicInteger(1000 * 1000 * 1000);
    // Create instances of each test class and dynamically assign their properties some arbitrary values
    val expectedValues = new HashMap<Class<?>, Object>();
    val b = ServiceBuilderConfig.builder();
    for (Map.Entry<Class<?>, Supplier<ConfigBuilder<?>>> e : testClasses.entrySet()) {
        Class<?> c = e.getKey();
        ConfigBuilder<?> configBuilder = e.getValue().get();
        for (Field f : c.getDeclaredFields()) {
            // type contains one of the supported types.
            if (Modifier.isStatic(f.getModifiers()) && f.getType().isAssignableFrom(Property.class) && isSupportedType(f.getGenericType().getTypeName())) {
                Property p = (Property) f.get(null);
                if (p.getDefaultValue() != null && p.getDefaultValue() instanceof Boolean) {
                    configBuilder.with(p, nextValue.incrementAndGet() % 2 == 0);
                } else {
                    // Any number can be interpreted as a string or number.
                    configBuilder.with(p, Integer.toString(nextValue.incrementAndGet()));
                }
            }
        }
        // Collect the built config object for later use.
        expectedValues.put(c, configBuilder.build());
        // Include the builder in the main builder.
        b.include(configBuilder);
    }
    // Create the ServiceBuilderConfig, and verify that the created Config classes (using getConfig()) match the
    // expected ones.
    val builderConfig = b.build();
    for (Map.Entry<Class<?>, Supplier<ConfigBuilder<?>>> e : testClasses.entrySet()) {
        Class<?> c = e.getKey();
        Object expectedConfig = expectedValues.get(c);
        Object actualConfig = builderConfig.getConfig(e.getValue());
        // All the properties we care about are public getters with no arguments - only check those.
        for (Method m : c.getDeclaredMethods()) {
            if (m.getName().startsWith("get") && m.getParameterCount() == 0 && !Modifier.isStatic(m.getModifiers()) && Modifier.isPublic(m.getModifiers())) {
                Object expectedValue = m.invoke(expectedConfig);
                Object actualValue = m.invoke(actualConfig);
                if (expectedValue == null) {
                    Assert.assertNull("Expected a null value for " + getPropName(c, m), actualValue);
                } else {
                    Assert.assertNotNull("Not expected a null value for " + getPropName(c, m), actualValue);
                }
                if (isSupportedType(expectedValue)) {
                    Assert.assertEquals("Unexpected value for " + getPropName(c, m), expectedValue, actualValue);
                }
            }
        }
    }
}

Example 3 with DurableLogConfig

use of io.pravega.segmentstore.server.logs.DurableLogConfig in project pravega by pravega.

the class StreamSegmentContainerTests method testMetadataCleanup.

/**
 * Tests the ability to clean up SegmentMetadata for those segments which have not been used recently.
 * This test does the following:
 * 1. Sets up a custom SegmentContainer with a hook into the metadataCleanup task
 * 2. Creates a segment and appends something to it, each time updating attributes (and verifies they were updated correctly).
 * 3. Waits for the segment to be forgotten (evicted).
 * 4. Requests info on the segment, validates it, then makes another append, seals it, at each step verifying it was done
 * correctly (checking Metadata, Attributes and Storage).
 * 5. Deletes the segment, waits for metadata to be cleared (via forcing another log truncation), re-creates the
 * same segment and validates that the old attributes did not "bleed in".
 */
@Test
public void testMetadataCleanup() throws Exception {
    final String segmentName = "segment";
    final UUID[] attributes = new UUID[] { Attributes.CREATION_TIME, UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID() };
    final byte[] appendData = "hello".getBytes();
    final Map<UUID, Long> expectedAttributes = new HashMap<>();
    // We need a special DL config so that we can force truncations after every operation - this will speed up metadata
    // eviction eligibility.
    final DurableLogConfig durableLogConfig = DurableLogConfig.builder().with(DurableLogConfig.CHECKPOINT_MIN_COMMIT_COUNT, 1).with(DurableLogConfig.CHECKPOINT_COMMIT_COUNT, 5).with(DurableLogConfig.CHECKPOINT_TOTAL_COMMIT_LENGTH, 10 * 1024 * 1024L).build();
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(250));
    @Cleanup TestContext context = new TestContext(containerConfig);
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(durableLogConfig, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.writerFactory, context.storageFactory, executorService());
    localContainer.startAsync().awaitRunning();
    // Create segment with initial attributes and verify they were set correctly.
    val initialAttributes = createAttributeUpdates(attributes);
    applyAttributes(initialAttributes, expectedAttributes);
    localContainer.createStreamSegment(segmentName, initialAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after segment creation.", expectedAttributes, sp);
    // Add one append with some attribute changes and verify they were set correctly.
    val appendAttributes = createAttributeUpdates(attributes);
    applyAttributes(appendAttributes, expectedAttributes);
    localContainer.append(segmentName, appendData, appendAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after append.", expectedAttributes, sp);
    // Wait until the segment is forgotten.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Now get attributes again and verify them.
    sp = localContainer.getStreamSegmentInfo(segmentName, true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction & resurrection.", expectedAttributes, sp);
    // Append again, and make sure we can append at the right offset.
    val secondAppendAttributes = createAttributeUpdates(attributes);
    applyAttributes(secondAppendAttributes, expectedAttributes);
    localContainer.append(segmentName, appendData.length, appendData, secondAppendAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertEquals("Unexpected length from segment after eviction & resurrection.", 2 * appendData.length, sp.getLength());
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction & resurrection.", expectedAttributes, sp);
    // Seal (this should clear out non-dynamic attributes).
    expectedAttributes.keySet().removeIf(Attributes::isDynamic);
    localContainer.sealStreamSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after seal.", expectedAttributes, sp);
    // Verify the segment actually made to Storage in one piece.
    waitForSegmentInStorage(sp, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val storageInfo = context.storage.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertEquals("Unexpected length in storage for segment.", sp.getLength(), storageInfo.getLength());
    // Delete segment and wait until it is forgotten again (we need to create another dummy segment so that we can
    // force a Metadata Truncation in order to facilitate that; this is the purpose of segment2).
    localContainer.deleteStreamSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Wait for the segment to be forgotten again.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Now Create the Segment again and verify the old attributes were not "remembered".
    val newAttributes = createAttributeUpdates(attributes);
    applyAttributes(newAttributes, expectedAttributes);
    localContainer.createStreamSegment(segmentName, newAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after deletion and re-creation.", expectedAttributes, sp);
}

Example 4 with DurableLogConfig

use of io.pravega.segmentstore.server.logs.DurableLogConfig in project pravega by pravega.

the class StreamSegmentContainerTests method testForcedMetadataCleanup.

/**
 * Tests the case when the ContainerMetadata has filled up to capacity (with segments and we cannot map anymore segments).
 */
@Test
public void testForcedMetadataCleanup() throws Exception {
    final int maxSegmentCount = 3;
    final ContainerConfig containerConfig = ContainerConfig.builder().with(ContainerConfig.SEGMENT_METADATA_EXPIRATION_SECONDS, (int) DEFAULT_CONFIG.getSegmentMetadataExpiration().getSeconds()).with(ContainerConfig.MAX_ACTIVE_SEGMENT_COUNT, maxSegmentCount).build();
    // We need a special DL config so that we can force truncations after every operation - this will speed up metadata
    // eviction eligibility.
    final DurableLogConfig durableLogConfig = DurableLogConfig.builder().with(DurableLogConfig.CHECKPOINT_MIN_COMMIT_COUNT, 1).with(DurableLogConfig.CHECKPOINT_COMMIT_COUNT, 5).with(DurableLogConfig.CHECKPOINT_TOTAL_COMMIT_LENGTH, 10L * 1024 * 1024).build();
    @Cleanup TestContext context = new TestContext(containerConfig);
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(durableLogConfig, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.writerFactory, context.storageFactory, executorService());
    localContainer.startAsync().awaitRunning();
    // Create 4 segments and one transaction.
    String segment0 = getSegmentName(0);
    localContainer.createStreamSegment(segment0, null, TIMEOUT).join();
    String segment1 = getSegmentName(1);
    localContainer.createStreamSegment(segment1, null, TIMEOUT).join();
    String segment2 = getSegmentName(2);
    localContainer.createStreamSegment(segment2, null, TIMEOUT).join();
    String segment3 = getSegmentName(3);
    localContainer.createStreamSegment(segment3, null, TIMEOUT).join();
    String txn1 = localContainer.createTransaction(segment3, UUID.randomUUID(), null, TIMEOUT).join();
    // Activate one segment.
    activateSegment(segment2, localContainer).join();
    // Activate the transaction; this should fill up the metadata (itself + parent).
    activateSegment(txn1, localContainer).join();
    // Verify the transaction's parent has been activated.
    Assert.assertNotNull("Transaction's parent has not been activated.", localContainer.getStreamSegmentInfo(segment3, false, TIMEOUT).join());
    // At this point, the active segments should be: 2, 3 and Txn.
    // Verify we cannot activate any other segment.
    AssertExtensions.assertThrows("getSegmentId() allowed mapping more segments than the metadata can support.", () -> activateSegment(segment1, localContainer), ex -> ex instanceof TooManyActiveSegmentsException);
    AssertExtensions.assertThrows("getSegmentId() allowed mapping more segments than the metadata can support.", () -> activateSegment(segment0, localContainer), ex -> ex instanceof TooManyActiveSegmentsException);
    // Test the ability to forcefully evict items from the metadata when there is pressure and we need to register something new.
    // Case 1: following a Segment deletion.
    localContainer.deleteStreamSegment(segment2, TIMEOUT).join();
    val segment1Activation = tryActivate(localContainer, segment1, segment3);
    val segment1Info = segment1Activation.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertNotNull("Unable to properly activate dormant segment (1).", segment1Info);
    // Case 2: following a Merge.
    localContainer.sealStreamSegment(txn1, TIMEOUT).join();
    localContainer.mergeTransaction(txn1, TIMEOUT).join();
    val segment0Activation = tryActivate(localContainer, segment0, segment3);
    val segment0Info = segment0Activation.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertNotNull("Unable to properly activate dormant segment (0).", segment0Info);
    tryActivate(localContainer, segment1, segment3).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // At this point the active segments should be: 0, 1 and 3.
    Assert.assertNotNull("Pre-activated segment did not stay in metadata (3).", localContainer.getStreamSegmentInfo(segment3, false, TIMEOUT).join());
    Assert.assertNotNull("Pre-activated segment did not stay in metadata (1).", localContainer.getStreamSegmentInfo(segment1, false, TIMEOUT).join());
    Assert.assertNotNull("Pre-activated segment did not stay in metadata (0).", localContainer.getStreamSegmentInfo(segment0, false, TIMEOUT).join());
    context.container.stopAsync().awaitTerminated();
}