Examples with SegmentContainer io.pravega.segmentstore.server.SegmentContainer used on opensource projects

Example 1 with SegmentContainer

use of io.pravega.segmentstore.server.SegmentContainer in project pravega by pravega.

the class StreamSegmentService method invoke.

// endregion
// region Helpers
/**
 * Executes the given Function on the SegmentContainer that the given Segment maps to.
 *
 * @param streamSegmentName The name of the StreamSegment to fetch the Container for.
 * @param toInvoke          A Function that will be invoked on the Container.
 * @param methodName        The name of the calling method (for logging purposes).
 * @param logArgs           (Optional) A vararg array of items to be logged.
 * @param <T>               Resulting type.
 * @return Either the result of toInvoke or a CompletableFuture completed exceptionally with a ContainerNotFoundException
 * in case the SegmentContainer that the Segment maps to does not exist in this StreamSegmentService.
 */
private <T> CompletableFuture<T> invoke(String streamSegmentName, Function<SegmentContainer, CompletableFuture<T>> toInvoke, String methodName, Object... logArgs) {
    long traceId = LoggerHelpers.traceEnter(log, methodName, logArgs);
    SegmentContainer container;
    try {
        int containerId = this.segmentToContainerMapper.getContainerId(streamSegmentName);
        container = this.segmentContainerRegistry.getContainer(containerId);
    } catch (ContainerNotFoundException ex) {
        return Futures.failedFuture(ex);
    }
    CompletableFuture<T> resultFuture = toInvoke.apply(container);
    if (log.isTraceEnabled()) {
        resultFuture.thenAccept(r -> LoggerHelpers.traceLeave(log, methodName, traceId, r));
    }
    return resultFuture;
}

Example 2 with SegmentContainer

use of io.pravega.segmentstore.server.SegmentContainer in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributeCleanup.

/**
 * Tests the ability to clean up Extended Attributes from Segment Metadatas that have not been used recently.
 * This test does the following:
 * 1. Sets up a custom SegmentContainer with a hook into the metadataCleanup task
 * 2. Creates a segment and appends something to it, each time updating attributes (and verifies they were updated correctly).
 * 3. Waits for the segment's attributes to be forgotten.
 * 4. Verifies that the forgotten attributes can be fetched from the Attribute Index and re-cached in memory.
 */
@Test
public void testAttributeCleanup() throws Exception {
    final String segmentName = "segment";
    final AttributeId[] attributes = new AttributeId[] { Attributes.EVENT_COUNT, AttributeId.uuid(0, 1), AttributeId.uuid(0, 2), AttributeId.uuid(0, 3) };
    Map<AttributeId, Long> allAttributes = new HashMap<>();
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(250));
    containerConfig.setMaxCachedExtendedAttributeCount(1);
    @Cleanup TestContext context = createContext(containerConfig);
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // Create segment with initial attributes and verify they were set correctly.
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Add one append with some attribute changes and verify they were set correctly.
    val appendAttributes = createAttributeUpdates(attributes);
    applyAttributes(appendAttributes, allAttributes);
    for (val au : appendAttributes) {
        localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(au), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    }
    SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after initial updateAttributes() call.", allAttributes, sp, AUTO_ATTRIBUTES);
    // Wait until the attributes are forgotten
    localContainer.triggerAttributeCleanup(segmentName, containerConfig.getMaxCachedExtendedAttributeCount()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Now get attributes again and verify them.
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // During attribute eviction, we expect all core attributes to be preserved, and only 1 extended attribute (as
    // defined in the config) to be preserved. This extended attribute should be the last one we updated.
    val expectedAttributes = new HashMap<>(Attributes.getCoreNonNullAttributes(allAttributes));
    val lastExtAttribute = appendAttributes.stream().filter(au -> !Attributes.isCoreAttribute(au.getAttributeId())).reduce((a, b) -> b).get();
    expectedAttributes.put(lastExtAttribute.getAttributeId(), lastExtAttribute.getValue());
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    val fetchedAttributes = localContainer.getAttributes(segmentName, allAttributes.keySet(), true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    AssertExtensions.assertMapEquals("Unexpected attributes after eviction & reload.", allAttributes, fetchedAttributes);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction & reload+getInfo.", allAttributes, sp, AUTO_ATTRIBUTES);
}

Example 3 with SegmentContainer

use of io.pravega.segmentstore.server.SegmentContainer in project pravega by pravega.

the class StreamSegmentContainerTests method waitForOperationsInReadIndex.

/**
 * Blocks until all operations processed so far have been added to the ReadIndex and InMemoryOperationLog.
 * This is needed to simplify test verification due to the fact that the the OperationProcessor commits operations to
 * the ReadIndex and InMemoryOperationLog asynchronously, after those operations were ack-ed. This method makes use
 * of the fact that the OperationProcessor/MemoryStateUpdater will still commit such operations in sequence; it
 * creates a new segment, writes 1 byte to it and issues a read (actual/future) and waits until it's completed - when
 * it is, it is guaranteed that everything prior to that has been committed.
 */
private static void waitForOperationsInReadIndex(SegmentContainer container) throws Exception {
    TimeoutTimer timer = new TimeoutTimer(TIMEOUT);
    String segmentName = "test" + System.nanoTime();
    container.createStreamSegment(segmentName, BASIC_TYPE, null, timer.getRemaining()).thenCompose(v -> container.append(segmentName, new ByteArraySegment(new byte[1]), null, timer.getRemaining())).thenCompose(v -> container.read(segmentName, 0, 1, timer.getRemaining())).thenCompose(rr -> {
        ReadResultEntry rre = rr.next();
        rre.requestContent(TIMEOUT);
        return rre.getContent().thenRun(rr::close);
    }).thenCompose(v -> container.deleteStreamSegment(segmentName, timer.getRemaining())).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
}

Example 4 with SegmentContainer

use of io.pravega.segmentstore.server.SegmentContainer in project pravega by pravega.

the class StreamSegmentContainerTests method testForceFlush.

/**
 * Tests the {@link SegmentContainer#flushToStorage} method.
 */
@Test
public void testForceFlush() throws Exception {
    final AttributeId attributeReplace = AttributeId.randomUUID();
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    final int entriesPerSegment = 10;
    @Cleanup TestContext context = new TestContext(DEFAULT_CONFIG, NO_TRUNCATIONS_DURABLE_LOG_CONFIG, INFREQUENT_FLUSH_WRITER_CONFIG, null);
    val durableLog = new AtomicReference<OperationLog>();
    val durableLogFactory = new WatchableOperationLogFactory(context.operationLogFactory, durableLog::set);
    @Cleanup val container = new StreamSegmentContainer(CONTAINER_ID, DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    container.startAsync().awaitRunning();
    Assert.assertNotNull(durableLog.get());
    val tableStore = container.getExtension(ContainerTableExtension.class);
    // 1. Create the StreamSegments and Table Segments.
    ArrayList<String> segmentNames = new ArrayList<>();
    ArrayList<String> tableSegmentNames = new ArrayList<>();
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        String segmentName = getSegmentName(i);
        segmentNames.add(segmentName);
        opFutures.add(container.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT));
    }
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        String segmentName = getSegmentName(i) + "_Table";
        tableSegmentNames.add(segmentName);
        val type = SegmentType.builder(getSegmentType(segmentName)).tableSegment().build();
        opFutures.add(tableStore.createSegment(segmentName, type, TIMEOUT));
    }
    // 1.1 Wait for all segments to be created prior to using them.
    Futures.allOf(opFutures).join();
    opFutures.clear();
    // 2. Add some appends and update some of the attributes.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    for (String segmentName : segmentNames) {
        for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
            val attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            val appendData = getAppendData(segmentName, i);
            long expectedLength = lengths.getOrDefault(segmentName, 0L) + appendData.getLength();
            val append = (i % 2 == 0) ? container.append(segmentName, appendData, attributeUpdates, TIMEOUT) : container.append(segmentName, lengths.get(segmentName), appendData, attributeUpdates, TIMEOUT);
            opFutures.add(Futures.toVoid(append));
            lengths.put(segmentName, expectedLength);
            recordAppend(segmentName, appendData, segmentContents, null);
        }
        for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
            val attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, APPENDS_PER_SEGMENT + i + 1));
            opFutures.add(container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    // 2.2 Add some entries to the table segments.
    final BiFunction<String, Integer, TableEntry> createTableEntry = (segmentName, entryId) -> TableEntry.unversioned(new ByteArraySegment(String.format("Key_%s_%s", segmentName, entryId).getBytes()), new ByteArraySegment(String.format("Value_%s_%s", segmentName, entryId).getBytes()));
    for (String segmentName : tableSegmentNames) {
        for (int i = 0; i < entriesPerSegment; i++) {
            opFutures.add(Futures.toVoid(tableStore.put(segmentName, Collections.singletonList(createTableEntry.apply(segmentName, i)), TIMEOUT)));
        }
    }
    Futures.allOf(opFutures).join();
    // 3. Instead of waiting for the Writer to move data to Storage, we invoke the flushToStorage to verify that all
    // operations have been applied to Storage.
    val forceFlush = container.flushToStorage(TIMEOUT);
    forceFlush.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, container, context.storage);
    // 4. Truncate all the data in the DurableLog and immediately shut down the container.
    val truncateSeqNo = container.metadata.getClosestValidTruncationPoint(container.metadata.getOperationSequenceNumber());
    durableLog.get().truncate(truncateSeqNo, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    container.close();
    // 5. Create a new container instance (from the nearly empty DurableLog) and with an empty cache.
    @Cleanup val container2 = new StreamSegmentContainer(CONTAINER_ID, DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    container2.startAsync().awaitRunning();
    // 5.1 Verify Segment Data.
    for (val sc : segmentContents.entrySet()) {
        // Contents.
        byte[] expectedData = sc.getValue().toByteArray();
        byte[] actualData = new byte[expectedData.length];
        container2.read(sc.getKey(), 0, actualData.length, TIMEOUT).join().readRemaining(actualData, TIMEOUT);
        Assert.assertArrayEquals("Unexpected contents for " + sc.getKey(), expectedData, actualData);
        // Length.
        val si = container2.getStreamSegmentInfo(sc.getKey(), TIMEOUT).join();
        Assert.assertEquals("Unexpected length for " + sc.getKey(), expectedData.length, si.getLength());
        // Attributes.
        val attributes = container2.getAttributes(sc.getKey(), Collections.singleton(attributeReplace), false, TIMEOUT).join();
        Assert.assertEquals("Unexpected attribute for " + sc.getKey(), expectedAttributeValue, (long) attributes.get(attributeReplace));
    }
    // 5.2 Verify table segment data.
    val tableStore2 = container2.getExtension(ContainerTableExtension.class);
    for (String segmentName : tableSegmentNames) {
        for (int i = 0; i < entriesPerSegment; i++) {
            val expected = createTableEntry.apply(segmentName, i);
            val actual = tableStore2.get(segmentName, Collections.singletonList(expected.getKey().getKey()), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).get(0);
            Assert.assertTrue("Unexpected Table Entry for " + segmentName + " at position " + i, expected.getKey().getKey().equals(actual.getKey().getKey()) && expected.getValue().equals(actual.getValue()));
        }
    }
// Ending Note: if all the above tests passed, we have implicitly validated that the Container Metadata Segment has also
// been properly flushed.
}

Example 5 with SegmentContainer

use of io.pravega.segmentstore.server.SegmentContainer in project pravega by pravega.

the class StreamSegmentContainerRegistryTests method testGetContainers.

/**
 * Tests the getContainers method for registered and unregistered containers.
 */
@Test
public void testGetContainers() throws Exception {
    final int containerCount = 1000;
    TestContainerFactory factory = new TestContainerFactory();
    @Cleanup StreamSegmentContainerRegistry registry = new StreamSegmentContainerRegistry(factory, executorService());
    HashSet<Integer> expectedContainerIds = new HashSet<>();
    for (int containerId = 0; containerId < containerCount; containerId++) {
        registry.startContainer(containerId, TIMEOUT);
        expectedContainerIds.add(containerId);
    }
    HashSet<Integer> actualHandleIds = new HashSet<>();
    for (SegmentContainer segmentContainer : registry.getContainers()) {
        actualHandleIds.add(segmentContainer.getId());
        Assert.assertTrue("Wrong container Java type.", segmentContainer instanceof TestContainer);
        segmentContainer.close();
    }
    AssertExtensions.assertContainsSameElements("Unexpected container ids registered.", expectedContainerIds, actualHandleIds);
    AssertExtensions.assertThrows("getContainer did not throw when passed an invalid container id.", () -> registry.getContainer(containerCount + 1), ex -> ex instanceof ContainerNotFoundException);
}