Examples with AttributeId io.pravega.segmentstore.contracts.AttributeId used on opensource projects

Example 16 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentContainerTests method testExtendedAttributesConditionalUpdates.

/**
 * Test conditional updates for Extended Attributes when they are not loaded in memory (i.e., they will need to be
 * auto-fetched from the AttributeIndex so that the operation may succeed).
 */
@Test
public void testExtendedAttributesConditionalUpdates() throws Exception {
    final AttributeId ea1 = AttributeId.uuid(0, 1);
    final AttributeId ea2 = AttributeId.uuid(0, 2);
    final List<AttributeId> allAttributes = Stream.of(ea1, ea2).collect(Collectors.toList());
    // We set a longer Segment Expiration time, since this test executes more operations than the others.
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_LONG));
    AtomicInteger expectedAttributeValue = new AtomicInteger(0);
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the StreamSegments and set the initial attribute values.
    ArrayList<String> segmentNames = createSegments(localContainer);
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    // 2. Update some of the attributes.
    expectedAttributeValue.set(1);
    for (String segmentName : segmentNames) {
        AttributeUpdateCollection attributeUpdates = allAttributes.stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Accumulate, 1)).collect(Collectors.toCollection(AttributeUpdateCollection::new));
        opFutures.add(localContainer.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Force these segments out of memory, so that we may verify conditional appends/updates on extended attributes.
    localContainer.triggerMetadataCleanup(segmentNames).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Execute various conditional operations using these attributes as comparison.
    long compare = expectedAttributeValue.getAndIncrement();
    long set = expectedAttributeValue.get();
    boolean badUpdate = false;
    for (String segmentName : segmentNames) {
        if (badUpdate) {
            // For every other segment, try to do a bad update, then a good one. This helps us verify both direct
            // conditional operations, failed conditional operations and conditional operations with cached attributes.
            AssertExtensions.assertSuppliedFutureThrows("Conditional append succeeded with incorrect compare value.", () -> localContainer.append(segmentName, getAppendData(segmentName, 0), AttributeUpdateCollection.from(new AttributeUpdate(ea1, AttributeUpdateType.ReplaceIfEquals, set, compare - 1)), TIMEOUT), ex -> (ex instanceof BadAttributeUpdateException) && !((BadAttributeUpdateException) ex).isPreviousValueMissing());
            AssertExtensions.assertSuppliedFutureThrows("Conditional update-attributes succeeded with incorrect compare value.", () -> localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(ea2, AttributeUpdateType.ReplaceIfEquals, set, compare - 1)), TIMEOUT), ex -> (ex instanceof BadAttributeUpdateException) && !((BadAttributeUpdateException) ex).isPreviousValueMissing());
        }
        opFutures.add(Futures.toVoid(localContainer.append(segmentName, getAppendData(segmentName, 0), AttributeUpdateCollection.from(new AttributeUpdate(ea1, AttributeUpdateType.ReplaceIfEquals, set, compare)), TIMEOUT)));
        opFutures.add(localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(ea2, AttributeUpdateType.ReplaceIfEquals, set, compare)), TIMEOUT));
        badUpdate = !badUpdate;
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Evict the segment from memory, then verify results.
    localContainer.triggerMetadataCleanup(segmentNames).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    for (String segmentName : segmentNames) {
        // Verify all attribute values.
        val attributeValues = localContainer.getAttributes(segmentName, allAttributes, true, TIMEOUT).join();
        val sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        for (val attributeId : allAttributes) {
            Assert.assertEquals("Unexpected value for non-cached attribute " + attributeId + " for segment " + segmentName, expectedAttributeValue.get(), (long) attributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            Assert.assertEquals("Unexpected value for metadata attribute " + attributeId + " for segment " + segmentName, expectedAttributeValue.get(), (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
        }
    }
    localContainer.stopAsync().awaitTerminated();
}

Example 17 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributes.

/**
 * Tests the ability to set attributes (via append() or updateAttributes()), then fetch them back using getAttributes(),
 * emphasizing on Extended Attributes that are dumped into Storage and cleared from memory.
 */
@Test
public void testAttributes() throws Exception {
    final AttributeId coreAttribute = Attributes.EVENT_COUNT;
    final int variableAttributeIdLength = 4;
    final List<AttributeId> extendedAttributesUUID = Arrays.asList(AttributeId.randomUUID(), AttributeId.randomUUID());
    final List<AttributeId> extendedAttributesVariable = Arrays.asList(AttributeId.random(variableAttributeIdLength), AttributeId.random(variableAttributeIdLength));
    final List<AttributeId> allAttributesWithUUID = Stream.concat(extendedAttributesUUID.stream(), Stream.of(coreAttribute)).collect(Collectors.toList());
    final List<AttributeId> allAttributesWithVariable = Stream.concat(extendedAttributesVariable.stream(), Stream.of(coreAttribute)).collect(Collectors.toList());
    final AttributeId segmentLengthAttributeUUID = AttributeId.randomUUID();
    final AttributeId segmentLengthAttributeVariable = AttributeId.random(variableAttributeIdLength);
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT));
    containerConfig.setMaxCachedExtendedAttributeCount(SEGMENT_COUNT * allAttributesWithUUID.size());
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    val segmentNames = IntStream.range(0, SEGMENT_COUNT).boxed().collect(Collectors.toMap(StreamSegmentContainerTests::getSegmentName, i -> i % 2 == 0 ? variableAttributeIdLength : 0));
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (val sn : segmentNames.entrySet()) {
        opFutures.add(localContainer.createStreamSegment(sn.getKey(), SegmentType.STREAM_SEGMENT, AttributeUpdateCollection.from(new AttributeUpdate(Attributes.ATTRIBUTE_ID_LENGTH, AttributeUpdateType.None, sn.getValue())), TIMEOUT));
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Predicate<Map.Entry<String, Integer>> isUUIDOnly = e -> e.getValue() == 0;
    // 2. Add some appends.
    for (val sn : segmentNames.entrySet()) {
        boolean isUUID = isUUIDOnly.test(sn);
        for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
            AttributeUpdateCollection attributeUpdates = (isUUID ? allAttributesWithUUID : allAttributesWithVariable).stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Accumulate, 1)).collect(Collectors.toCollection(AttributeUpdateCollection::new));
            opFutures.add(Futures.toVoid(localContainer.append(sn.getKey(), getAppendData(sn.getKey(), i), attributeUpdates, TIMEOUT)));
        }
    }
    // 2.1 Update some of the attributes.
    for (val sn : segmentNames.entrySet()) {
        boolean isUUID = isUUIDOnly.test(sn);
        for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
            AttributeUpdateCollection attributeUpdates = (isUUID ? allAttributesWithUUID : allAttributesWithVariable).stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Accumulate, 1)).collect(Collectors.toCollection(AttributeUpdateCollection::new));
            opFutures.add(localContainer.updateAttributes(sn.getKey(), attributeUpdates, TIMEOUT));
        }
        // Verify that we are not allowed to update attributes of the wrong type.
        val badUpdate = new AttributeUpdate(isUUID ? AttributeId.random(variableAttributeIdLength) : AttributeId.randomUUID(), AttributeUpdateType.Accumulate, 1);
        AssertExtensions.assertSuppliedFutureThrows("updateAttributes allowed updating attributes with wrong type and/or length.", () -> localContainer.updateAttributes(sn.getKey(), AttributeUpdateCollection.from(badUpdate), TIMEOUT), ex -> ex instanceof AttributeIdLengthMismatchException);
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 2.2 Dynamic attributes.
    for (val sn : segmentNames.entrySet()) {
        boolean isUUID = isUUIDOnly.test(sn);
        val dynamicId = isUUID ? segmentLengthAttributeUUID : segmentLengthAttributeVariable;
        val dynamicAttributes = AttributeUpdateCollection.from(new DynamicAttributeUpdate(dynamicId, AttributeUpdateType.Replace, DynamicAttributeValue.segmentLength(10)));
        val appendData = getAppendData(sn.getKey(), 1000);
        val lastOffset = localContainer.append(sn.getKey(), appendData, dynamicAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        val expectedValue = lastOffset - appendData.getLength() + 10;
        Assert.assertEquals(expectedValue, (long) localContainer.getAttributes(sn.getKey(), Collections.singleton(dynamicId), false, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).get(dynamicId));
    }
    // 3. getSegmentInfo
    for (val sn : segmentNames.entrySet()) {
        val segmentName = sn.getKey();
        val allAttributes = isUUIDOnly.test(sn) ? allAttributesWithUUID : allAttributesWithVariable;
        val allAttributeValues = localContainer.getAttributes(segmentName, allAttributes, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected number of attributes retrieved via getAttributes().", allAttributes.size(), allAttributeValues.size());
        // Verify all attribute values.
        SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        for (val attributeId : allAttributes) {
            Assert.assertEquals("Unexpected value for attribute " + attributeId + " via getInfo() for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            Assert.assertEquals("Unexpected value for attribute " + attributeId + " via getAttributes() for segment " + segmentName, expectedAttributeValue, (long) allAttributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
        }
        // Verify we can't request wrong lengths/types.
        val badId = isUUIDOnly.test(sn) ? AttributeId.random(variableAttributeIdLength) : AttributeId.randomUUID();
        AssertExtensions.assertSuppliedFutureThrows("getAttributes allowed getting attributes with wrong type and/or length.", () -> localContainer.getAttributes(segmentName, Collections.singleton(badId), true, TIMEOUT), ex -> ex instanceof IllegalArgumentException);
    }
    // Force these segments out of memory, so that we may verify that extended attributes are still recoverable.
    localContainer.triggerMetadataCleanup(segmentNames.keySet()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    for (val sn : segmentNames.entrySet()) {
        val segmentName = sn.getKey();
        val allAttributes = isUUIDOnly.test(sn) ? allAttributesWithUUID : allAttributesWithVariable;
        val allAttributeValues = localContainer.getAttributes(segmentName, allAttributes, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected number of attributes retrieved via getAttributes() after recovery for segment " + segmentName, allAttributes.size(), allAttributeValues.size());
        // Verify all attribute values. Core attributes should still be loaded in memory, while extended attributes can
        // only be fetched via their special API.
        SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        for (val attributeId : allAttributes) {
            Assert.assertEquals("Unexpected value for attribute " + attributeId + " via getAttributes() after recovery for segment " + segmentName, expectedAttributeValue, (long) allAttributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            if (Attributes.isCoreAttribute(attributeId)) {
                Assert.assertEquals("Expecting core attribute to be loaded in memory.", expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            } else {
                Assert.assertEquals("Not expecting extended attribute to be loaded in memory.", Attributes.NULL_ATTRIBUTE_VALUE, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            }
        }
        // Now instruct the Container to cache missing values (do it a few times so we make sure it's idempotent).
        // Also introduce some random new attribute to fetch. We want to make sure we can properly handle caching
        // missing attribute values.
        val missingAttributeId = isUUIDOnly.test(sn) ? AttributeId.randomUUID() : AttributeId.random(variableAttributeIdLength);
        val attributesToCache = new ArrayList<>(allAttributes);
        attributesToCache.add(missingAttributeId);
        val attributesToCacheValues = new HashMap<>(allAttributeValues);
        attributesToCacheValues.put(missingAttributeId, Attributes.NULL_ATTRIBUTE_VALUE);
        Map<AttributeId, Long> allAttributeValuesWithCache;
        for (int i = 0; i < 2; i++) {
            allAttributeValuesWithCache = localContainer.getAttributes(segmentName, attributesToCache, true, TIMEOUT).join();
            AssertExtensions.assertMapEquals("Inconsistent results from getAttributes(cache=true, attempt=" + i + ").", attributesToCacheValues, allAttributeValuesWithCache);
            sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
            for (val attributeId : allAttributes) {
                Assert.assertEquals("Expecting all attributes to be loaded in memory.", expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            }
            Assert.assertEquals("Unexpected value for missing Attribute Id", Attributes.NULL_ATTRIBUTE_VALUE, (long) sp.getAttributes().get(missingAttributeId));
        }
    }
    // 4. Make an update, then immediately seal the segment, then verify the update updated the root pointer.
    AttributeId attr = Attributes.ATTRIBUTE_SEGMENT_ROOT_POINTER;
    val oldRootPointers = new HashMap<String, Long>();
    for (val sn : segmentNames.entrySet()) {
        val segmentName = sn.getKey();
        val newAttributeId = isUUIDOnly.test(sn) ? AttributeId.randomUUID() : AttributeId.random(variableAttributeIdLength);
        // Get the old root pointer, then make a random attribute update, then immediately seal the segment.
        localContainer.getAttributes(segmentName, Collections.singleton(attr), false, TIMEOUT).thenCompose(values -> {
            oldRootPointers.put(segmentName, values.get(attr));
            return CompletableFuture.allOf(localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(newAttributeId, AttributeUpdateType.Replace, 1L)), TIMEOUT), localContainer.sealStreamSegment(segmentName, TIMEOUT));
        }).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    }
    // which indicates the StorageWriter was able to successfully record it after its final Attribute Index update.
    for (String segmentName : segmentNames.keySet()) {
        Long oldValue = oldRootPointers.get(segmentName);
        TestUtils.await(() -> {
            val newVal = localContainer.getAttributes(segmentName, Collections.singleton(attr), false, TIMEOUT).join().get(attr);
            return oldValue < newVal;
        }, 10, TIMEOUT.toMillis());
    }
    waitForSegmentsInStorage(segmentNames.keySet(), localContainer, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    localContainer.stopAsync().awaitTerminated();
}

Example 18 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributeCleanup.

/**
 * Tests the ability to clean up Extended Attributes from Segment Metadatas that have not been used recently.
 * This test does the following:
 * 1. Sets up a custom SegmentContainer with a hook into the metadataCleanup task
 * 2. Creates a segment and appends something to it, each time updating attributes (and verifies they were updated correctly).
 * 3. Waits for the segment's attributes to be forgotten.
 * 4. Verifies that the forgotten attributes can be fetched from the Attribute Index and re-cached in memory.
 */
@Test
public void testAttributeCleanup() throws Exception {
    final String segmentName = "segment";
    final AttributeId[] attributes = new AttributeId[] { Attributes.EVENT_COUNT, AttributeId.uuid(0, 1), AttributeId.uuid(0, 2), AttributeId.uuid(0, 3) };
    Map<AttributeId, Long> allAttributes = new HashMap<>();
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(250));
    containerConfig.setMaxCachedExtendedAttributeCount(1);
    @Cleanup TestContext context = createContext(containerConfig);
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // Create segment with initial attributes and verify they were set correctly.
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Add one append with some attribute changes and verify they were set correctly.
    val appendAttributes = createAttributeUpdates(attributes);
    applyAttributes(appendAttributes, allAttributes);
    for (val au : appendAttributes) {
        localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(au), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    }
    SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after initial updateAttributes() call.", allAttributes, sp, AUTO_ATTRIBUTES);
    // Wait until the attributes are forgotten
    localContainer.triggerAttributeCleanup(segmentName, containerConfig.getMaxCachedExtendedAttributeCount()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Now get attributes again and verify them.
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // During attribute eviction, we expect all core attributes to be preserved, and only 1 extended attribute (as
    // defined in the config) to be preserved. This extended attribute should be the last one we updated.
    val expectedAttributes = new HashMap<>(Attributes.getCoreNonNullAttributes(allAttributes));
    val lastExtAttribute = appendAttributes.stream().filter(au -> !Attributes.isCoreAttribute(au.getAttributeId())).reduce((a, b) -> b).get();
    expectedAttributes.put(lastExtAttribute.getAttributeId(), lastExtAttribute.getValue());
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    val fetchedAttributes = localContainer.getAttributes(segmentName, allAttributes.keySet(), true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    AssertExtensions.assertMapEquals("Unexpected attributes after eviction & reload.", allAttributes, fetchedAttributes);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction & reload+getInfo.", allAttributes, sp, AUTO_ATTRIBUTES);
}

Example 19 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class AttributeIndexTests method testTruncatedRootPointer.

/**
 * Tests the ability of the Attribute Index to recover correctly after an update has been successfully written to Storage,
 * but the previous value of the {@link Attributes#ATTRIBUTE_SEGMENT_ROOT_POINTER} has been truncated out without
 * having the new value persisted (most likely due to a system crash).
 * In this case, the {@link Attributes#ATTRIBUTE_SEGMENT_ROOT_POINTER} value should be ignored and the index should
 * be attempted to be read from Storage without providing hints to where to start reading from.
 */
@Test
public void testTruncatedRootPointer() throws Exception {
    val attributeSegmentName = NameUtils.getAttributeSegmentName(SEGMENT_NAME);
    val config = AttributeIndexConfig.builder().with(AttributeIndexConfig.MAX_INDEX_PAGE_SIZE, 1024).with(AttributeIndexConfig.ATTRIBUTE_SEGMENT_ROLLING_SIZE, 8).build();
    final int attributeCount = 20;
    val attributes = IntStream.range(0, attributeCount).mapToObj(i -> AttributeId.uuid(i, i)).collect(Collectors.toList());
    @Cleanup val context = new TestContext(config);
    populateSegments(context);
    // 1. Populate and verify first index.
    val expectedValues = new HashMap<AttributeId, Long>();
    long nextValue = 0;
    long previousRootPointer = -1;
    boolean invalidRootPointer = false;
    for (AttributeId attributeId : attributes) {
        val idx = context.index.forSegment(SEGMENT_ID, TIMEOUT).join();
        val value = nextValue++;
        expectedValues.put(attributeId, value);
        val updateBatch = Collections.singletonMap(attributeId, value);
        // Inject a callback to know when we finished truncating, as it is an async task.
        @Cleanup("release") val truncated = new ReusableLatch();
        context.storage.truncateCallback = (s, o) -> truncated.release();
        val rootPointer = idx.update(updateBatch, TIMEOUT).join();
        truncated.await(TIMEOUT.toMillis());
        val startOffset = context.storage.getStreamSegmentInfo(attributeSegmentName, TIMEOUT).join().getStartOffset();
        if (previousRootPointer >= 0) {
            // We always set the Root Pointer to be one "value behind". Since we truncate the Attribute Segment with
            // every update (small rolling size), doing this ensures that its value should always be less than the
            // segment's start offset. This is further validated by asserting that invalidRootPointer is true at the
            // end of this loop.
            context.containerMetadata.getStreamSegmentMetadata(SEGMENT_ID).updateAttributes(Collections.singletonMap(Attributes.ATTRIBUTE_SEGMENT_ROOT_POINTER, previousRootPointer));
            invalidRootPointer |= previousRootPointer < startOffset;
        }
        previousRootPointer = rootPointer;
        // Clean up the index cache and force a reload. Verify that we can read from the index.
        context.index.cleanup(null);
        val storageRead = new AtomicBoolean();
        context.storage.readInterceptor = (name, offset, length, storage) -> CompletableFuture.runAsync(() -> storageRead.set(true));
        val idx2 = context.index.forSegment(SEGMENT_ID, TIMEOUT).join();
        checkIndex(idx2, expectedValues);
        Assert.assertTrue("Expecting storage reads after reload.", storageRead.get());
    }
    Assert.assertTrue("No invalid Root Pointers generated during the test.", invalidRootPointer);
    // 3. Remove all values (and thus force an update - validates conditional updates still work in this case).
    context.index.cleanup(null);
    val idx2 = context.index.forSegment(SEGMENT_ID, TIMEOUT).join();
    idx2.update(toDelete(expectedValues.keySet()), TIMEOUT).join();
    expectedValues.replaceAll((key, v) -> Attributes.NULL_ATTRIBUTE_VALUE);
    checkIndex(idx2, expectedValues);
}

Example 20 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class AttributeIndexTests method testRegularOperations.

private void testRegularOperations(int attributeCount, int batchSize, int repeats, AttributeIndexConfig config) {
    val attributes = IntStream.range(0, attributeCount).mapToObj(i -> AttributeId.uuid(i, i)).collect(Collectors.toList());
    @Cleanup val context = new TestContext(config);
    populateSegments(context);
    // 1. Populate and verify first index.
    val idx = context.index.forSegment(SEGMENT_ID, TIMEOUT).join();
    val expectedValues = new HashMap<AttributeId, Long>();
    // Record every time we read from Storage.
    AtomicBoolean storageRead = new AtomicBoolean(false);
    context.storage.readInterceptor = (name, offset, length, storage) -> CompletableFuture.runAsync(() -> storageRead.set(true));
    // Populate data.
    val updateBatch = new HashMap<AttributeId, Long>();
    AtomicLong nextValue = new AtomicLong(0);
    for (int r = 0; r < repeats; r++) {
        for (AttributeId attributeId : attributes) {
            long value = nextValue.getAndIncrement();
            expectedValues.put(attributeId, value);
            updateBatch.put(attributeId, value);
            if (updateBatch.size() % batchSize == 0) {
                idx.update(updateBatch, TIMEOUT).join();
                updateBatch.clear();
            }
        }
    }
    // Commit any leftovers.
    if (updateBatch.size() > 0) {
        idx.update(updateBatch, TIMEOUT).join();
    }
    storageRead.set(false);
    checkIndex(idx, expectedValues);
    Assert.assertFalse("Not expecting any storage reads.", storageRead.get());
    // 2. Reload index and verify it still has the correct values. This also forces a cache cleanup so we read data
    // directly from Storage.
    context.index.cleanup(null);
    val idx2 = context.index.forSegment(SEGMENT_ID, TIMEOUT).join();
    storageRead.set(false);
    checkIndex(idx2, expectedValues);
    Assert.assertTrue("Expecting storage reads after reload.", storageRead.get());
    // 3. Remove all values.
    idx2.update(toDelete(expectedValues.keySet()), TIMEOUT).join();
    expectedValues.replaceAll((key, v) -> Attributes.NULL_ATTRIBUTE_VALUE);
    checkIndex(idx2, expectedValues);
}