Examples with AttributeId io.pravega.segmentstore.contracts.AttributeId used on opensource projects

Example 41 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentContainerTests method testSegmentRegularOperations.

/**
 * Tests the createSegment, append, updateAttributes, read, getSegmentInfo, getActiveSegments.
 */
@Test
public void testSegmentRegularOperations() throws Exception {
    final AttributeId attributeAccumulate = AttributeId.randomUUID();
    final AttributeId attributeReplace = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfGreater = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfEquals = AttributeId.randomUUID();
    final AttributeId attributeNoUpdate = AttributeId.randomUUID();
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    checkActiveSegments(context.container, 0);
    activateAllSegments(segmentNames, context);
    checkActiveSegments(context.container, segmentNames.size());
    // 2. Add some appends.
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    ArrayList<RefCountByteArraySegment> appends = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
        for (String segmentName : segmentNames) {
            val attributeUpdates = new AttributeUpdateCollection();
            attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, i == 0 ? AttributeUpdateType.Replace : AttributeUpdateType.ReplaceIfEquals, i + 1, i));
            RefCountByteArraySegment appendData = getAppendData(segmentName, i);
            long expectedLength = lengths.getOrDefault(segmentName, 0L) + appendData.getLength();
            val append = (i % 2 == 0) ? context.container.append(segmentName, appendData, attributeUpdates, TIMEOUT) : context.container.append(segmentName, lengths.get(segmentName), appendData, attributeUpdates, TIMEOUT);
            opFutures.add(append.thenApply(length -> {
                assertEquals(expectedLength, length.longValue());
                return null;
            }));
            lengths.put(segmentName, expectedLength);
            recordAppend(segmentName, appendData, segmentContents, appends);
        }
    }
    // 2.1 Update some of the attributes.
    for (String segmentName : segmentNames) {
        // Record a one-off update.
        opFutures.add(context.container.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, expectedAttributeValue)), TIMEOUT));
        for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
            val attributeUpdates = new AttributeUpdateCollection();
            attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, APPENDS_PER_SEGMENT + i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, APPENDS_PER_SEGMENT + i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, APPENDS_PER_SEGMENT + i + 1, APPENDS_PER_SEGMENT + i));
            opFutures.add(context.container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. getSegmentInfo
    for (String segmentName : segmentNames) {
        SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        long expectedLength = lengths.get(segmentName);
        Assert.assertEquals("Unexpected StartOffset for non-truncated segment " + segmentName, 0, sp.getStartOffset());
        Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength, sp.getLength());
        Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
        Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted());
        // Verify all attribute values.
        Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeNoUpdate, Attributes.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeAccumulate, Attributes.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplace + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplace, Attributes.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplaceIfGreater + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplaceIfGreater, Attributes.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplaceIfEquals + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplaceIfEquals, Attributes.NULL_ATTRIBUTE_VALUE));
        val expectedType = getSegmentType(segmentName);
        val actualType = SegmentType.fromAttributes(sp.getAttributes());
        Assert.assertEquals("Unexpected Segment Type.", expectedType, actualType);
    }
    checkActiveSegments(context.container, segmentNames.size());
    // 4. Reads (regular reads, not tail reads).
    checkReadIndex(segmentContents, lengths, context);
    // 4.1. After we ensured that all data has been ingested and processed, verify that all data buffers have been released.
    checkAppendLeaks(appends);
    // 5. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}

Example 42 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentContainerTests method testMetadataCleanup.

/**
 * Tests the ability to clean up SegmentMetadata for those segments which have not been used recently.
 * This test does the following:
 * 1. Sets up a custom SegmentContainer with a hook into the metadataCleanup task
 * 2. Creates a segment and appends something to it, each time updating attributes (and verifies they were updated correctly).
 * 3. Waits for the segment to be forgotten (evicted).
 * 4. Requests info on the segment, validates it, then makes another append, seals it, at each step verifying it was done
 * correctly (checking Metadata, Attributes and Storage).
 * 5. Deletes the segment, waits for metadata to be cleared (via forcing another log truncation), re-creates the
 * same segment and validates that the old attributes did not "bleed in".
 */
@Test
public void testMetadataCleanup() throws Exception {
    final String segmentName = "segment";
    final AttributeId[] attributes = new AttributeId[] { Attributes.CREATION_TIME, Attributes.EVENT_COUNT, AttributeId.randomUUID() };
    final ByteArraySegment appendData = new ByteArraySegment("hello".getBytes());
    Map<AttributeId, Long> expectedAttributes = new HashMap<>();
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT));
    @Cleanup TestContext context = createContext(containerConfig);
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // Create segment with initial attributes and verify they were set correctly.
    val initialAttributes = createAttributeUpdates(attributes);
    applyAttributes(initialAttributes, expectedAttributes);
    // We expect extended attributes to be dropped in this case.
    expectedAttributes = Attributes.getCoreNonNullAttributes(expectedAttributes);
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), initialAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after segment creation.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    // Add one append with some attribute changes and verify they were set correctly.
    val appendAttributes = createAttributeUpdates(attributes);
    applyAttributes(appendAttributes, expectedAttributes);
    localContainer.append(segmentName, appendData, appendAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after append.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    // Wait until the segment is forgotten.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Now get attributes again and verify them.
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // We expect extended attributes to be dropped in this case.
    expectedAttributes = Attributes.getCoreNonNullAttributes(expectedAttributes);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction & resurrection.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    // Append again, and make sure we can append at the right offset.
    val secondAppendAttributes = createAttributeUpdates(attributes);
    applyAttributes(secondAppendAttributes, expectedAttributes);
    localContainer.append(segmentName, appendData.getLength(), appendData, secondAppendAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertEquals("Unexpected length from segment after eviction & resurrection.", 2 * appendData.getLength(), sp.getLength());
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction & resurrection.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    // Seal.
    localContainer.sealStreamSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after seal.", expectedAttributes, sp, AUTO_ATTRIBUTES);
    // Verify the segment actually made to Storage in one piece.
    waitForSegmentInStorage(sp, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val storageInfo = context.storage.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Assert.assertEquals("Unexpected length in storage for segment.", sp.getLength(), storageInfo.getLength());
    // Delete segment and wait until it is forgotten again (we need to create another dummy segment so that we can
    // force a Metadata Truncation in order to facilitate that; this is the purpose of segment2).
    localContainer.deleteStreamSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Wait for the segment to be forgotten again.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Now Create the Segment again and verify the old attributes were not "remembered".
    val newAttributes = createAttributeUpdates(attributes);
    applyAttributes(newAttributes, expectedAttributes);
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), newAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // We expect extended attributes to be dropped in this case.
    expectedAttributes = Attributes.getCoreNonNullAttributes(expectedAttributes);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after deletion and re-creation.", expectedAttributes, sp, AUTO_ATTRIBUTES);
}

Example 43 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class AttributeIndexTests method testCacheEviction.

/**
 * Tests the ability to process Cache Eviction signals and re-caching evicted values.
 */
@Test
public void testCacheEviction() {
    int attributeCount = 1000;
    val attributes = IntStream.range(0, attributeCount).mapToObj(i -> AttributeId.uuid(i, i)).collect(Collectors.toList());
    val config = AttributeIndexConfig.builder().with(AttributeIndexConfig.MAX_INDEX_PAGE_SIZE, 1024).build();
    @Cleanup val context = new TestContext(config);
    populateSegments(context);
    // 1. Populate and verify first index.
    @Cleanup val idx = (SegmentAttributeBTreeIndex) context.index.forSegment(SEGMENT_ID, TIMEOUT).join();
    val expectedValues = new HashMap<AttributeId, Long>();
    // Populate data.
    AtomicLong nextValue = new AtomicLong(0);
    for (AttributeId attributeId : attributes) {
        long value = nextValue.getAndIncrement();
        expectedValues.put(attributeId, value);
    }
    idx.update(expectedValues, TIMEOUT).join();
    // Everything should already be cached, so four our first check we don't expect any Storage reads.
    context.storage.readInterceptor = (String streamSegmentName, long offset, int length, SyncStorage wrappedStorage) -> Futures.failedFuture(new AssertionError("Not expecting storage reads yet."));
    checkIndex(idx, expectedValues);
    val cacheStatus = idx.getCacheStatus();
    Assert.assertEquals("Not expecting different generations yet.", cacheStatus.getOldestGeneration(), cacheStatus.getNewestGeneration());
    val newGen = cacheStatus.getNewestGeneration() + 1;
    boolean anythingRemoved = idx.updateGenerations(newGen, newGen, false);
    Assert.assertTrue("Expecting something to be evicted.", anythingRemoved);
    // Re-check the index and verify at least one Storage Read happened.
    AtomicBoolean intercepted = new AtomicBoolean(false);
    context.storage.readInterceptor = (String streamSegmentName, long offset, int length, SyncStorage wrappedStorage) -> {
        intercepted.set(true);
        return CompletableFuture.completedFuture(null);
    };
    checkIndex(idx, expectedValues);
    Assert.assertTrue("Expected at least one Storage read.", intercepted.get());
    // Now everything should be cached again.
    intercepted.set(false);
    checkIndex(idx, expectedValues);
    Assert.assertFalse("Not expecting any Storage read.", intercepted.get());
}

Example 44 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class AttributeIndexTests method testSeal.

/**
 * Tests the ability to Seal an Attribute Segment (create a final snapshot and disallow new changes).
 */
@Test
public void testSeal() {
    int attributeCount = 1000;
    val attributes = IntStream.range(0, attributeCount).mapToObj(i -> AttributeId.uuid(i, i)).collect(Collectors.toList());
    val config = AttributeIndexConfig.builder().with(AttributeIndexConfig.MAX_INDEX_PAGE_SIZE, DEFAULT_CONFIG.getMaxIndexPageSize()).with(AttributeIndexConfig.ATTRIBUTE_SEGMENT_ROLLING_SIZE, 10).build();
    @Cleanup val context = new TestContext(config);
    populateSegments(context);
    // 1. Populate and verify first index.
    val idx = context.index.forSegment(SEGMENT_ID, TIMEOUT).join();
    val expectedValues = new HashMap<AttributeId, Long>();
    // Populate data.
    AtomicLong nextValue = new AtomicLong(0);
    for (AttributeId attributeId : attributes) {
        long value = nextValue.getAndIncrement();
        expectedValues.put(attributeId, value);
    }
    idx.update(expectedValues, TIMEOUT).join();
    // Check index before sealing.
    checkIndex(idx, expectedValues);
    // Seal twice (to check idempotence).
    idx.seal(TIMEOUT).join();
    idx.seal(TIMEOUT).join();
    AssertExtensions.assertSuppliedFutureThrows("Index allowed adding new values after being sealed.", () -> idx.update(Collections.singletonMap(AttributeId.randomUUID(), 1L), TIMEOUT), ex -> ex instanceof StreamSegmentSealedException);
    // Check index again, after sealing.
    checkIndex(idx, expectedValues);
}

Example 45 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentContainerTests method testConditionalTransactionOperations.

/**
 * Test the createTransaction, append-to-Transaction, mergeTransaction methods with attribute updates.
 */
@Test
public void testConditionalTransactionOperations() throws Exception {
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    HashMap<String, ArrayList<String>> transactionsBySegment = createTransactions(segmentNames, context);
    activateAllSegments(segmentNames, context);
    transactionsBySegment.values().forEach(s -> activateAllSegments(s, context));
    // 2. Add some appends.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    appendToParentsAndTransactions(segmentNames, transactionsBySegment, lengths, segmentContents, context);
    // 3. Correctly update attribute on parent Segments. Each source Segment will be initialized with a value and
    // after the merge, that value should have been updated.
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (Map.Entry<String, ArrayList<String>> e : transactionsBySegment.entrySet()) {
        String parentName = e.getKey();
        for (String transactionName : e.getValue()) {
            opFutures.add(context.container.updateAttributes(parentName, AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.fromUUID(UUID.nameUUIDFromBytes(transactionName.getBytes())), AttributeUpdateType.None, transactionName.hashCode())), TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Merge all the Transactions. Now this should work.
    Futures.allOf(mergeTransactions(transactionsBySegment, lengths, segmentContents, context, true)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 5. Add more appends (to the parent segments)
    ArrayList<CompletableFuture<Long>> appendFutures = new ArrayList<>();
    HashMap<String, CompletableFuture<Map<AttributeId, Long>>> getAttributeFutures = new HashMap<>();
    for (int i = 0; i < 5; i++) {
        for (String segmentName : segmentNames) {
            RefCountByteArraySegment appendData = getAppendData(segmentName, APPENDS_PER_SEGMENT + i);
            appendFutures.add(context.container.append(segmentName, appendData, null, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.getLength());
            recordAppend(segmentName, appendData, segmentContents, null);
            // Verify that we can no longer append to Transaction.
            for (String transactionName : transactionsBySegment.get(segmentName)) {
                AssertExtensions.assertThrows("An append was allowed to a merged Transaction " + transactionName, context.container.append(transactionName, new ByteArraySegment("foo".getBytes()), null, TIMEOUT)::join, ex -> ex instanceof StreamSegmentMergedException || ex instanceof StreamSegmentNotExistsException);
                getAttributeFutures.put(transactionName, context.container.getAttributes(segmentName, Collections.singletonList(AttributeId.fromUUID(UUID.nameUUIDFromBytes(transactionName.getBytes()))), true, TIMEOUT));
            }
        }
    }
    Futures.allOf(appendFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Futures.allOf(getAttributeFutures.values()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 6. Verify their contents.
    checkReadIndex(segmentContents, lengths, context);
    // 7. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    // 8. Verify that the parent Segment contains the expected attributes updated.
    for (Map.Entry<String, CompletableFuture<Map<AttributeId, Long>>> transactionAndAttribute : getAttributeFutures.entrySet()) {
        Map<AttributeId, Long> segmentAttributeUpdated = transactionAndAttribute.getValue().join();
        AttributeId transactionAttributeId = AttributeId.fromUUID(UUID.nameUUIDFromBytes(transactionAndAttribute.getKey().getBytes()));
        // Conditional merges in mergeTransactions() update the attribute value to adding 1.
        Assert.assertEquals(transactionAndAttribute.getKey().hashCode() + 1, segmentAttributeUpdated.get(transactionAttributeId).longValue());
    }
    context.container.stopAsync().awaitTerminated();
}