Examples with AttributeId io.pravega.segmentstore.contracts.AttributeId used on opensource projects

Example 26 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method createMetadataTransient.

private UpdateableContainerMetadata createMetadataTransient() {
    UpdateableContainerMetadata metadata = createBlankMetadata();
    UpdateableSegmentMetadata segmentMetadata = metadata.mapStreamSegmentId(TRANSIENT_SEGMENT_NAME, TRANSIENT_SEGMENT_ID);
    segmentMetadata.setLength(SEGMENT_LENGTH);
    segmentMetadata.setStorageLength(SEGMENT_LENGTH - 1);
    Map<AttributeId, Long> attributes = new HashMap<>();
    attributes.put(Attributes.ATTRIBUTE_SEGMENT_TYPE, SegmentType.TRANSIENT_SEGMENT.getValue());
    for (long i = 0; i < SegmentMetadataUpdateTransaction.TRANSIENT_ATTRIBUTE_LIMIT - (TRANSIENT_ATTRIBUTE_REMAINING + 1); i++) {
        attributes.put(AttributeId.uuid(Attributes.CORE_ATTRIBUTE_ID_PREFIX + 1, i), i);
    }
    segmentMetadata.updateAttributes(attributes);
    segmentMetadata.refreshDerivedProperties();
    return metadata;
}

Example 27 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testUpdateAttributesSealedSegment.

/**
 * Tests the ability of the ContainerMetadataUpdateTransaction to handle UpdateAttributeOperations after the segment
 * has been sealed.
 */
@Test
public void testUpdateAttributesSealedSegment() throws Exception {
    final AttributeId coreAttribute = Attributes.ATTRIBUTE_SEGMENT_ROOT_POINTER;
    final AttributeId extAttribute = AttributeId.randomUUID();
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    // Seal the segment.
    val sealOp = createSeal();
    txn.preProcessOperation(sealOp);
    txn.acceptOperation(sealOp);
    // 1. Core attribute, but not internal.
    val coreUpdate = new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(new AttributeUpdate(coreAttribute, AttributeUpdateType.Replace, 1L)));
    AssertExtensions.assertThrows("Non-internal update of core attribute succeeded.", () -> txn.preProcessOperation(coreUpdate), ex -> ex instanceof StreamSegmentSealedException);
    // 2. Core attribute, internal update.
    coreUpdate.setInternal(true);
    txn.preProcessOperation(coreUpdate);
    txn.acceptOperation(coreUpdate);
    Assert.assertEquals("Core attribute was not updated.", 1L, (long) txn.getStreamSegmentMetadata(SEGMENT_ID).getAttributes().get(coreAttribute));
    // 3. Extended attributes.
    val extUpdate1 = new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(new AttributeUpdate(extAttribute, AttributeUpdateType.Replace, 1L)));
    extUpdate1.setInternal(true);
    AssertExtensions.assertThrows("Extended attribute update succeeded.", () -> txn.preProcessOperation(extUpdate1), ex -> ex instanceof StreamSegmentSealedException);
    val extUpdate2 = new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(new AttributeUpdate(coreAttribute, AttributeUpdateType.Replace, 2L), new AttributeUpdate(extAttribute, AttributeUpdateType.Replace, 3L)));
    extUpdate1.setInternal(true);
    AssertExtensions.assertThrows("Mixed attribute update succeeded.", () -> txn.preProcessOperation(extUpdate2), ex -> ex instanceof StreamSegmentSealedException);
}

Example 28 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributeIterators.

/**
 * Tests the ability to run attribute iterators over all or a subset of attributes in a segment.
 */
@Test
public void testAttributeIterators() throws Exception {
    final List<AttributeId> sortedAttributes = IntStream.range(0, 100).mapToObj(i -> AttributeId.uuid(i, i)).sorted().collect(Collectors.toList());
    final Map<AttributeId, Long> expectedValues = sortedAttributes.stream().collect(Collectors.toMap(id -> id, id -> id.getBitGroup(0)));
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT));
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the Segment.
    String segmentName = getSegmentName(0);
    localContainer.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val segment1 = localContainer.forSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 2. Set some initial attribute values and verify in-memory iterator.
    AttributeUpdateCollection attributeUpdates = sortedAttributes.stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Replace, expectedValues.get(attributeId))).collect(Collectors.toCollection(AttributeUpdateCollection::new));
    segment1.updateAttributes(attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment1, sortedAttributes, expectedValues);
    // 3. Force these segments out of memory and verify out-of-memory iterator.
    localContainer.triggerMetadataCleanup(Collections.singleton(segmentName)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val segment2 = localContainer.forSegment(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment2, sortedAttributes, expectedValues);
    // 4. Update some values, and verify mixed iterator.
    attributeUpdates.clear();
    for (int i = 0; i < sortedAttributes.size(); i += 3) {
        AttributeId attributeId = sortedAttributes.get(i);
        expectedValues.put(attributeId, expectedValues.get(attributeId) + 1);
        attributeUpdates.add(new AttributeUpdate(attributeId, AttributeUpdateType.Replace, expectedValues.get(attributeId)));
    }
    segment2.updateAttributes(attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkAttributeIterators(segment2, sortedAttributes, expectedValues);
    localContainer.stopAsync().awaitTerminated();
}

Example 29 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentMetadataTests method testAttributes.

/**
 * Tests that Attributes are properly recorded and updated
 */
@Test
public void testAttributes() {
    StreamSegmentMetadata metadata = new StreamSegmentMetadata(SEGMENT_NAME, SEGMENT_ID, CONTAINER_ID);
    // Step 1: initial set of attributes.
    Random rnd = new Random(0);
    val expectedAttributes = generateAttributes(rnd);
    metadata.updateAttributes(expectedAttributes);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after initial set.", expectedAttributes, metadata);
    // Step 2: Update half of attributes and add 50% more.
    int count = 0;
    val keyIterator = expectedAttributes.keySet().iterator();
    val attributeUpdates = new HashMap<AttributeId, Long>();
    // Update
    while (count < ATTRIBUTE_COUNT / 2 && keyIterator.hasNext()) {
        attributeUpdates.put(keyIterator.next(), rnd.nextLong());
        count++;
    }
    // Now add a few more.
    while (attributeUpdates.size() < ATTRIBUTE_COUNT) {
        attributeUpdates.put(AttributeId.randomUUID(), rnd.nextLong());
    }
    attributeUpdates.forEach(expectedAttributes::put);
    metadata.updateAttributes(attributeUpdates);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after update.", expectedAttributes, metadata);
    // Check getAttributes(filter).
    BiPredicate<AttributeId, Long> filter = (key, value) -> key.getBitGroup(1) % 2 == 0;
    val expectedFilteredAttributes = expectedAttributes.entrySet().stream().filter(e -> filter.test(e.getKey(), e.getValue())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    val actualFilteredAttributes = metadata.getAttributes(filter);
    AssertExtensions.assertMapEquals("Unexpected result from getAttributes(Filter).", expectedFilteredAttributes, actualFilteredAttributes);
    // Step 3: Remove all attributes (Note that attributes are not actually removed; they're set to the NULL_ATTRIBUTE_VALUE).
    expectedAttributes.entrySet().forEach(e -> e.setValue(Attributes.NULL_ATTRIBUTE_VALUE));
    metadata.updateAttributes(expectedAttributes);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after removal.", expectedAttributes, metadata);
}

Example 30 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StreamSegmentMetadataTests method testCleanupAttributes.

/**
 * Tests the ability to cleanup Extended Attributes.
 */
@Test
public void testCleanupAttributes() {
    final AttributeId coreAttributeId = Attributes.EVENT_COUNT;
    final int attributeCount = 10000;
    final int maxAttributeCount = attributeCount / 10;
    // Initial population.
    StreamSegmentMetadata metadata = new StreamSegmentMetadata(SEGMENT_NAME, SEGMENT_ID, CONTAINER_ID);
    val extendedAttributes = new ArrayList<AttributeId>();
    val expectedValues = new HashMap<AttributeId, Long>();
    expectedValues.put(coreAttributeId, 1000L);
    metadata.updateAttributes(Collections.singletonMap(coreAttributeId, 1000L));
    for (int i = 0; i < attributeCount; i++) {
        AttributeId attributeId = AttributeId.uuid(0, i);
        extendedAttributes.add(attributeId);
        metadata.setLastUsed(i);
        metadata.updateAttributes(Collections.singletonMap(attributeId, (long) i));
        expectedValues.put(attributeId, (long) i);
    }
    checkAttributesEqual(expectedValues, metadata.getAttributes());
    // Evict first half of the attributes.
    int half = attributeCount / 2;
    int step = maxAttributeCount / 10;
    for (int i = 0; i <= half; i += step) {
        int evicted = metadata.cleanupAttributes(maxAttributeCount, i);
        if (i == 0) {
            Assert.assertEquals("Not expecting any evictions.", 0, evicted);
        } else {
            Assert.assertEquals("Unexpected number of evictions", step, evicted);
            for (int j = i - step; j < i; j++) {
                expectedValues.remove(extendedAttributes.get(j));
            }
        }
        checkAttributesEqual(expectedValues, metadata.getAttributes());
    }
    // For the second half, every 3rd attribute is not touched, every 3rd+1 is updated and every 3rd+2 is fetched.
    // We then verify that only the untouched ones will get evicted.
    int expectedEvicted = 0;
    long cutoff = metadata.getLastUsed();
    metadata.setLastUsed(cutoff + 1);
    for (int i = half; i < attributeCount; i++) {
        val attributeId = extendedAttributes.get(i);
        if (i % 3 == 1) {
            // We reuse the same value; it's simpler.
            metadata.updateAttributes(Collections.singletonMap(attributeId, (long) i));
        } else if (i % 3 == 2) {
            metadata.getAttributes().get(attributeId);
        } else {
            expectedValues.remove(attributeId);
            expectedEvicted++;
        }
    }
    // Force an eviction of all attributes, and verify that only the ones eligible for removal were removed.
    int evicted = metadata.cleanupAttributes(maxAttributeCount, cutoff + 1);
    Assert.assertEquals("Unexpected final eviction count.", expectedEvicted, evicted);
    checkAttributesEqual(expectedValues, metadata.getAttributes());
}