Examples with AttributeId io.pravega.segmentstore.contracts.AttributeId used on opensource projects

Example 21 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class AttributeIndexTests method testIterator.

/**
 * Tests the ability to iterate through a certain range of attributes in the Index.
 */
@Test
public void testIterator() {
    final int count = 2000;
    final int checkSkipCount = 10;
    val attributes = IntStream.range(-count / 2, count / 2).mapToObj(i -> AttributeId.uuid(i, -i)).collect(Collectors.toList());
    @Cleanup val context = new TestContext(DEFAULT_CONFIG);
    populateSegments(context);
    // 1. Populate and verify first index.
    val idx = context.index.forSegment(SEGMENT_ID, TIMEOUT).join();
    val expectedValues = new HashMap<AttributeId, Long>();
    // Populate data.
    AtomicLong nextValue = new AtomicLong(0);
    for (AttributeId attributeId : attributes) {
        long value = nextValue.getAndIncrement();
        expectedValues.put(attributeId, value);
    }
    idx.update(expectedValues, TIMEOUT).join();
    // Check iterator.
    // Sort the keys using natural order (AttributeId comparator).
    val sortedKeys = expectedValues.keySet().stream().sorted().collect(Collectors.toList());
    // Add some values outside of the bounds.
    sortedKeys.add(0, AttributeId.uuid(Long.MIN_VALUE, Long.MIN_VALUE));
    sortedKeys.add(AttributeId.uuid(Long.MAX_VALUE, Long.MAX_VALUE));
    // Check various combinations.
    for (int startIndex = 0; startIndex < sortedKeys.size() / 2; startIndex += checkSkipCount) {
        int lastIndex = sortedKeys.size() - startIndex - 1;
        val fromId = sortedKeys.get(startIndex);
        val toId = sortedKeys.get(lastIndex);
        // The expected keys are all the Keys from the start index to the end index, excluding the outside-the-bounds values.
        val expectedIterator = sortedKeys.subList(Math.max(1, startIndex), Math.min(sortedKeys.size() - 1, lastIndex + 1)).iterator();
        val iterator = idx.iterator(fromId, toId, TIMEOUT);
        iterator.forEachRemaining(batch -> batch.forEach(attribute -> {
            Assert.assertTrue("Not expecting any more attributes in the iteration.", expectedIterator.hasNext());
            val expectedId = expectedIterator.next();
            val expectedValue = expectedValues.get(expectedId);
            Assert.assertEquals("Unexpected Id.", expectedId, attribute.getKey());
            Assert.assertEquals("Unexpected value for " + expectedId, expectedValue, attribute.getValue());
        }), executorService()).join();
        // Verify there are no more attributes that we are expecting.
        Assert.assertFalse("Not all expected attributes were returned.", expectedIterator.hasNext());
    }
}

Example 22 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class AttributeIndexTests method checkIndex.

private void checkIndex(AttributeIndex index, Map<AttributeId, Long> expectedValues) {
    val actual = index.get(expectedValues.keySet(), TIMEOUT).join();
    val expected = expectedValues.entrySet().stream().filter(e -> e.getValue() != Attributes.NULL_ATTRIBUTE_VALUE).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    AssertExtensions.assertMapEquals("Unexpected attributes in index.", expected, actual);
}

Example 23 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class DebugStreamSegmentContainerTests method testDataRecoveryContainerLevel.

/**
 * The test creates a container. Using it, some segments are created, data is written to them and their attributes are
 * updated. After Tier1 is flushed to the storage, the container is closed. From the storage, we recover the
 * segments and update their attributes. The data and attributes for each of the segment are verified to validate the
 * recovery process.
 */
@Test
public void testDataRecoveryContainerLevel() throws Exception {
    int attributesUpdatesPerSegment = 50;
    int segmentsCount = 10;
    final AttributeId attributeReplace = AttributeId.uuid(CORE_ATTRIBUTE_ID_PREFIX, RANDOM.nextInt(10));
    final long expectedAttributeValue = attributesUpdatesPerSegment;
    int containerId = 0;
    int containerCount = 1;
    // 1MB
    int maxDataSize = 1024 * 1024;
    StorageFactory storageFactory = new InMemoryStorageFactory(executorService());
    @Cleanup TestContext context = createContext(executorService());
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer container = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, storageFactory, context.getDefaultExtensions(), executorService());
    container.startAsync().awaitRunning();
    // 1. Create segments.
    ArrayList<String> segmentNames = new ArrayList<>();
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (int i = 0; i < segmentsCount; i++) {
        String segmentName = getSegmentName(i);
        segmentNames.add(segmentName);
        opFutures.add(container.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT));
    }
    // 1.1 Wait for all segments to be created prior to using them.
    Futures.allOf(opFutures).join();
    opFutures.clear();
    // 2. Write data and update some of the attributes.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArraySegment> segmentContents = new HashMap<>();
    for (String segmentName : segmentNames) {
        val dataSize = RANDOM.nextInt(maxDataSize);
        byte[] writeData = populate(dataSize);
        val appendData = new ByteArraySegment(writeData);
        val append = container.append(segmentName, appendData, null, TIMEOUT);
        opFutures.add(Futures.toVoid(append));
        lengths.put(segmentName, (long) dataSize);
        segmentContents.put(segmentName, appendData);
        for (int i = 0; i < attributesUpdatesPerSegment; i++) {
            AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            opFutures.add(container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Instead of waiting for the Writer to move data to Storage, we invoke the flushToStorage to verify that all
    // operations have been applied to Storage.
    val forceFlush = container.flushToStorage(TIMEOUT);
    forceFlush.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    container.close();
    // Get the storage instance
    @Cleanup Storage storage = storageFactory.createStorageAdapter();
    // 4. Move container metadata and its attribute segment to back up segments.
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    OperationLogFactory localDurableLogFactory2 = new DurableLogFactory(NO_TRUNCATIONS_DURABLE_LOG_CONFIG, new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), executorService());
    // Starts a DebugSegmentContainer with new Durable Log
    @Cleanup TestContext context1 = createContext(executorService());
    @Cleanup MetadataCleanupContainer container2 = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory2, context1.readIndexFactory, context1.attributeIndexFactory, context1.writerFactory, storageFactory, context1.getDefaultExtensions(), executorService());
    container2.startAsync().awaitRunning();
    Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainersMap = new HashMap<>();
    debugStreamSegmentContainersMap.put(containerId, container2);
    // 4. Recover all segments.
    recoverAllSegments(storage, debugStreamSegmentContainersMap, executorService(), TIMEOUT);
    // 5. Update core attributes using back up segments.
    updateCoreAttributes(backUpMetadataSegments, debugStreamSegmentContainersMap, executorService(), TIMEOUT);
    // 6. Verify Segment Data.
    for (val sc : segmentContents.entrySet()) {
        // Contents.
        byte[] expectedData = sc.getValue().array();
        byte[] actualData = new byte[expectedData.length];
        container2.read(sc.getKey(), 0, actualData.length, TIMEOUT).join().readRemaining(actualData, TIMEOUT);
        Assert.assertArrayEquals("Unexpected contents for " + sc.getKey(), expectedData, actualData);
        // Length.
        val si = container2.getStreamSegmentInfo(sc.getKey(), TIMEOUT).join();
        Assert.assertEquals("Unexpected length for " + sc.getKey(), expectedData.length, si.getLength());
        // Attributes.
        val attributes = container2.getAttributes(sc.getKey(), Collections.singleton(attributeReplace), false, TIMEOUT).join();
        Assert.assertEquals("Unexpected attribute for " + sc.getKey(), expectedAttributeValue, (long) attributes.get(attributeReplace));
    }
}

Example 24 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testWithBadAttributes.

private void testWithBadAttributes(Function<AttributeUpdateCollection, Operation> createOperation, Consumer<Operation> checkAfterSuccess, Consumer<Operation> checkAfterRejection) throws Exception {
    final AttributeId attributeNoUpdate = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfGreater = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfEquals = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfEqualsNullValue = AttributeId.randomUUID();
    if (checkAfterSuccess == null) {
        checkAfterSuccess = TestUtils::doNothing;
    }
    if (checkAfterRejection == null) {
        checkAfterRejection = TestUtils::doNothing;
    }
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    BiFunction<Throwable, Boolean, Boolean> exceptionChecker = (ex, expectNoPreviousValue) -> (ex instanceof BadAttributeUpdateException) && ((BadAttributeUpdateException) ex).isPreviousValueMissing() == expectNoPreviousValue;
    // Values not set
    AttributeUpdateCollection attributeUpdates = new AttributeUpdateCollection();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.ReplaceIfEquals, 0, 0));
    val op1 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to CAS-update an attribute with no previous value.", () -> txn.preProcessOperation(op1), ex -> exceptionChecker.apply(ex, true));
    checkAfterRejection.accept(op1);
    // Append #1.
    attributeUpdates.clear();
    // Initial add, so it's ok.
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    // Initial Add.
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.Replace, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEqualsNullValue, AttributeUpdateType.None, Attributes.NULL_ATTRIBUTE_VALUE));
    val expectedValues = attributeUpdates.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue));
    expectedValues.put(Attributes.ATTRIBUTE_SEGMENT_TYPE, DEFAULT_TYPE.getValue());
    Operation op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    checkAfterSuccess.accept(op);
    // ReplaceIfEquals fails when the current attribute value is NULL_ATTRIBUTE_VALUE (this should not set the IsPreviousValueMissing flag).
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEqualsNullValue, AttributeUpdateType.ReplaceIfEquals, 1, 1));
    val op2 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to CAS-update an attribute with no previous value.", () -> txn.preProcessOperation(op2), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op2);
    // Append #2: Try to update attribute that cannot be updated.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 3));
    val op3 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an unmodifiable attribute.", () -> txn.preProcessOperation(op3), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op2);
    // Append #3: Try to update attribute with bad value for ReplaceIfGreater attribute.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 1));
    val op4 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an attribute with the wrong value for ReplaceIfGreater.", () -> txn.preProcessOperation(op4), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op4);
    // Append #4: Try to update attribute with bad value for ReplaceIfEquals attribute.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, 3, 3));
    val op5 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an attribute with the wrong comparison value for ReplaceIfGreater.", () -> txn.preProcessOperation(op5), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op5);
    // Reset the attribute update list to its original state so we can do the final verification.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, 2, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEqualsNullValue, AttributeUpdateType.None, Attributes.NULL_ATTRIBUTE_VALUE));
    verifyAttributeUpdates("after rejected operations", txn, attributeUpdates, expectedValues);
    txn.commit(metadata);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in segment metadata after commit.", expectedValues, metadata.getStreamSegmentMetadata(SEGMENT_ID));
}

Example 25 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testAttributeIdLengthValidation.

/**
 * Tests the ability to validate the type and lengths of Attribute Ids coming in via appends or update attributes,
 * in accordance with the Segment's declared attribute id length.
 */
@Test
public void testAttributeIdLengthValidation() throws Exception {
    final AttributeId coreAttribute = Attributes.ATTRIBUTE_SEGMENT_ROOT_POINTER;
    final AttributeId extAttributeUUID = AttributeId.randomUUID();
    final AttributeId extAttributeShort1 = AttributeId.random(AttributeId.UUID.ATTRIBUTE_ID_LENGTH);
    final AttributeId extAttributeShort2 = AttributeId.random(AttributeId.UUID.ATTRIBUTE_ID_LENGTH);
    final AttributeId extAttributeLong1 = AttributeId.random(AttributeId.Variable.MAX_LENGTH);
    Function<AttributeId, UpdateAttributesOperation> createOp = id -> new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(new AttributeUpdate(id, AttributeUpdateType.Replace, 1L)));
    val metadata = createMetadata();
    val sm = metadata.getStreamSegmentMetadata(SEGMENT_ID);
    // 1. All UUIDs
    val txn1 = createUpdateTransaction(metadata);
    // Core attributes must always be allowed.
    txn1.preProcessOperation(createOp.apply(coreAttribute));
    // Extended UUID attribute should be allowed in this case.
    txn1.preProcessOperation(createOp.apply(extAttributeUUID));
    AssertExtensions.assertThrows("Variable-Length accepted when no length declared", () -> txn1.preProcessOperation(createOp.apply(extAttributeShort1)), ex -> ex instanceof AttributeIdLengthMismatchException);
    // 2. Declare UUID, try Variable length.
    sm.updateAttributes(Collections.singletonMap(Attributes.ATTRIBUTE_ID_LENGTH, 0L));
    sm.refreshDerivedProperties();
    val txn2 = createUpdateTransaction(metadata);
    // Core attributes must always be allowed.
    txn2.preProcessOperation(createOp.apply(coreAttribute));
    // Extended UUID attribute should be allowed in this case.
    txn2.preProcessOperation(createOp.apply(extAttributeUUID));
    AssertExtensions.assertThrows("Variable-Length accepted when length declared to be 0 (UUID).", () -> txn2.preProcessOperation(createOp.apply(extAttributeShort1)), ex -> ex instanceof AttributeIdLengthMismatchException);
    // 3. Variable Lengths declared
    sm.updateAttributes(Collections.singletonMap(Attributes.ATTRIBUTE_ID_LENGTH, (long) extAttributeShort1.byteCount()));
    sm.refreshDerivedProperties();
    val txn3 = createUpdateTransaction(metadata);
    // Core attributes must always be allowed.
    txn3.preProcessOperation(createOp.apply(coreAttribute));
    txn3.preProcessOperation(createOp.apply(extAttributeShort1));
    txn3.preProcessOperation(createOp.apply(extAttributeShort2));
    AssertExtensions.assertThrows("UUID accepted when length declared to be Variable.", () -> txn3.preProcessOperation(createOp.apply(extAttributeUUID)), ex -> ex instanceof AttributeIdLengthMismatchException);
    AssertExtensions.assertThrows("Wrong-length accepted when length declared to be Variable.", () -> txn3.preProcessOperation(createOp.apply(extAttributeLong1)), ex -> ex instanceof AttributeIdLengthMismatchException);
}