Examples with AttributeUpdate io.pravega.segmentstore.contracts.AttributeUpdate used on opensource projects

Example 36 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentContainerTests method testForSegmentPriority.

/**
 * Tests {@link StreamSegmentContainer#forSegment(String, OperationPriority, Duration)}.
 */
@Test
public void testForSegmentPriority() throws Exception {
    val segmentName = "Test";
    @Cleanup val context = new TestContext(DEFAULT_CONFIG, NO_TRUNCATIONS_DURABLE_LOG_CONFIG, INFREQUENT_FLUSH_WRITER_CONFIG, null);
    val durableLog = new AtomicReference<OperationLog>();
    val durableLogFactory = new WatchableOperationLogFactory(context.operationLogFactory, durableLog::set);
    @Cleanup val container = new StreamSegmentContainer(CONTAINER_ID, DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.attributeIndexFactory, new NoOpWriterFactory(), context.storageFactory, context.getDefaultExtensions(), executorService());
    container.startAsync().awaitRunning();
    container.createStreamSegment(segmentName, SegmentType.STREAM_SEGMENT, null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Create a few operations using the forSegment with desired priority.
    val s1 = container.forSegment(segmentName, OperationPriority.Critical, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    val futures = new ArrayList<CompletableFuture<Void>>();
    futures.add(Futures.toVoid(s1.append(new ByteArraySegment(new byte[1]), null, TIMEOUT)));
    futures.add(s1.updateAttributes(AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.randomUUID(), AttributeUpdateType.Replace, 1)), TIMEOUT));
    futures.add(s1.truncate(1, TIMEOUT));
    futures.add(Futures.toVoid(s1.seal(TIMEOUT)));
    Futures.allOf(futures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Await all operations to be added to the durable log, then fetch them all. We stop when we encounter the Seal we just added.
    val ops = readDurableLog(durableLog.get(), op -> op instanceof StreamSegmentSealOperation);
    // For those operations that we do care about, verify they have the right priority.
    int count = 0;
    for (val op : ops) {
        if (op instanceof SegmentOperation && ((SegmentOperation) op).getStreamSegmentId() == s1.getSegmentId()) {
            count++;
            Assert.assertEquals("Unexpected priority for " + op, OperationPriority.Critical, op.getDesiredPriority());
        }
    }
    AssertExtensions.assertGreaterThan("Expected at least one operation to be verified.", 0, count);
}

Example 37 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class DebugStreamSegmentContainerTests method testDataRecoveryContainerLevel.

/**
 * The test creates a container. Using it, some segments are created, data is written to them and their attributes are
 * updated. After Tier1 is flushed to the storage, the container is closed. From the storage, we recover the
 * segments and update their attributes. The data and attributes for each of the segment are verified to validate the
 * recovery process.
 */
@Test
public void testDataRecoveryContainerLevel() throws Exception {
    int attributesUpdatesPerSegment = 50;
    int segmentsCount = 10;
    final AttributeId attributeReplace = AttributeId.uuid(CORE_ATTRIBUTE_ID_PREFIX, RANDOM.nextInt(10));
    final long expectedAttributeValue = attributesUpdatesPerSegment;
    int containerId = 0;
    int containerCount = 1;
    // 1MB
    int maxDataSize = 1024 * 1024;
    StorageFactory storageFactory = new InMemoryStorageFactory(executorService());
    @Cleanup TestContext context = createContext(executorService());
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer container = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, storageFactory, context.getDefaultExtensions(), executorService());
    container.startAsync().awaitRunning();
    // 1. Create segments.
    ArrayList<String> segmentNames = new ArrayList<>();
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (int i = 0; i < segmentsCount; i++) {
        String segmentName = getSegmentName(i);
        segmentNames.add(segmentName);
        opFutures.add(container.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT));
    }
    // 1.1 Wait for all segments to be created prior to using them.
    Futures.allOf(opFutures).join();
    opFutures.clear();
    // 2. Write data and update some of the attributes.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArraySegment> segmentContents = new HashMap<>();
    for (String segmentName : segmentNames) {
        val dataSize = RANDOM.nextInt(maxDataSize);
        byte[] writeData = populate(dataSize);
        val appendData = new ByteArraySegment(writeData);
        val append = container.append(segmentName, appendData, null, TIMEOUT);
        opFutures.add(Futures.toVoid(append));
        lengths.put(segmentName, (long) dataSize);
        segmentContents.put(segmentName, appendData);
        for (int i = 0; i < attributesUpdatesPerSegment; i++) {
            AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            opFutures.add(container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Instead of waiting for the Writer to move data to Storage, we invoke the flushToStorage to verify that all
    // operations have been applied to Storage.
    val forceFlush = container.flushToStorage(TIMEOUT);
    forceFlush.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    container.close();
    // Get the storage instance
    @Cleanup Storage storage = storageFactory.createStorageAdapter();
    // 4. Move container metadata and its attribute segment to back up segments.
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    OperationLogFactory localDurableLogFactory2 = new DurableLogFactory(NO_TRUNCATIONS_DURABLE_LOG_CONFIG, new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), executorService());
    // Starts a DebugSegmentContainer with new Durable Log
    @Cleanup TestContext context1 = createContext(executorService());
    @Cleanup MetadataCleanupContainer container2 = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory2, context1.readIndexFactory, context1.attributeIndexFactory, context1.writerFactory, storageFactory, context1.getDefaultExtensions(), executorService());
    container2.startAsync().awaitRunning();
    Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainersMap = new HashMap<>();
    debugStreamSegmentContainersMap.put(containerId, container2);
    // 4. Recover all segments.
    recoverAllSegments(storage, debugStreamSegmentContainersMap, executorService(), TIMEOUT);
    // 5. Update core attributes using back up segments.
    updateCoreAttributes(backUpMetadataSegments, debugStreamSegmentContainersMap, executorService(), TIMEOUT);
    // 6. Verify Segment Data.
    for (val sc : segmentContents.entrySet()) {
        // Contents.
        byte[] expectedData = sc.getValue().array();
        byte[] actualData = new byte[expectedData.length];
        container2.read(sc.getKey(), 0, actualData.length, TIMEOUT).join().readRemaining(actualData, TIMEOUT);
        Assert.assertArrayEquals("Unexpected contents for " + sc.getKey(), expectedData, actualData);
        // Length.
        val si = container2.getStreamSegmentInfo(sc.getKey(), TIMEOUT).join();
        Assert.assertEquals("Unexpected length for " + sc.getKey(), expectedData.length, si.getLength());
        // Attributes.
        val attributes = container2.getAttributes(sc.getKey(), Collections.singleton(attributeReplace), false, TIMEOUT).join();
        Assert.assertEquals("Unexpected attribute for " + sc.getKey(), expectedAttributeValue, (long) attributes.get(attributeReplace));
    }
}

Example 38 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testStreamSegmentAppendWithOffset.

/**
 * Tests the ability of the ContainerMetadataUpdateTransaction to process (and accept) StreamSegmentAppendOperations with
 * predefined offsets.
 */
@Test
public void testStreamSegmentAppendWithOffset() throws Exception {
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    // Append #1 (at offset 0).
    long offset = metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength();
    StreamSegmentAppendOperation appendOp = createAppendWithOffset(offset);
    txn.preProcessOperation(appendOp);
    Assert.assertEquals("Unexpected StreamSegmentOffset after call to preProcess in non-recovery mode.", offset, appendOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(appendOp, "call to preProcess in non-recovery mode");
    Assert.assertEquals("preProcess(Append) seems to have changed the Updater internal state.", offset, txn.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("preProcess(Append) seems to have changed the metadata.", offset, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    txn.acceptOperation(appendOp);
    // Append #2 (after Append #1)
    offset = appendOp.getStreamSegmentOffset() + appendOp.getLength();
    appendOp = createAppendWithOffset(offset);
    txn.preProcessOperation(appendOp);
    Assert.assertEquals("Unexpected StreamSegmentOffset after call to preProcess in non-recovery mode.", offset, appendOp.getStreamSegmentOffset());
    checkNoSequenceNumberAssigned(appendOp, "call to preProcess in non-recovery mode");
    Assert.assertEquals("preProcess(Append) seems to have changed the Updater internal state.", offset, txn.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    Assert.assertEquals("preProcess(Append) seems to have changed the metadata.", SEGMENT_LENGTH, metadata.getStreamSegmentMetadata(SEGMENT_ID).getLength());
    txn.acceptOperation(appendOp);
    // Append #3 (wrong offset)
    offset = appendOp.getStreamSegmentOffset() + appendOp.getLength() - 1;
    StreamSegmentAppendOperation badAppendOp = createAppendWithOffset(offset);
    AssertExtensions.assertThrows("preProcessOperations accepted an append with the wrong offset.", () -> txn.preProcessOperation(badAppendOp), ex -> ex instanceof BadOffsetException);
    // Append #4 (wrong offset + wrong attribute). AS PER SEGMENT STORE CONTRACT, BadAttributeUpdateException takes precedence.
    badAppendOp.getAttributeUpdates().add(new AttributeUpdate(AttributeId.randomUUID(), AttributeUpdateType.ReplaceIfEquals, 1, 1234));
    AssertExtensions.assertThrows("preProcessOperations failed with wrong exception when append has both bad offset and bad attribute.", () -> txn.preProcessOperation(badAppendOp), ex -> ex instanceof BadAttributeUpdateException);
    AssertExtensions.assertThrows("acceptOperation accepted an append that was rejected during preProcessing.", () -> txn.acceptOperation(badAppendOp), ex -> ex instanceof MetadataUpdateException);
}

Example 39 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testWithBadAttributes.

private void testWithBadAttributes(Function<AttributeUpdateCollection, Operation> createOperation, Consumer<Operation> checkAfterSuccess, Consumer<Operation> checkAfterRejection) throws Exception {
    final AttributeId attributeNoUpdate = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfGreater = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfEquals = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfEqualsNullValue = AttributeId.randomUUID();
    if (checkAfterSuccess == null) {
        checkAfterSuccess = TestUtils::doNothing;
    }
    if (checkAfterRejection == null) {
        checkAfterRejection = TestUtils::doNothing;
    }
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    BiFunction<Throwable, Boolean, Boolean> exceptionChecker = (ex, expectNoPreviousValue) -> (ex instanceof BadAttributeUpdateException) && ((BadAttributeUpdateException) ex).isPreviousValueMissing() == expectNoPreviousValue;
    // Values not set
    AttributeUpdateCollection attributeUpdates = new AttributeUpdateCollection();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.ReplaceIfEquals, 0, 0));
    val op1 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to CAS-update an attribute with no previous value.", () -> txn.preProcessOperation(op1), ex -> exceptionChecker.apply(ex, true));
    checkAfterRejection.accept(op1);
    // Append #1.
    attributeUpdates.clear();
    // Initial add, so it's ok.
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    // Initial Add.
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.Replace, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEqualsNullValue, AttributeUpdateType.None, Attributes.NULL_ATTRIBUTE_VALUE));
    val expectedValues = attributeUpdates.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue));
    expectedValues.put(Attributes.ATTRIBUTE_SEGMENT_TYPE, DEFAULT_TYPE.getValue());
    Operation op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    checkAfterSuccess.accept(op);
    // ReplaceIfEquals fails when the current attribute value is NULL_ATTRIBUTE_VALUE (this should not set the IsPreviousValueMissing flag).
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEqualsNullValue, AttributeUpdateType.ReplaceIfEquals, 1, 1));
    val op2 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to CAS-update an attribute with no previous value.", () -> txn.preProcessOperation(op2), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op2);
    // Append #2: Try to update attribute that cannot be updated.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 3));
    val op3 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an unmodifiable attribute.", () -> txn.preProcessOperation(op3), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op2);
    // Append #3: Try to update attribute with bad value for ReplaceIfGreater attribute.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 1));
    val op4 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an attribute with the wrong value for ReplaceIfGreater.", () -> txn.preProcessOperation(op4), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op4);
    // Append #4: Try to update attribute with bad value for ReplaceIfEquals attribute.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, 3, 3));
    val op5 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an attribute with the wrong comparison value for ReplaceIfGreater.", () -> txn.preProcessOperation(op5), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op5);
    // Reset the attribute update list to its original state so we can do the final verification.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, 2, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEqualsNullValue, AttributeUpdateType.None, Attributes.NULL_ATTRIBUTE_VALUE));
    verifyAttributeUpdates("after rejected operations", txn, attributeUpdates, expectedValues);
    txn.commit(metadata);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in segment metadata after commit.", expectedValues, metadata.getStreamSegmentMetadata(SEGMENT_ID));
}

Example 40 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testAttributeIdLengthValidation.

/**
 * Tests the ability to validate the type and lengths of Attribute Ids coming in via appends or update attributes,
 * in accordance with the Segment's declared attribute id length.
 */
@Test
public void testAttributeIdLengthValidation() throws Exception {
    final AttributeId coreAttribute = Attributes.ATTRIBUTE_SEGMENT_ROOT_POINTER;
    final AttributeId extAttributeUUID = AttributeId.randomUUID();
    final AttributeId extAttributeShort1 = AttributeId.random(AttributeId.UUID.ATTRIBUTE_ID_LENGTH);
    final AttributeId extAttributeShort2 = AttributeId.random(AttributeId.UUID.ATTRIBUTE_ID_LENGTH);
    final AttributeId extAttributeLong1 = AttributeId.random(AttributeId.Variable.MAX_LENGTH);
    Function<AttributeId, UpdateAttributesOperation> createOp = id -> new UpdateAttributesOperation(SEGMENT_ID, AttributeUpdateCollection.from(new AttributeUpdate(id, AttributeUpdateType.Replace, 1L)));
    val metadata = createMetadata();
    val sm = metadata.getStreamSegmentMetadata(SEGMENT_ID);
    // 1. All UUIDs
    val txn1 = createUpdateTransaction(metadata);
    // Core attributes must always be allowed.
    txn1.preProcessOperation(createOp.apply(coreAttribute));
    // Extended UUID attribute should be allowed in this case.
    txn1.preProcessOperation(createOp.apply(extAttributeUUID));
    AssertExtensions.assertThrows("Variable-Length accepted when no length declared", () -> txn1.preProcessOperation(createOp.apply(extAttributeShort1)), ex -> ex instanceof AttributeIdLengthMismatchException);
    // 2. Declare UUID, try Variable length.
    sm.updateAttributes(Collections.singletonMap(Attributes.ATTRIBUTE_ID_LENGTH, 0L));
    sm.refreshDerivedProperties();
    val txn2 = createUpdateTransaction(metadata);
    // Core attributes must always be allowed.
    txn2.preProcessOperation(createOp.apply(coreAttribute));
    // Extended UUID attribute should be allowed in this case.
    txn2.preProcessOperation(createOp.apply(extAttributeUUID));
    AssertExtensions.assertThrows("Variable-Length accepted when length declared to be 0 (UUID).", () -> txn2.preProcessOperation(createOp.apply(extAttributeShort1)), ex -> ex instanceof AttributeIdLengthMismatchException);
    // 3. Variable Lengths declared
    sm.updateAttributes(Collections.singletonMap(Attributes.ATTRIBUTE_ID_LENGTH, (long) extAttributeShort1.byteCount()));
    sm.refreshDerivedProperties();
    val txn3 = createUpdateTransaction(metadata);
    // Core attributes must always be allowed.
    txn3.preProcessOperation(createOp.apply(coreAttribute));
    txn3.preProcessOperation(createOp.apply(extAttributeShort1));
    txn3.preProcessOperation(createOp.apply(extAttributeShort2));
    AssertExtensions.assertThrows("UUID accepted when length declared to be Variable.", () -> txn3.preProcessOperation(createOp.apply(extAttributeUUID)), ex -> ex instanceof AttributeIdLengthMismatchException);
    AssertExtensions.assertThrows("Wrong-length accepted when length declared to be Variable.", () -> txn3.preProcessOperation(createOp.apply(extAttributeLong1)), ex -> ex instanceof AttributeIdLengthMismatchException);
}