Examples with AttributeUpdateCollection io.pravega.segmentstore.contracts.AttributeUpdateCollection used on opensource projects

Example 11 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class StreamSegmentContainerTests method testBasicConditionalMergeScenarios.

/**
 * Test in detail the basic situations that a conditional segment merge can face.
 */
@Test
public void testBasicConditionalMergeScenarios() throws Exception {
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    final String parentSegment = "parentSegment";
    // This will be the attribute update to execute against the parent segment.
    Function<String, AttributeUpdateCollection> attributeUpdateForTxn = txnName -> AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.fromUUID(UUID.nameUUIDFromBytes(txnName.getBytes())), AttributeUpdateType.ReplaceIfEquals, txnName.hashCode() + 1, txnName.hashCode()));
    Function<String, Long> getAttributeValue = txnName -> {
        AttributeId attributeId = AttributeId.fromUUID(UUID.nameUUIDFromBytes(txnName.getBytes()));
        return context.container.getAttributes(parentSegment, Collections.singletonList(attributeId), true, TIMEOUT).join().get(attributeId);
    };
    // Create a parent Segment.
    context.container.createStreamSegment(parentSegment, getSegmentType(parentSegment), null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentType segmentType = getSegmentType(parentSegment);
    // Case 1: Create and empty transaction that fails to merge conditionally due to bad attributes.
    String txnName = NameUtils.getTransactionNameFromId(parentSegment, UUID.randomUUID());
    AttributeId txnAttributeId = AttributeId.fromUUID(UUID.nameUUIDFromBytes(txnName.getBytes()));
    context.container.createStreamSegment(txnName, segmentType, null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    AttributeUpdateCollection attributeUpdates = attributeUpdateForTxn.apply(txnName);
    AssertExtensions.assertFutureThrows("Transaction was expected to fail on attribute update", context.container.mergeStreamSegment(parentSegment, txnName, attributeUpdates, TIMEOUT), ex -> ex instanceof BadAttributeUpdateException);
    Assert.assertEquals(Attributes.NULL_ATTRIBUTE_VALUE, (long) getAttributeValue.apply(txnName));
    // Case 2: Now, we prepare the attributes in the parent segment so the merge of the empty transaction succeeds.
    context.container.updateAttributes(parentSegment, AttributeUpdateCollection.from(new AttributeUpdate(txnAttributeId, AttributeUpdateType.Replace, txnName.hashCode())), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // As the source segment is empty, the amount of merged data should be 0.
    Assert.assertEquals(0L, context.container.mergeStreamSegment(parentSegment, txnName, attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).getMergedDataLength());
    // But the attribute related to that transaction merge on the parent segment should have been updated.
    Assert.assertEquals(txnName.hashCode() + 1L, (long) context.container.getAttributes(parentSegment, Collections.singletonList(txnAttributeId), true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).get(txnAttributeId));
    // Case 3: Create a non-empty transaction that should fail due to a conditional attribute update failure.
    txnName = NameUtils.getTransactionNameFromId(parentSegment, UUID.randomUUID());
    txnAttributeId = AttributeId.fromUUID(UUID.nameUUIDFromBytes(txnName.getBytes()));
    attributeUpdates = attributeUpdateForTxn.apply(txnName);
    context.container.createStreamSegment(txnName, segmentType, null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Add some appends to the transaction.
    RefCountByteArraySegment appendData = getAppendData(txnName, 1);
    context.container.append(txnName, appendData, null, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Attempt the conditional merge.
    AssertExtensions.assertFutureThrows("Transaction was expected to fail on attribute update", context.container.mergeStreamSegment(parentSegment, txnName, attributeUpdates, TIMEOUT), ex -> ex instanceof BadAttributeUpdateException);
    Assert.assertEquals(Attributes.NULL_ATTRIBUTE_VALUE, (long) getAttributeValue.apply(txnName));
    // Case 4: Now, we prepare the attributes in the parent segment so the merge of the non-empty transaction succeeds.
    context.container.updateAttributes(parentSegment, AttributeUpdateCollection.from(new AttributeUpdate(txnAttributeId, AttributeUpdateType.Replace, txnName.hashCode())), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // As the source segment is non-empty, the amount of merged data should be greater than 0.
    Assert.assertTrue(context.container.mergeStreamSegment(parentSegment, txnName, attributeUpdates, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).getMergedDataLength() > 0);
    // The attribute related to that transaction merge on the parent segment should have been updated as well.
    Assert.assertEquals(txnName.hashCode() + 1L, (long) context.container.getAttributes(parentSegment, Collections.singletonList(txnAttributeId), true, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).get(txnAttributeId));
    context.container.stopAsync().awaitTerminated();
}

Example 12 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class StreamSegmentContainerTests method testExtendedAttributesConditionalUpdates.

/**
 * Test conditional updates for Extended Attributes when they are not loaded in memory (i.e., they will need to be
 * auto-fetched from the AttributeIndex so that the operation may succeed).
 */
@Test
public void testExtendedAttributesConditionalUpdates() throws Exception {
    final AttributeId ea1 = AttributeId.uuid(0, 1);
    final AttributeId ea2 = AttributeId.uuid(0, 2);
    final List<AttributeId> allAttributes = Stream.of(ea1, ea2).collect(Collectors.toList());
    // We set a longer Segment Expiration time, since this test executes more operations than the others.
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_LONG));
    AtomicInteger expectedAttributeValue = new AtomicInteger(0);
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the StreamSegments and set the initial attribute values.
    ArrayList<String> segmentNames = createSegments(localContainer);
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    // 2. Update some of the attributes.
    expectedAttributeValue.set(1);
    for (String segmentName : segmentNames) {
        AttributeUpdateCollection attributeUpdates = allAttributes.stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Accumulate, 1)).collect(Collectors.toCollection(AttributeUpdateCollection::new));
        opFutures.add(localContainer.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Force these segments out of memory, so that we may verify conditional appends/updates on extended attributes.
    localContainer.triggerMetadataCleanup(segmentNames).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Execute various conditional operations using these attributes as comparison.
    long compare = expectedAttributeValue.getAndIncrement();
    long set = expectedAttributeValue.get();
    boolean badUpdate = false;
    for (String segmentName : segmentNames) {
        if (badUpdate) {
            // For every other segment, try to do a bad update, then a good one. This helps us verify both direct
            // conditional operations, failed conditional operations and conditional operations with cached attributes.
            AssertExtensions.assertSuppliedFutureThrows("Conditional append succeeded with incorrect compare value.", () -> localContainer.append(segmentName, getAppendData(segmentName, 0), AttributeUpdateCollection.from(new AttributeUpdate(ea1, AttributeUpdateType.ReplaceIfEquals, set, compare - 1)), TIMEOUT), ex -> (ex instanceof BadAttributeUpdateException) && !((BadAttributeUpdateException) ex).isPreviousValueMissing());
            AssertExtensions.assertSuppliedFutureThrows("Conditional update-attributes succeeded with incorrect compare value.", () -> localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(ea2, AttributeUpdateType.ReplaceIfEquals, set, compare - 1)), TIMEOUT), ex -> (ex instanceof BadAttributeUpdateException) && !((BadAttributeUpdateException) ex).isPreviousValueMissing());
        }
        opFutures.add(Futures.toVoid(localContainer.append(segmentName, getAppendData(segmentName, 0), AttributeUpdateCollection.from(new AttributeUpdate(ea1, AttributeUpdateType.ReplaceIfEquals, set, compare)), TIMEOUT)));
        opFutures.add(localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(ea2, AttributeUpdateType.ReplaceIfEquals, set, compare)), TIMEOUT));
        badUpdate = !badUpdate;
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Evict the segment from memory, then verify results.
    localContainer.triggerMetadataCleanup(segmentNames).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    for (String segmentName : segmentNames) {
        // Verify all attribute values.
        val attributeValues = localContainer.getAttributes(segmentName, allAttributes, true, TIMEOUT).join();
        val sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        for (val attributeId : allAttributes) {
            Assert.assertEquals("Unexpected value for non-cached attribute " + attributeId + " for segment " + segmentName, expectedAttributeValue.get(), (long) attributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            Assert.assertEquals("Unexpected value for metadata attribute " + attributeId + " for segment " + segmentName, expectedAttributeValue.get(), (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
        }
    }
    localContainer.stopAsync().awaitTerminated();
}

Example 13 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class StreamSegmentContainerTests method testAttributes.

/**
 * Tests the ability to set attributes (via append() or updateAttributes()), then fetch them back using getAttributes(),
 * emphasizing on Extended Attributes that are dumped into Storage and cleared from memory.
 */
@Test
public void testAttributes() throws Exception {
    final AttributeId coreAttribute = Attributes.EVENT_COUNT;
    final int variableAttributeIdLength = 4;
    final List<AttributeId> extendedAttributesUUID = Arrays.asList(AttributeId.randomUUID(), AttributeId.randomUUID());
    final List<AttributeId> extendedAttributesVariable = Arrays.asList(AttributeId.random(variableAttributeIdLength), AttributeId.random(variableAttributeIdLength));
    final List<AttributeId> allAttributesWithUUID = Stream.concat(extendedAttributesUUID.stream(), Stream.of(coreAttribute)).collect(Collectors.toList());
    final List<AttributeId> allAttributesWithVariable = Stream.concat(extendedAttributesVariable.stream(), Stream.of(coreAttribute)).collect(Collectors.toList());
    final AttributeId segmentLengthAttributeUUID = AttributeId.randomUUID();
    final AttributeId segmentLengthAttributeVariable = AttributeId.random(variableAttributeIdLength);
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    final TestContainerConfig containerConfig = new TestContainerConfig();
    containerConfig.setSegmentMetadataExpiration(Duration.ofMillis(EVICTION_SEGMENT_EXPIRATION_MILLIS_SHORT));
    containerConfig.setMaxCachedExtendedAttributeCount(SEGMENT_COUNT * allAttributesWithUUID.size());
    @Cleanup TestContext context = createContext();
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(FREQUENT_TRUNCATIONS_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer localContainer = new MetadataCleanupContainer(CONTAINER_ID, containerConfig, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    localContainer.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    val segmentNames = IntStream.range(0, SEGMENT_COUNT).boxed().collect(Collectors.toMap(StreamSegmentContainerTests::getSegmentName, i -> i % 2 == 0 ? variableAttributeIdLength : 0));
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (val sn : segmentNames.entrySet()) {
        opFutures.add(localContainer.createStreamSegment(sn.getKey(), SegmentType.STREAM_SEGMENT, AttributeUpdateCollection.from(new AttributeUpdate(Attributes.ATTRIBUTE_ID_LENGTH, AttributeUpdateType.None, sn.getValue())), TIMEOUT));
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Predicate<Map.Entry<String, Integer>> isUUIDOnly = e -> e.getValue() == 0;
    // 2. Add some appends.
    for (val sn : segmentNames.entrySet()) {
        boolean isUUID = isUUIDOnly.test(sn);
        for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
            AttributeUpdateCollection attributeUpdates = (isUUID ? allAttributesWithUUID : allAttributesWithVariable).stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Accumulate, 1)).collect(Collectors.toCollection(AttributeUpdateCollection::new));
            opFutures.add(Futures.toVoid(localContainer.append(sn.getKey(), getAppendData(sn.getKey(), i), attributeUpdates, TIMEOUT)));
        }
    }
    // 2.1 Update some of the attributes.
    for (val sn : segmentNames.entrySet()) {
        boolean isUUID = isUUIDOnly.test(sn);
        for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
            AttributeUpdateCollection attributeUpdates = (isUUID ? allAttributesWithUUID : allAttributesWithVariable).stream().map(attributeId -> new AttributeUpdate(attributeId, AttributeUpdateType.Accumulate, 1)).collect(Collectors.toCollection(AttributeUpdateCollection::new));
            opFutures.add(localContainer.updateAttributes(sn.getKey(), attributeUpdates, TIMEOUT));
        }
        // Verify that we are not allowed to update attributes of the wrong type.
        val badUpdate = new AttributeUpdate(isUUID ? AttributeId.random(variableAttributeIdLength) : AttributeId.randomUUID(), AttributeUpdateType.Accumulate, 1);
        AssertExtensions.assertSuppliedFutureThrows("updateAttributes allowed updating attributes with wrong type and/or length.", () -> localContainer.updateAttributes(sn.getKey(), AttributeUpdateCollection.from(badUpdate), TIMEOUT), ex -> ex instanceof AttributeIdLengthMismatchException);
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 2.2 Dynamic attributes.
    for (val sn : segmentNames.entrySet()) {
        boolean isUUID = isUUIDOnly.test(sn);
        val dynamicId = isUUID ? segmentLengthAttributeUUID : segmentLengthAttributeVariable;
        val dynamicAttributes = AttributeUpdateCollection.from(new DynamicAttributeUpdate(dynamicId, AttributeUpdateType.Replace, DynamicAttributeValue.segmentLength(10)));
        val appendData = getAppendData(sn.getKey(), 1000);
        val lastOffset = localContainer.append(sn.getKey(), appendData, dynamicAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        val expectedValue = lastOffset - appendData.getLength() + 10;
        Assert.assertEquals(expectedValue, (long) localContainer.getAttributes(sn.getKey(), Collections.singleton(dynamicId), false, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).get(dynamicId));
    }
    // 3. getSegmentInfo
    for (val sn : segmentNames.entrySet()) {
        val segmentName = sn.getKey();
        val allAttributes = isUUIDOnly.test(sn) ? allAttributesWithUUID : allAttributesWithVariable;
        val allAttributeValues = localContainer.getAttributes(segmentName, allAttributes, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected number of attributes retrieved via getAttributes().", allAttributes.size(), allAttributeValues.size());
        // Verify all attribute values.
        SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        for (val attributeId : allAttributes) {
            Assert.assertEquals("Unexpected value for attribute " + attributeId + " via getInfo() for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            Assert.assertEquals("Unexpected value for attribute " + attributeId + " via getAttributes() for segment " + segmentName, expectedAttributeValue, (long) allAttributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
        }
        // Verify we can't request wrong lengths/types.
        val badId = isUUIDOnly.test(sn) ? AttributeId.random(variableAttributeIdLength) : AttributeId.randomUUID();
        AssertExtensions.assertSuppliedFutureThrows("getAttributes allowed getting attributes with wrong type and/or length.", () -> localContainer.getAttributes(segmentName, Collections.singleton(badId), true, TIMEOUT), ex -> ex instanceof IllegalArgumentException);
    }
    // Force these segments out of memory, so that we may verify that extended attributes are still recoverable.
    localContainer.triggerMetadataCleanup(segmentNames.keySet()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    for (val sn : segmentNames.entrySet()) {
        val segmentName = sn.getKey();
        val allAttributes = isUUIDOnly.test(sn) ? allAttributesWithUUID : allAttributesWithVariable;
        val allAttributeValues = localContainer.getAttributes(segmentName, allAttributes, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected number of attributes retrieved via getAttributes() after recovery for segment " + segmentName, allAttributes.size(), allAttributeValues.size());
        // Verify all attribute values. Core attributes should still be loaded in memory, while extended attributes can
        // only be fetched via their special API.
        SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        for (val attributeId : allAttributes) {
            Assert.assertEquals("Unexpected value for attribute " + attributeId + " via getAttributes() after recovery for segment " + segmentName, expectedAttributeValue, (long) allAttributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            if (Attributes.isCoreAttribute(attributeId)) {
                Assert.assertEquals("Expecting core attribute to be loaded in memory.", expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            } else {
                Assert.assertEquals("Not expecting extended attribute to be loaded in memory.", Attributes.NULL_ATTRIBUTE_VALUE, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            }
        }
        // Now instruct the Container to cache missing values (do it a few times so we make sure it's idempotent).
        // Also introduce some random new attribute to fetch. We want to make sure we can properly handle caching
        // missing attribute values.
        val missingAttributeId = isUUIDOnly.test(sn) ? AttributeId.randomUUID() : AttributeId.random(variableAttributeIdLength);
        val attributesToCache = new ArrayList<>(allAttributes);
        attributesToCache.add(missingAttributeId);
        val attributesToCacheValues = new HashMap<>(allAttributeValues);
        attributesToCacheValues.put(missingAttributeId, Attributes.NULL_ATTRIBUTE_VALUE);
        Map<AttributeId, Long> allAttributeValuesWithCache;
        for (int i = 0; i < 2; i++) {
            allAttributeValuesWithCache = localContainer.getAttributes(segmentName, attributesToCache, true, TIMEOUT).join();
            AssertExtensions.assertMapEquals("Inconsistent results from getAttributes(cache=true, attempt=" + i + ").", attributesToCacheValues, allAttributeValuesWithCache);
            sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).join();
            for (val attributeId : allAttributes) {
                Assert.assertEquals("Expecting all attributes to be loaded in memory.", expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
            }
            Assert.assertEquals("Unexpected value for missing Attribute Id", Attributes.NULL_ATTRIBUTE_VALUE, (long) sp.getAttributes().get(missingAttributeId));
        }
    }
    // 4. Make an update, then immediately seal the segment, then verify the update updated the root pointer.
    AttributeId attr = Attributes.ATTRIBUTE_SEGMENT_ROOT_POINTER;
    val oldRootPointers = new HashMap<String, Long>();
    for (val sn : segmentNames.entrySet()) {
        val segmentName = sn.getKey();
        val newAttributeId = isUUIDOnly.test(sn) ? AttributeId.randomUUID() : AttributeId.random(variableAttributeIdLength);
        // Get the old root pointer, then make a random attribute update, then immediately seal the segment.
        localContainer.getAttributes(segmentName, Collections.singleton(attr), false, TIMEOUT).thenCompose(values -> {
            oldRootPointers.put(segmentName, values.get(attr));
            return CompletableFuture.allOf(localContainer.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(newAttributeId, AttributeUpdateType.Replace, 1L)), TIMEOUT), localContainer.sealStreamSegment(segmentName, TIMEOUT));
        }).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    }
    // which indicates the StorageWriter was able to successfully record it after its final Attribute Index update.
    for (String segmentName : segmentNames.keySet()) {
        Long oldValue = oldRootPointers.get(segmentName);
        TestUtils.await(() -> {
            val newVal = localContainer.getAttributes(segmentName, Collections.singleton(attr), false, TIMEOUT).join().get(attr);
            return oldValue < newVal;
        }, 10, TIMEOUT.toMillis());
    }
    waitForSegmentsInStorage(segmentNames.keySet(), localContainer, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    localContainer.stopAsync().awaitTerminated();
}

Example 14 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class DebugStreamSegmentContainerTests method testDataRecoveryContainerLevel.

/**
 * The test creates a container. Using it, some segments are created, data is written to them and their attributes are
 * updated. After Tier1 is flushed to the storage, the container is closed. From the storage, we recover the
 * segments and update their attributes. The data and attributes for each of the segment are verified to validate the
 * recovery process.
 */
@Test
public void testDataRecoveryContainerLevel() throws Exception {
    int attributesUpdatesPerSegment = 50;
    int segmentsCount = 10;
    final AttributeId attributeReplace = AttributeId.uuid(CORE_ATTRIBUTE_ID_PREFIX, RANDOM.nextInt(10));
    final long expectedAttributeValue = attributesUpdatesPerSegment;
    int containerId = 0;
    int containerCount = 1;
    // 1MB
    int maxDataSize = 1024 * 1024;
    StorageFactory storageFactory = new InMemoryStorageFactory(executorService());
    @Cleanup TestContext context = createContext(executorService());
    OperationLogFactory localDurableLogFactory = new DurableLogFactory(DEFAULT_DURABLE_LOG_CONFIG, context.dataLogFactory, executorService());
    @Cleanup MetadataCleanupContainer container = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, storageFactory, context.getDefaultExtensions(), executorService());
    container.startAsync().awaitRunning();
    // 1. Create segments.
    ArrayList<String> segmentNames = new ArrayList<>();
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (int i = 0; i < segmentsCount; i++) {
        String segmentName = getSegmentName(i);
        segmentNames.add(segmentName);
        opFutures.add(container.createStreamSegment(segmentName, getSegmentType(segmentName), null, TIMEOUT));
    }
    // 1.1 Wait for all segments to be created prior to using them.
    Futures.allOf(opFutures).join();
    opFutures.clear();
    // 2. Write data and update some of the attributes.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArraySegment> segmentContents = new HashMap<>();
    for (String segmentName : segmentNames) {
        val dataSize = RANDOM.nextInt(maxDataSize);
        byte[] writeData = populate(dataSize);
        val appendData = new ByteArraySegment(writeData);
        val append = container.append(segmentName, appendData, null, TIMEOUT);
        opFutures.add(Futures.toVoid(append));
        lengths.put(segmentName, (long) dataSize);
        segmentContents.put(segmentName, appendData);
        for (int i = 0; i < attributesUpdatesPerSegment; i++) {
            AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            opFutures.add(container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Instead of waiting for the Writer to move data to Storage, we invoke the flushToStorage to verify that all
    // operations have been applied to Storage.
    val forceFlush = container.flushToStorage(TIMEOUT);
    forceFlush.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    container.close();
    // Get the storage instance
    @Cleanup Storage storage = storageFactory.createStorageAdapter();
    // 4. Move container metadata and its attribute segment to back up segments.
    Map<Integer, String> backUpMetadataSegments = ContainerRecoveryUtils.createBackUpMetadataSegments(storage, containerCount, executorService(), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    OperationLogFactory localDurableLogFactory2 = new DurableLogFactory(NO_TRUNCATIONS_DURABLE_LOG_CONFIG, new InMemoryDurableDataLogFactory(MAX_DATA_LOG_APPEND_SIZE, executorService()), executorService());
    // Starts a DebugSegmentContainer with new Durable Log
    @Cleanup TestContext context1 = createContext(executorService());
    @Cleanup MetadataCleanupContainer container2 = new MetadataCleanupContainer(containerId, CONTAINER_CONFIG, localDurableLogFactory2, context1.readIndexFactory, context1.attributeIndexFactory, context1.writerFactory, storageFactory, context1.getDefaultExtensions(), executorService());
    container2.startAsync().awaitRunning();
    Map<Integer, DebugStreamSegmentContainer> debugStreamSegmentContainersMap = new HashMap<>();
    debugStreamSegmentContainersMap.put(containerId, container2);
    // 4. Recover all segments.
    recoverAllSegments(storage, debugStreamSegmentContainersMap, executorService(), TIMEOUT);
    // 5. Update core attributes using back up segments.
    updateCoreAttributes(backUpMetadataSegments, debugStreamSegmentContainersMap, executorService(), TIMEOUT);
    // 6. Verify Segment Data.
    for (val sc : segmentContents.entrySet()) {
        // Contents.
        byte[] expectedData = sc.getValue().array();
        byte[] actualData = new byte[expectedData.length];
        container2.read(sc.getKey(), 0, actualData.length, TIMEOUT).join().readRemaining(actualData, TIMEOUT);
        Assert.assertArrayEquals("Unexpected contents for " + sc.getKey(), expectedData, actualData);
        // Length.
        val si = container2.getStreamSegmentInfo(sc.getKey(), TIMEOUT).join();
        Assert.assertEquals("Unexpected length for " + sc.getKey(), expectedData.length, si.getLength());
        // Attributes.
        val attributes = container2.getAttributes(sc.getKey(), Collections.singleton(attributeReplace), false, TIMEOUT).join();
        Assert.assertEquals("Unexpected attribute for " + sc.getKey(), expectedAttributeValue, (long) attributes.get(attributeReplace));
    }
}

Example 15 with AttributeUpdateCollection

use of io.pravega.segmentstore.contracts.AttributeUpdateCollection in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testWithBadAttributes.

private void testWithBadAttributes(Function<AttributeUpdateCollection, Operation> createOperation, Consumer<Operation> checkAfterSuccess, Consumer<Operation> checkAfterRejection) throws Exception {
    final AttributeId attributeNoUpdate = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfGreater = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfEquals = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfEqualsNullValue = AttributeId.randomUUID();
    if (checkAfterSuccess == null) {
        checkAfterSuccess = TestUtils::doNothing;
    }
    if (checkAfterRejection == null) {
        checkAfterRejection = TestUtils::doNothing;
    }
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    BiFunction<Throwable, Boolean, Boolean> exceptionChecker = (ex, expectNoPreviousValue) -> (ex instanceof BadAttributeUpdateException) && ((BadAttributeUpdateException) ex).isPreviousValueMissing() == expectNoPreviousValue;
    // Values not set
    AttributeUpdateCollection attributeUpdates = new AttributeUpdateCollection();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.ReplaceIfEquals, 0, 0));
    val op1 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to CAS-update an attribute with no previous value.", () -> txn.preProcessOperation(op1), ex -> exceptionChecker.apply(ex, true));
    checkAfterRejection.accept(op1);
    // Append #1.
    attributeUpdates.clear();
    // Initial add, so it's ok.
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    // Initial Add.
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.Replace, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEqualsNullValue, AttributeUpdateType.None, Attributes.NULL_ATTRIBUTE_VALUE));
    val expectedValues = attributeUpdates.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue));
    expectedValues.put(Attributes.ATTRIBUTE_SEGMENT_TYPE, DEFAULT_TYPE.getValue());
    Operation op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    checkAfterSuccess.accept(op);
    // ReplaceIfEquals fails when the current attribute value is NULL_ATTRIBUTE_VALUE (this should not set the IsPreviousValueMissing flag).
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEqualsNullValue, AttributeUpdateType.ReplaceIfEquals, 1, 1));
    val op2 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to CAS-update an attribute with no previous value.", () -> txn.preProcessOperation(op2), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op2);
    // Append #2: Try to update attribute that cannot be updated.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 3));
    val op3 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an unmodifiable attribute.", () -> txn.preProcessOperation(op3), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op2);
    // Append #3: Try to update attribute with bad value for ReplaceIfGreater attribute.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 1));
    val op4 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an attribute with the wrong value for ReplaceIfGreater.", () -> txn.preProcessOperation(op4), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op4);
    // Append #4: Try to update attribute with bad value for ReplaceIfEquals attribute.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, 3, 3));
    val op5 = createOperation.apply(attributeUpdates);
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an attribute with the wrong comparison value for ReplaceIfGreater.", () -> txn.preProcessOperation(op5), ex -> exceptionChecker.apply(ex, false));
    checkAfterRejection.accept(op5);
    // Reset the attribute update list to its original state so we can do the final verification.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, 2, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEqualsNullValue, AttributeUpdateType.None, Attributes.NULL_ATTRIBUTE_VALUE));
    verifyAttributeUpdates("after rejected operations", txn, attributeUpdates, expectedValues);
    txn.commit(metadata);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in segment metadata after commit.", expectedValues, metadata.getStreamSegmentMetadata(SEGMENT_ID));
}