Examples with AttributeUpdate io.pravega.segmentstore.contracts.AttributeUpdate used on opensource projects

Example 1 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentMapperTests method testCreateAlreadyExists.

/**
 * General test for verifying behavior when a Segment/Transaction is attempted to be created but it already exists.
 *
 * @param segmentName   The name of the segment/transaction to create.
 * @param createSegment A BiFunction that is given an instance of a StreamSegmentMapper and a Collection of AttributeUpdates
 *                      that, when invoked, will create the given segment.
 */
private void testCreateAlreadyExists(String segmentName, BiFunction<StreamSegmentMapper, Collection<AttributeUpdate>, CompletableFuture<?>> createSegment) {
    final String stateSegmentName = StreamSegmentNameUtils.getStateSegmentName(segmentName);
    final Map<UUID, Long> correctAttributes = Collections.singletonMap(UUID.randomUUID(), 123L);
    final Collection<AttributeUpdate> correctAttributeUpdates = correctAttributes.entrySet().stream().map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())).collect(Collectors.toList());
    final Map<UUID, Long> badAttributes = Collections.singletonMap(UUID.randomUUID(), 456L);
    final Collection<AttributeUpdate> badAttributeUpdates = badAttributes.entrySet().stream().map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())).collect(Collectors.toList());
    @Cleanup TestContext context = new TestContext();
    @Cleanup val storage = InMemoryStorageFactory.newStorage(executorService());
    storage.initialize(1);
    val store = new SegmentStateStore(storage, executorService());
    val mapper = new StreamSegmentMapper(context.metadata, context.operationLog, store, context.noOpMetadataCleanup, storage, executorService());
    // 1. Segment Exists, and so does State File (and it's not corrupted) -> Exception must be bubbled up.
    createSegment.apply(mapper, correctAttributeUpdates).join();
    AssertExtensions.assertThrows("createNewStreamSegment did not fail when Segment already exists.", () -> createSegment.apply(mapper, badAttributeUpdates), ex -> ex instanceof StreamSegmentExistsException);
    val state1 = store.get(segmentName, TIMEOUT).join();
    AssertExtensions.assertMapEquals("Unexpected attributes after failed attempt to recreate correctly created segment", correctAttributes, state1.getAttributes());
    // 2. Segment Exists, but with empty State File: State file re-created & no exception bubbled up.
    storage.openWrite(stateSegmentName).thenCompose(handle -> storage.delete(handle, TIMEOUT)).thenCompose(v -> storage.create(stateSegmentName, TIMEOUT)).join();
    Assert.assertNull("Expected a null SegmentState.", store.get(segmentName, TIMEOUT).join());
    createSegment.apply(mapper, correctAttributeUpdates).join();
    val state2 = store.get(segmentName, TIMEOUT).join();
    AssertExtensions.assertMapEquals("Unexpected attributes after successful attempt to complete segment creation (missing state file)", correctAttributes, state2.getAttributes());
    // 3. Segment Exists, but with corrupted State File: State file re-created & no exception bubbled up.
    storage.openWrite(stateSegmentName).thenCompose(handle -> storage.delete(handle, TIMEOUT)).thenCompose(v -> storage.create(stateSegmentName, TIMEOUT)).thenCompose(v -> storage.openWrite(stateSegmentName)).thenCompose(handle -> storage.write(handle, 0, new ByteArrayInputStream(new byte[1]), 1, TIMEOUT)).join();
    AssertExtensions.assertThrows("Expected a DataCorruptionException when reading a corrupted State File.", () -> store.get(segmentName, TIMEOUT), ex -> ex instanceof DataCorruptionException);
    createSegment.apply(mapper, correctAttributeUpdates).join();
    val state3 = store.get(segmentName, TIMEOUT).join();
    AssertExtensions.assertMapEquals("Unexpected attributes after successful attempt to complete segment creation (corrupted state file)", correctAttributes, state3.getAttributes());
    // 4. Segment Exists with non-zero length, but with empty/corrupted State File: State File re-created and exception thrown.
    storage.openWrite(stateSegmentName).thenCompose(handle -> storage.delete(handle, TIMEOUT)).thenCompose(v -> storage.create(stateSegmentName, TIMEOUT)).thenCompose(v -> storage.openWrite(segmentName)).thenCompose(handle -> storage.write(handle, 0, new ByteArrayInputStream(new byte[1]), 1, TIMEOUT)).join();
    AssertExtensions.assertThrows("createNewStreamSegment did not fail when Segment already exists (non-zero length, missing state file).", () -> createSegment.apply(mapper, correctAttributeUpdates), ex -> ex instanceof StreamSegmentExistsException);
    val state4 = store.get(segmentName, TIMEOUT).join();
    AssertExtensions.assertMapEquals("Unexpected attributes after failed attempt to recreate segment with non-zero length", correctAttributes, state4.getAttributes());
}

Example 2 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentMapperTests method testCreateNewStreamSegment.

/**
 * Tests the ability of the StreamSegmentMapper to create a new StreamSegment.
 */
@Test
public void testCreateNewStreamSegment() {
    final int segmentCount = 10;
    final int transactionsPerSegment = 5;
    @Cleanup TestContext context = new TestContext();
    HashMap<String, SegmentRollingPolicy> storageSegments = new HashMap<>();
    HashMap<String, Integer> expectedRolloverSizes = new HashMap<>();
    setupStorageCreateHandler(context, storageSegments);
    setupStorageGetHandler(context, storageSegments.keySet(), segmentName -> StreamSegmentInformation.builder().name(segmentName).build());
    // Create some Segments and Transaction and verify they are properly created and registered.
    for (int i = 0; i < segmentCount; i++) {
        String segmentName = getName(i);
        val segmentAttributes = createAttributes(ATTRIBUTE_COUNT);
        if (i % 2 == 0) {
            segmentAttributes.add(new AttributeUpdate(Attributes.ROLLOVER_SIZE, AttributeUpdateType.Replace, i + 100));
            expectedRolloverSizes.put(segmentName, i + 100);
        }
        context.mapper.createNewStreamSegment(segmentName, segmentAttributes, TIMEOUT).join();
        assertSegmentCreated(segmentName, segmentAttributes, context);
        for (int j = 0; j < transactionsPerSegment; j++) {
            val transactionAttributes = createAttributes(ATTRIBUTE_COUNT);
            int rolloverSize = j % 2 == 0 ? j + 100 : -1;
            if (rolloverSize >= 0) {
                transactionAttributes.add(new AttributeUpdate(Attributes.ROLLOVER_SIZE, AttributeUpdateType.Replace, rolloverSize));
            }
            String transactionName = context.mapper.createNewTransactionStreamSegment(segmentName, UUID.randomUUID(), transactionAttributes, TIMEOUT).join();
            if (rolloverSize >= 0) {
                expectedRolloverSizes.put(transactionName, rolloverSize);
            }
            assertSegmentCreated(transactionName, transactionAttributes, context);
        }
    }
    for (val e : storageSegments.entrySet()) {
        if (e.getValue() == null) {
            Assert.assertFalse("Segment was expected to have a rollover policy defined.", expectedRolloverSizes.containsKey(e.getKey()));
        } else {
            long expectedValue = expectedRolloverSizes.containsKey(e.getKey()) ? expectedRolloverSizes.get(e.getKey()) : SegmentRollingPolicy.NO_ROLLING.getMaxLength();
            Assert.assertEquals("Unexpected rollover policy set.", expectedValue, e.getValue().getMaxLength());
        }
    }
}

Example 3 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentMapperTests method createAttributes.

private Collection<AttributeUpdate> createAttributes(int count) {
    Collection<AttributeUpdate> result = new ArrayList<>(count);
    for (int i = 0; i < count; i++) {
        AttributeUpdateType ut = AttributeUpdateType.values()[i % AttributeUpdateType.values().length];
        result.add(new AttributeUpdate(UUID.randomUUID(), ut, i, i));
    }
    return result;
}

Example 4 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testMaxAttributeLimit.

/**
 * Tests the ability of the ContainerMetadataUpdateTransaction to enforce the maximum attribute limit on Segments.
 */
@Test
public void testMaxAttributeLimit() throws Exception {
    // We check all operations that can update attributes.
    val ops = new HashMap<String, Function<Collection<AttributeUpdate>, Operation>>();
    ops.put("UpdateAttributes", u -> new UpdateAttributesOperation(SEGMENT_ID, u));
    ops.put("Append", u -> new StreamSegmentAppendOperation(SEGMENT_ID, DEFAULT_APPEND_DATA, u));
    // Set the maximum allowed number of attributes on a segment.
    UpdateableContainerMetadata metadata = createMetadata();
    val initialUpdates = new ArrayList<AttributeUpdate>(SegmentMetadata.MAXIMUM_ATTRIBUTE_COUNT);
    val expectedValues = new HashMap<UUID, Long>();
    for (int i = 0; i < SegmentMetadata.MAXIMUM_ATTRIBUTE_COUNT; i++) {
        UUID attributeId;
        do {
            attributeId = UUID.randomUUID();
        } while (expectedValues.containsKey(attributeId));
        initialUpdates.add(new AttributeUpdate(attributeId, AttributeUpdateType.None, i));
        expectedValues.put(attributeId, (long) i);
    }
    // And load them up into an UpdateTransaction.
    val txn = createUpdateTransaction(metadata);
    val initialOp = new UpdateAttributesOperation(SEGMENT_ID, initialUpdates);
    txn.preProcessOperation(initialOp);
    txn.acceptOperation(initialOp);
    // invokes preProcessOperation() - which is responsible with validation, so no changes are made to the UpdateTransaction.
    for (val opGenerator : ops.entrySet()) {
        // Value replacement.
        val replacementUpdates = new ArrayList<AttributeUpdate>();
        int i = 0;
        for (val e : expectedValues.entrySet()) {
            AttributeUpdate u;
            switch((i++) % 4) {
                case 0:
                    u = new AttributeUpdate(e.getKey(), AttributeUpdateType.ReplaceIfEquals, e.getValue() + 1, e.getValue());
                    break;
                case 1:
                    u = new AttributeUpdate(e.getKey(), AttributeUpdateType.ReplaceIfGreater, e.getValue() + 1);
                    break;
                case 2:
                    u = new AttributeUpdate(e.getKey(), AttributeUpdateType.Accumulate, 1);
                    break;
                default:
                    u = new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, 1);
                    break;
            }
            replacementUpdates.add(u);
        }
        // This should not throw anything.
        txn.preProcessOperation(opGenerator.getValue().apply(replacementUpdates));
        // Removal - this should not throw anything either.
        val toRemoveId = initialUpdates.get(0).getAttributeId();
        val toRemoveUpdate = new AttributeUpdate(toRemoveId, AttributeUpdateType.Replace, SegmentMetadata.NULL_ATTRIBUTE_VALUE);
        txn.preProcessOperation(opGenerator.getValue().apply(Collections.singleton(toRemoveUpdate)));
        // Addition - this should throw.
        UUID toAddId;
        do {
            toAddId = UUID.randomUUID();
        } while (expectedValues.containsKey(toAddId));
        val toAddUpdate = new AttributeUpdate(toAddId, AttributeUpdateType.None, 1);
        AssertExtensions.assertThrows("Too many attributes were accepted for operation " + opGenerator.getKey(), () -> txn.preProcessOperation(opGenerator.getValue().apply(Collections.singleton(toAddUpdate))), ex -> ex instanceof TooManyAttributesException);
        // Removal+Addition+Replacement: this particular setup should not throw anything.
        val mixedUpdates = Arrays.asList(new AttributeUpdate(toAddId, AttributeUpdateType.None, 1), new AttributeUpdate(toRemoveId, AttributeUpdateType.Replace, SegmentMetadata.NULL_ATTRIBUTE_VALUE), new AttributeUpdate(initialUpdates.get(1).getAttributeId(), AttributeUpdateType.Replace, 10));
        txn.preProcessOperation(opGenerator.getValue().apply(mixedUpdates));
    }
}

Example 5 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method verifyAttributeUpdates.

private void verifyAttributeUpdates(String stepName, ContainerMetadata containerMetadata, Collection<AttributeUpdate> attributeUpdates, Map<UUID, Long> expectedValues) {
    // Verify that the Attribute Updates have their expected values and that the updater has internalized the attribute updates.
    val transactionMetadata = containerMetadata.getStreamSegmentMetadata(SEGMENT_ID);
    val expectedTransactionAttributes = new HashMap<UUID, Long>(expectedValues);
    attributeUpdates.forEach(au -> expectedTransactionAttributes.put(au.getAttributeId(), au.getValue()));
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in transaction metadata " + stepName + ".", expectedTransactionAttributes, transactionMetadata);
    for (AttributeUpdate au : attributeUpdates) {
        Assert.assertEquals("Unexpected updated value for AttributeUpdate[" + au.getUpdateType() + "] " + stepName, (long) expectedValues.get(au.getAttributeId()), au.getValue());
    }
}