Examples with AttributeUpdate io.pravega.segmentstore.contracts.AttributeUpdate used on opensource projects

Example 16 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class OperationMetadataUpdaterTests method recordAppend.

private void recordAppend(long segmentId, int length, OperationMetadataUpdater updater, UpdateableContainerMetadata referenceMetadata) throws Exception {
    byte[] data = new byte[length];
    val attributeUpdates = Arrays.asList(new AttributeUpdate(Attributes.CREATION_TIME, AttributeUpdateType.Replace, NEXT_ATTRIBUTE_VALUE.get()), new AttributeUpdate(Attributes.EVENT_COUNT, AttributeUpdateType.Accumulate, NEXT_ATTRIBUTE_VALUE.get()));
    val op = new StreamSegmentAppendOperation(segmentId, data, attributeUpdates);
    process(op, updater);
    if (referenceMetadata != null) {
        val rsm = referenceMetadata.getStreamSegmentMetadata(segmentId);
        rsm.setLength(rsm.getLength() + length);
        val attributes = new HashMap<UUID, Long>();
        op.getAttributeUpdates().forEach(au -> attributes.put(au.getAttributeId(), au.getValue()));
        rsm.updateAttributes(attributes);
    }
}

Example 17 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class OperationComparer method assertSame.

private void assertSame(String message, Collection<AttributeUpdate> expected, Collection<AttributeUpdate> actual) {
    if (expected == null) {
        Assert.assertNull(message + " Not expecting attributes.", actual);
        return;
    } else {
        Assert.assertNotNull(message + " Expected attributes, but none found.", actual);
    }
    Assert.assertEquals(message + " Unexpected number of attributes.", expected.size(), actual.size());
    val expectedIndexed = expected.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue));
    for (AttributeUpdate au : actual) {
        Assert.assertTrue(message + " Found extra AttributeUpdate: " + au, expectedIndexed.containsKey(au.getAttributeId()));
        long expectedValue = expectedIndexed.get(au.getAttributeId());
        Assert.assertEquals(message + " Unexpected value for AttributeUpdate: " + au, expectedValue, au.getValue());
    }
}

Example 18 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentContainerTests method testConcurrentSegmentActivation.

/**
 * Tests the ability for the StreamSegmentContainer to handle concurrent actions on a Segment that it does not know
 * anything about, and handling the resulting concurrency.
 * Note: this is tested with a single segment. It could be tested with multiple segments, but different segments
 * are mostly independent of each other, so we would not be gaining much by doing so.
 */
@Test
public void testConcurrentSegmentActivation() throws Exception {
    final UUID attributeAccumulate = UUID.randomUUID();
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    final int appendLength = 10;
    @Cleanup TestContext context = new TestContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    String segmentName = createSegments(context).get(0);
    // 2. Add some appends.
    List<CompletableFuture<Void>> opFutures = Collections.synchronizedList(new ArrayList<>());
    AtomicLong expectedLength = new AtomicLong();
    @Cleanup("shutdown") ExecutorService testExecutor = newScheduledThreadPool(Math.min(20, APPENDS_PER_SEGMENT), "testConcurrentSegmentActivation");
    val submitFutures = new ArrayList<Future<?>>();
    for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
        final byte fillValue = (byte) i;
        submitFutures.add(testExecutor.submit(() -> {
            Collection<AttributeUpdate> attributeUpdates = Collections.singleton(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            byte[] appendData = new byte[appendLength];
            Arrays.fill(appendData, (byte) (fillValue + 1));
            opFutures.add(context.container.append(segmentName, appendData, attributeUpdates, TIMEOUT));
            expectedLength.addAndGet(appendData.length);
        }));
    }
    // 2.1 Update the attribute.
    for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
        submitFutures.add(testExecutor.submit(() -> {
            Collection<AttributeUpdate> attributeUpdates = new ArrayList<>();
            attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            opFutures.add(context.container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }));
    }
    // Wait for the submittal of tasks to complete.
    submitFutures.forEach(this::await);
    // Now wait for all the appends to finish.
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. getSegmentInfo: verify final state of the attribute.
    SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
    Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength.get(), sp.getLength());
    Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
    Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted());
    // Verify all attribute values.
    Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeAccumulate, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
    checkActiveSegments(context.container, 1);
    // 4. Written data.
    waitForOperationsInReadIndex(context.container);
    byte[] actualData = new byte[(int) expectedLength.get()];
    int offset = 0;
    @Cleanup ReadResult readResult = context.container.read(segmentName, 0, actualData.length, TIMEOUT).join();
    while (readResult.hasNext()) {
        ReadResultEntry readEntry = readResult.next();
        ReadResultEntryContents readEntryContents = readEntry.getContent().join();
        AssertExtensions.assertLessThanOrEqual("Too much to read.", actualData.length, offset + actualData.length);
        StreamHelpers.readAll(readEntryContents.getData(), actualData, offset, actualData.length);
        offset += actualData.length;
    }
    Assert.assertEquals("Unexpected number of bytes read.", actualData.length, offset);
    Assert.assertTrue("Unexpected number of bytes read (multiple of appendLength).", actualData.length % appendLength == 0);
    boolean[] observedValues = new boolean[APPENDS_PER_SEGMENT + 1];
    for (int i = 0; i < actualData.length; i += appendLength) {
        byte value = actualData[i];
        Assert.assertFalse("Append with value " + value + " was written multiple times.", observedValues[value]);
        observedValues[value] = true;
        for (int j = 1; j < appendLength; j++) {
            Assert.assertEquals("Append was not written atomically at offset " + (i + j), value, actualData[i + j]);
        }
    }
    // Verify all the appends made it (we purposefully did not write 0, since that's the default fill value in an array).
    Assert.assertFalse("Not expecting 0 as a value.", observedValues[0]);
    for (int i = 1; i < observedValues.length; i++) {
        Assert.assertTrue("Append with value " + i + " was not written.", observedValues[i]);
    }
    context.container.stopAsync().awaitTerminated();
}

Example 19 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentContainerTests method testSegmentRegularOperations.

/**
 * Tests the createSegment, append, updateAttributes, read, getSegmentInfo, getActiveSegments.
 */
@Test
public void testSegmentRegularOperations() throws Exception {
    final UUID attributeAccumulate = UUID.randomUUID();
    final UUID attributeReplace = UUID.randomUUID();
    final UUID attributeReplaceIfGreater = UUID.randomUUID();
    final UUID attributeReplaceIfEquals = UUID.randomUUID();
    final UUID attributeNoUpdate = UUID.randomUUID();
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    @Cleanup TestContext context = new TestContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    checkActiveSegments(context.container, 0);
    activateAllSegments(segmentNames, context);
    checkActiveSegments(context.container, segmentNames.size());
    // 2. Add some appends.
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
        for (String segmentName : segmentNames) {
            Collection<AttributeUpdate> attributeUpdates = new ArrayList<>();
            attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, i == 0 ? AttributeUpdateType.Replace : AttributeUpdateType.ReplaceIfEquals, i + 1, i));
            byte[] appendData = getAppendData(segmentName, i);
            opFutures.add(context.container.append(segmentName, appendData, attributeUpdates, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
            recordAppend(segmentName, appendData, segmentContents);
        }
    }
    // 2.1 Update some of the attributes.
    for (String segmentName : segmentNames) {
        // Record a one-off update.
        opFutures.add(context.container.updateAttributes(segmentName, Collections.singleton(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, expectedAttributeValue)), TIMEOUT));
        for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
            Collection<AttributeUpdate> attributeUpdates = new ArrayList<>();
            attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, APPENDS_PER_SEGMENT + i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, APPENDS_PER_SEGMENT + i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, APPENDS_PER_SEGMENT + i + 1, APPENDS_PER_SEGMENT + i));
            opFutures.add(context.container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. getSegmentInfo
    for (String segmentName : segmentNames) {
        SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        long expectedLength = lengths.get(segmentName);
        Assert.assertEquals("Unexpected StartOffset for non-truncated segment " + segmentName, 0, sp.getStartOffset());
        Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength, sp.getLength());
        Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
        Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted());
        // Verify all attribute values.
        Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeNoUpdate, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeAccumulate, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplace + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplace, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplaceIfGreater + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplaceIfGreater, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplaceIfEquals + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplaceIfEquals, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
    }
    checkActiveSegments(context.container, segmentNames.size());
    // 4. Reads (regular reads, not tail reads).
    checkReadIndex(segmentContents, lengths, context);
    // 5. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}

Example 20 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testWithBadAttributes.

private void testWithBadAttributes(Function<Collection<AttributeUpdate>, Operation> createOperation) throws Exception {
    final UUID attributeNoUpdate = UUID.randomUUID();
    final UUID attributeReplaceIfGreater = UUID.randomUUID();
    final UUID attributeReplaceIfEquals = UUID.randomUUID();
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    // Append #1.
    Collection<AttributeUpdate> attributeUpdates = new ArrayList<>();
    // Initial add, so it's ok.
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    // Initial Add.
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.Replace, 2));
    val expectedValues = attributeUpdates.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue));
    Operation op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    // Append #2: Try to update attribute that cannot be updated.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 3));
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an unmodifiable attribute.", () -> txn.preProcessOperation(createOperation.apply(attributeUpdates)), ex -> ex instanceof BadAttributeUpdateException);
    // Append #3: Try to update attribute with bad value for ReplaceIfGreater attribute.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 1));
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an attribute with the wrong value for ReplaceIfGreater.", () -> txn.preProcessOperation(createOperation.apply(attributeUpdates)), ex -> ex instanceof BadAttributeUpdateException);
    // Append #4: Try to update attribute with bad value for ReplaceIfEquals attribute.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, 3, 3));
    AssertExtensions.assertThrows("preProcessOperation accepted an operation that was trying to update an attribute with the wrong comparison value for ReplaceIfGreater.", () -> txn.preProcessOperation(createOperation.apply(attributeUpdates)), ex -> ex instanceof BadAttributeUpdateException);
    // Reset the attribute update list to its original state so we can do the final verification.
    attributeUpdates.clear();
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfGreater, 2, 2));
    verifyAttributeUpdates("after rejected operations", txn, attributeUpdates, expectedValues);
    txn.commit(metadata);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in segment metadata after commit.", expectedValues, metadata.getStreamSegmentMetadata(SEGMENT_ID));
}