Examples with AttributeUpdate io.pravega.segmentstore.contracts.AttributeUpdate used on opensource projects

Example 61 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentContainerTests method mergeTransactions.

private ArrayList<CompletableFuture<Void>> mergeTransactions(HashMap<String, ArrayList<String>> transactionsBySegment, HashMap<String, Long> lengths, HashMap<String, ByteArrayOutputStream> segmentContents, TestContext context, boolean conditionalMerge) throws Exception {
    ArrayList<CompletableFuture<Void>> mergeFutures = new ArrayList<>();
    int i = 0;
    for (Map.Entry<String, ArrayList<String>> e : transactionsBySegment.entrySet()) {
        String parentName = e.getKey();
        for (String transactionName : e.getValue()) {
            if (++i % 2 == 0) {
                // Every other Merge operation, pre-seal the source. We want to verify we correctly handle this situation as well.
                mergeFutures.add(Futures.toVoid(context.container.sealStreamSegment(transactionName, TIMEOUT)));
            }
            // Use both calls, with and without attribute updates for mergeSegments.
            if (conditionalMerge) {
                AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.fromUUID(UUID.nameUUIDFromBytes(transactionName.getBytes())), AttributeUpdateType.ReplaceIfEquals, transactionName.hashCode() + 1, transactionName.hashCode()));
                mergeFutures.add(Futures.toVoid(context.container.mergeStreamSegment(parentName, transactionName, attributeUpdates, TIMEOUT)));
            } else {
                mergeFutures.add(Futures.toVoid(context.container.mergeStreamSegment(parentName, transactionName, TIMEOUT)));
            }
            // Update parent length.
            lengths.put(parentName, lengths.get(parentName) + lengths.get(transactionName));
            lengths.remove(transactionName);
            // Update parent contents.
            segmentContents.get(parentName).write(segmentContents.get(transactionName).toByteArray());
            segmentContents.remove(transactionName);
        }
    }
    return mergeFutures;
}

Example 62 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testWithAttributesByReference.

private void testWithAttributesByReference(Function<AttributeUpdateCollection, Operation> createOperation) throws Exception {
    final AttributeId referenceAttributeId = AttributeId.randomUUID();
    final AttributeId attributeSegmentLength = AttributeId.randomUUID();
    final long initialAttributeValue = 1234567;
    UpdateableContainerMetadata metadata = createMetadata();
    metadata.getStreamSegmentMetadata(SEGMENT_ID).updateAttributes(ImmutableMap.of(referenceAttributeId, initialAttributeValue));
    val txn = createUpdateTransaction(metadata);
    // Update #1.
    AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(referenceAttributeId, AttributeUpdateType.Accumulate, 2), new DynamicAttributeUpdate(attributeSegmentLength, AttributeUpdateType.None, DynamicAttributeValue.segmentLength(5)));
    Map<AttributeId, Long> expectedValues = ImmutableMap.of(Attributes.ATTRIBUTE_SEGMENT_TYPE, DEFAULT_TYPE.getValue(), referenceAttributeId, initialAttributeValue + 2, attributeSegmentLength, SEGMENT_LENGTH + 5);
    Operation op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    // Verify result.
    verifyAttributeUpdates("after acceptOperation", txn, attributeUpdates, expectedValues);
    txn.commit(metadata);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in segment metadata after final commit.", expectedValues, metadata.getStreamSegmentMetadata(SEGMENT_ID));
}

Example 63 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentContainerTests method testSegmentRegularOperations.

/**
 * Tests the createSegment, append, updateAttributes, read, getSegmentInfo, getActiveSegments.
 */
@Test
public void testSegmentRegularOperations() throws Exception {
    final AttributeId attributeAccumulate = AttributeId.randomUUID();
    final AttributeId attributeReplace = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfGreater = AttributeId.randomUUID();
    final AttributeId attributeReplaceIfEquals = AttributeId.randomUUID();
    final AttributeId attributeNoUpdate = AttributeId.randomUUID();
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    checkActiveSegments(context.container, 0);
    activateAllSegments(segmentNames, context);
    checkActiveSegments(context.container, segmentNames.size());
    // 2. Add some appends.
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    ArrayList<RefCountByteArraySegment> appends = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
        for (String segmentName : segmentNames) {
            val attributeUpdates = new AttributeUpdateCollection();
            attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, i == 0 ? AttributeUpdateType.Replace : AttributeUpdateType.ReplaceIfEquals, i + 1, i));
            RefCountByteArraySegment appendData = getAppendData(segmentName, i);
            long expectedLength = lengths.getOrDefault(segmentName, 0L) + appendData.getLength();
            val append = (i % 2 == 0) ? context.container.append(segmentName, appendData, attributeUpdates, TIMEOUT) : context.container.append(segmentName, lengths.get(segmentName), appendData, attributeUpdates, TIMEOUT);
            opFutures.add(append.thenApply(length -> {
                assertEquals(expectedLength, length.longValue());
                return null;
            }));
            lengths.put(segmentName, expectedLength);
            recordAppend(segmentName, appendData, segmentContents, appends);
        }
    }
    // 2.1 Update some of the attributes.
    for (String segmentName : segmentNames) {
        // Record a one-off update.
        opFutures.add(context.container.updateAttributes(segmentName, AttributeUpdateCollection.from(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, expectedAttributeValue)), TIMEOUT));
        for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
            val attributeUpdates = new AttributeUpdateCollection();
            attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, APPENDS_PER_SEGMENT + i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, APPENDS_PER_SEGMENT + i + 1));
            attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, APPENDS_PER_SEGMENT + i + 1, APPENDS_PER_SEGMENT + i));
            opFutures.add(context.container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. getSegmentInfo
    for (String segmentName : segmentNames) {
        SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        long expectedLength = lengths.get(segmentName);
        Assert.assertEquals("Unexpected StartOffset for non-truncated segment " + segmentName, 0, sp.getStartOffset());
        Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength, sp.getLength());
        Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
        Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted());
        // Verify all attribute values.
        Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeNoUpdate, Attributes.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeAccumulate, Attributes.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplace + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplace, Attributes.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplaceIfGreater + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplaceIfGreater, Attributes.NULL_ATTRIBUTE_VALUE));
        Assert.assertEquals("Unexpected value for attribute " + attributeReplaceIfEquals + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeReplaceIfEquals, Attributes.NULL_ATTRIBUTE_VALUE));
        val expectedType = getSegmentType(segmentName);
        val actualType = SegmentType.fromAttributes(sp.getAttributes());
        Assert.assertEquals("Unexpected Segment Type.", expectedType, actualType);
    }
    checkActiveSegments(context.container, segmentNames.size());
    // 4. Reads (regular reads, not tail reads).
    checkReadIndex(segmentContents, lengths, context);
    // 4.1. After we ensured that all data has been ingested and processed, verify that all data buffers have been released.
    checkAppendLeaks(appends);
    // 5. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}

Example 64 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentContainerTests method testConditionalTransactionOperations.

/**
 * Test the createTransaction, append-to-Transaction, mergeTransaction methods with attribute updates.
 */
@Test
public void testConditionalTransactionOperations() throws Exception {
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    HashMap<String, ArrayList<String>> transactionsBySegment = createTransactions(segmentNames, context);
    activateAllSegments(segmentNames, context);
    transactionsBySegment.values().forEach(s -> activateAllSegments(s, context));
    // 2. Add some appends.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    appendToParentsAndTransactions(segmentNames, transactionsBySegment, lengths, segmentContents, context);
    // 3. Correctly update attribute on parent Segments. Each source Segment will be initialized with a value and
    // after the merge, that value should have been updated.
    ArrayList<CompletableFuture<Void>> opFutures = new ArrayList<>();
    for (Map.Entry<String, ArrayList<String>> e : transactionsBySegment.entrySet()) {
        String parentName = e.getKey();
        for (String transactionName : e.getValue()) {
            opFutures.add(context.container.updateAttributes(parentName, AttributeUpdateCollection.from(new AttributeUpdate(AttributeId.fromUUID(UUID.nameUUIDFromBytes(transactionName.getBytes())), AttributeUpdateType.None, transactionName.hashCode())), TIMEOUT));
        }
    }
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 4. Merge all the Transactions. Now this should work.
    Futures.allOf(mergeTransactions(transactionsBySegment, lengths, segmentContents, context, true)).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 5. Add more appends (to the parent segments)
    ArrayList<CompletableFuture<Long>> appendFutures = new ArrayList<>();
    HashMap<String, CompletableFuture<Map<AttributeId, Long>>> getAttributeFutures = new HashMap<>();
    for (int i = 0; i < 5; i++) {
        for (String segmentName : segmentNames) {
            RefCountByteArraySegment appendData = getAppendData(segmentName, APPENDS_PER_SEGMENT + i);
            appendFutures.add(context.container.append(segmentName, appendData, null, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.getLength());
            recordAppend(segmentName, appendData, segmentContents, null);
            // Verify that we can no longer append to Transaction.
            for (String transactionName : transactionsBySegment.get(segmentName)) {
                AssertExtensions.assertThrows("An append was allowed to a merged Transaction " + transactionName, context.container.append(transactionName, new ByteArraySegment("foo".getBytes()), null, TIMEOUT)::join, ex -> ex instanceof StreamSegmentMergedException || ex instanceof StreamSegmentNotExistsException);
                getAttributeFutures.put(transactionName, context.container.getAttributes(segmentName, Collections.singletonList(AttributeId.fromUUID(UUID.nameUUIDFromBytes(transactionName.getBytes()))), true, TIMEOUT));
            }
        }
    }
    Futures.allOf(appendFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    Futures.allOf(getAttributeFutures.values()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 6. Verify their contents.
    checkReadIndex(segmentContents, lengths, context);
    // 7. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    // 8. Verify that the parent Segment contains the expected attributes updated.
    for (Map.Entry<String, CompletableFuture<Map<AttributeId, Long>>> transactionAndAttribute : getAttributeFutures.entrySet()) {
        Map<AttributeId, Long> segmentAttributeUpdated = transactionAndAttribute.getValue().join();
        AttributeId transactionAttributeId = AttributeId.fromUUID(UUID.nameUUIDFromBytes(transactionAndAttribute.getKey().getBytes()));
        // Conditional merges in mergeTransactions() update the attribute value to adding 1.
        Assert.assertEquals(transactionAndAttribute.getKey().hashCode() + 1, segmentAttributeUpdated.get(transactionAttributeId).longValue());
    }
    context.container.stopAsync().awaitTerminated();
}

Example 65 with AttributeUpdate

use of io.pravega.segmentstore.contracts.AttributeUpdate in project pravega by pravega.

the class StreamSegmentContainerTests method testConcurrentSegmentActivation.

/**
 * Tests the ability for the StreamSegmentContainer to handle concurrent actions on a Segment that it does not know
 * anything about, and handling the resulting concurrency.
 * Note: this is tested with a single segment. It could be tested with multiple segments, but different segments
 * are mostly independent of each other, so we would not be gaining much by doing so.
 */
@Test
public void testConcurrentSegmentActivation() throws Exception {
    final AttributeId attributeAccumulate = AttributeId.randomUUID();
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    final int appendLength = 10;
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    String segmentName = createSegments(context).get(0);
    // 2. Add some appends.
    List<CompletableFuture<Void>> opFutures = Collections.synchronizedList(new ArrayList<>());
    AtomicLong expectedLength = new AtomicLong();
    @Cleanup("shutdown") ExecutorService testExecutor = newScheduledThreadPool(Math.min(20, APPENDS_PER_SEGMENT), "testConcurrentSegmentActivation");
    val submitFutures = new ArrayList<Future<?>>();
    for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
        final byte fillValue = (byte) i;
        submitFutures.add(testExecutor.submit(() -> {
            val attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            byte[] appendData = new byte[appendLength];
            Arrays.fill(appendData, (byte) (fillValue + 1));
            opFutures.add(Futures.toVoid(context.container.append(segmentName, new ByteArraySegment(appendData), attributeUpdates, TIMEOUT)));
            expectedLength.addAndGet(appendData.length);
        }));
    }
    // 2.1 Update the attribute.
    for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
        submitFutures.add(testExecutor.submit(() -> {
            AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            opFutures.add(context.container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }));
    }
    // Wait for the submittal of tasks to complete.
    submitFutures.forEach(this::await);
    // Now wait for all the appends to finish.
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. getSegmentInfo: verify final state of the attribute.
    SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, TIMEOUT).join();
    Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength.get(), sp.getLength());
    Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
    Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted());
    Assert.assertEquals("Unexpected Segment Type.", getSegmentType(segmentName), SegmentType.fromAttributes(sp.getAttributes()));
    // Verify all attribute values.
    Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeAccumulate, Attributes.NULL_ATTRIBUTE_VALUE));
    checkActiveSegments(context.container, 1);
    // 4. Written data.
    waitForOperationsInReadIndex(context.container);
    byte[] actualData = new byte[(int) expectedLength.get()];
    int offset = 0;
    @Cleanup ReadResult readResult = context.container.read(segmentName, 0, actualData.length, TIMEOUT).join();
    while (readResult.hasNext()) {
        ReadResultEntry readEntry = readResult.next();
        BufferView readEntryContents = readEntry.getContent().join();
        AssertExtensions.assertLessThanOrEqual("Too much to read.", actualData.length, offset + actualData.length);
        readEntryContents.copyTo(ByteBuffer.wrap(actualData, offset, actualData.length));
        offset += actualData.length;
    }
    Assert.assertEquals("Unexpected number of bytes read.", actualData.length, offset);
    Assert.assertTrue("Unexpected number of bytes read (multiple of appendLength).", actualData.length % appendLength == 0);
    boolean[] observedValues = new boolean[APPENDS_PER_SEGMENT + 1];
    for (int i = 0; i < actualData.length; i += appendLength) {
        byte value = actualData[i];
        Assert.assertFalse("Append with value " + value + " was written multiple times.", observedValues[value]);
        observedValues[value] = true;
        for (int j = 1; j < appendLength; j++) {
            Assert.assertEquals("Append was not written atomically at offset " + (i + j), value, actualData[i + j]);
        }
    }
    // Verify all the appends made it (we purposefully did not write 0, since that's the default fill value in an array).
    Assert.assertFalse("Not expecting 0 as a value.", observedValues[0]);
    for (int i = 1; i < observedValues.length; i++) {
        Assert.assertTrue("Append with value " + i + " was not written.", observedValues[i]);
    }
    context.container.stopAsync().awaitTerminated();
}