Examples with AttributeId io.pravega.segmentstore.contracts.AttributeId used on opensource projects

Example 36 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class AppendProcessor method setupAppend.

/**
 * Setup an append so that subsequent append calls can occur.
 * This requires validating that the segment exists.
 * The reply: AppendSetup indicates that appends may proceed and contains the eventNumber which they should proceed
 * from (in the event that this is a reconnect from a producer we have seen before)
 */
@Override
public void setupAppend(SetupAppend setupAppend) {
    String newSegment = setupAppend.getSegment();
    UUID writer = setupAppend.getWriterId();
    log.info("Setting up appends for writer: {} on segment: {}", writer, newSegment);
    if (this.tokenVerifier != null) {
        try {
            JsonWebToken token = tokenVerifier.verifyToken(newSegment, setupAppend.getDelegationToken(), AuthHandler.Permissions.READ_UPDATE);
            setupTokenExpiryTask(setupAppend, token);
        } catch (TokenException e) {
            handleException(setupAppend.getWriterId(), setupAppend.getRequestId(), newSegment, "Update Segment Attribute", e);
            return;
        }
    }
    // Get the last Event Number for this writer from the Store. This operation (cache=true) will automatically put
    // the value in the Store's cache so it's faster to access later.
    AttributeId writerAttributeId = AttributeId.fromUUID(writer);
    Futures.exceptionallyComposeExpecting(store.getAttributes(newSegment, Collections.singleton(writerAttributeId), true, TIMEOUT), e -> e instanceof StreamSegmentSealedException, () -> store.getAttributes(newSegment, Collections.singleton(writerAttributeId), false, TIMEOUT)).whenComplete((attributes, u) -> {
        try {
            if (u != null) {
                handleException(writer, setupAppend.getRequestId(), newSegment, "setting up append", u);
            } else {
                // Last event number stored according to Segment store.
                long eventNumber = attributes.getOrDefault(writerAttributeId, Attributes.NULL_ATTRIBUTE_VALUE);
                // Create a new WriterState object based on the attribute value for the last event number for the writer.
                // It should be noted that only one connection for a given segment writer is created by the client.
                // The event number sent by the AppendSetup command is an implicit ack, the writer acks all events
                // below the specified event number.
                WriterState current = this.writerStates.put(Pair.of(newSegment, writer), new WriterState(eventNumber));
                if (current != null) {
                    log.info("SetupAppend invoked again for writer {}. Last event number from store is {}. Prev writer state {}", writer, eventNumber, current);
                }
                connection.send(new AppendSetup(setupAppend.getRequestId(), newSegment, writer, eventNumber));
            }
        } catch (Throwable e) {
            handleException(writer, setupAppend.getRequestId(), newSegment, "handling setupAppend result", e);
        }
    });
}

Example 37 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class StorageWriterTests method generateAttributeUpdates.

private AttributeUpdateCollection generateAttributeUpdates(UpdateableSegmentMetadata segmentMetadata) {
    long coreAttributeValue = segmentMetadata.getAttributes().getOrDefault(CORE_ATTRIBUTE_ID, 0L) + 1;
    val attributeUpdates = new AttributeUpdateCollection();
    attributeUpdates.add(new AttributeUpdate(CORE_ATTRIBUTE_ID, AttributeUpdateType.Accumulate, coreAttributeValue));
    for (int i = 0; i < EXTENDED_ATTRIBUTE_IDS.size(); i++) {
        AttributeId id = EXTENDED_ATTRIBUTE_IDS.get(i);
        long extendedAttributeValue = segmentMetadata.getAttributes().getOrDefault(id, 0L) + 13 + i;
        attributeUpdates.add(new AttributeUpdate(id, AttributeUpdateType.Replace, extendedAttributeValue));
    }
    segmentMetadata.updateAttributes(attributeUpdates.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue)));
    return attributeUpdates;
}

Example 38 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class AttributeAggregatorTests method testRecovery.

/**
 * Tests the ability to resume operations after a recovery.
 */
@Test
public void testRecovery() throws Exception {
    final WriterConfig config = DEFAULT_CONFIG;
    final int attributesPerUpdate = Math.max(1, config.getFlushAttributesThreshold() / 5);
    final int updateCount = config.getFlushAttributesThreshold() * 10;
    @Cleanup TestContext context = new TestContext(config);
    // Generate some data.
    val operations = new ArrayList<AttributeUpdaterOperation>();
    for (int i = 0; i < updateCount; i++) {
        // Add another operation.
        AttributeUpdaterOperation op = i % 2 == 0 ? generateAppendAndUpdateMetadata(attributesPerUpdate, context) : generateUpdateAttributesAndUpdateMetadata(attributesPerUpdate, context);
        operations.add(op);
    }
    // include all operations with indices less than or equal to recoveryId and observe the results.
    for (int recoveryId = 0; recoveryId < operations.size(); recoveryId++) {
        long lastPersistedSeqNo = context.segmentMetadata.getAttributes().getOrDefault(Attributes.ATTRIBUTE_SEGMENT_PERSIST_SEQ_NO, Operation.NO_SEQUENCE_NUMBER);
        val outstandingAttributes = new HashSet<AttributeId>();
        val firstOutstandingSeqNo = new AtomicLong(Operation.NO_SEQUENCE_NUMBER);
        val lastOutstandingSeqNo = new AtomicLong(Operation.NO_SEQUENCE_NUMBER);
        @Cleanup val aggregator = context.createAggregator();
        val expectedAttributes = new HashMap<AttributeId, Long>();
        for (int i = 0; i <= recoveryId; i++) {
            AttributeUpdaterOperation op = operations.get(i);
            // Collect the latest values from this update.
            op.getAttributeUpdates().stream().filter(au -> !Attributes.isCoreAttribute(au.getAttributeId())).forEach(au -> expectedAttributes.put(au.getAttributeId(), au.getValue()));
            aggregator.add(op);
            // We only expect to process an op if its SeqNo is beyond the last one we committed.
            boolean expectedToProcess = op.getSequenceNumber() > lastPersistedSeqNo;
            if (expectedToProcess) {
                addExtendedAttributes(op, outstandingAttributes);
                firstOutstandingSeqNo.compareAndSet(Operation.NO_SEQUENCE_NUMBER, op.getSequenceNumber());
                lastOutstandingSeqNo.set(op.getSequenceNumber());
            }
            Assert.assertEquals("Unexpected LUSN.", firstOutstandingSeqNo.get(), aggregator.getLowestUncommittedSequenceNumber());
            boolean expectFlush = outstandingAttributes.size() >= config.getFlushAttributesThreshold();
            Assert.assertEquals("Unexpected value returned by mustFlush() (count threshold).", expectFlush, aggregator.mustFlush());
            if (expectFlush) {
                // Call flush() and inspect the result.
                WriterFlushResult flushResult = aggregator.flush(TIMEOUT).join();
                Assert.assertEquals("Not all attributes were flushed (count threshold).", outstandingAttributes.size(), flushResult.getFlushedAttributes());
                // We want to verify just those attributes that we flushed, not all of them (not all may be in yet).
                AssertExtensions.assertMapEquals("Unexpected attributes stored in AttributeIndex.", expectedAttributes, context.dataSource.getPersistedAttributes(SEGMENT_ID));
                checkAutoAttributesEventual(lastOutstandingSeqNo.get(), context);
                outstandingAttributes.clear();
                firstOutstandingSeqNo.set(Operation.NO_SEQUENCE_NUMBER);
                lastOutstandingSeqNo.set(Operation.NO_SEQUENCE_NUMBER);
            }
        }
        // We have reached the end. Flush the rest and perform a full check.
        if (recoveryId == operations.size() - 1) {
            aggregator.add(generateSealAndUpdateMetadata(context));
            aggregator.flush(TIMEOUT).join();
            checkAttributes(context);
            checkAutoAttributesEventual(lastOutstandingSeqNo.get(), context);
        }
    }
}

Example 39 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class DataRecoveryTest method testRepairLogEditOperationCreateSegmentProperties.

@Test
public void testRepairLogEditOperationCreateSegmentProperties() throws IOException {
    // Setup command object.
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    STATE.get().getConfigBuilder().include(pravegaProperties);
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    // Create a SegmentProperties object via the command logic and verify that it is equal to the expected one.
    long timestamp = System.currentTimeMillis();
    Map<AttributeId, Long> attributes = new HashMap<>();
    UUID uuid = UUID.randomUUID();
    attributes.put(AttributeId.fromUUID(uuid), 10L);
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(2L).doReturn(3L).doReturn(1L).doReturn(10L).doReturn(timestamp).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("test").doReturn(uuid.toString()).when(command).getStringUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).getBooleanUserInput(Mockito.any());
    SegmentProperties segmentProperties = StreamSegmentInformation.builder().name("test").startOffset(2).length(3).storageLength(1).sealed(true).deleted(false).sealedInStorage(true).deletedInStorage(false).attributes(attributes).lastModified(new ImmutableDate(timestamp)).build();
    Assert.assertEquals(segmentProperties, command.createSegmentProperties());
    // Induce exceptions during the process of creating attributes to check error handling.
    segmentProperties = StreamSegmentInformation.builder().name("test").startOffset(2).length(3).storageLength(1).sealed(true).deleted(false).sealedInStorage(true).deletedInStorage(false).attributes(new HashMap<>()).lastModified(new ImmutableDate(timestamp)).build();
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(2L).doReturn(3L).doReturn(1L).doReturn(timestamp).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("test").doThrow(NumberFormatException.class).when(command).getStringUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).getBooleanUserInput(Mockito.any());
    Assert.assertEquals(segmentProperties, command.createSegmentProperties());
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(2L).doReturn(3L).doReturn(1L).doReturn(timestamp).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("test").doThrow(NullPointerException.class).when(command).getStringUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).getBooleanUserInput(Mockito.any());
    Assert.assertEquals(segmentProperties, command.createSegmentProperties());
}

Example 40 with AttributeId

use of io.pravega.segmentstore.contracts.AttributeId in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testWithAttributesByReference.

private void testWithAttributesByReference(Function<AttributeUpdateCollection, Operation> createOperation) throws Exception {
    final AttributeId referenceAttributeId = AttributeId.randomUUID();
    final AttributeId attributeSegmentLength = AttributeId.randomUUID();
    final long initialAttributeValue = 1234567;
    UpdateableContainerMetadata metadata = createMetadata();
    metadata.getStreamSegmentMetadata(SEGMENT_ID).updateAttributes(ImmutableMap.of(referenceAttributeId, initialAttributeValue));
    val txn = createUpdateTransaction(metadata);
    // Update #1.
    AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(referenceAttributeId, AttributeUpdateType.Accumulate, 2), new DynamicAttributeUpdate(attributeSegmentLength, AttributeUpdateType.None, DynamicAttributeValue.segmentLength(5)));
    Map<AttributeId, Long> expectedValues = ImmutableMap.of(Attributes.ATTRIBUTE_SEGMENT_TYPE, DEFAULT_TYPE.getValue(), referenceAttributeId, initialAttributeValue + 2, attributeSegmentLength, SEGMENT_LENGTH + 5);
    Operation op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    // Verify result.
    verifyAttributeUpdates("after acceptOperation", txn, attributeUpdates, expectedValues);
    txn.commit(metadata);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in segment metadata after final commit.", expectedValues, metadata.getStreamSegmentMetadata(SEGMENT_ID));
}