Examples with Operation io.pravega.segmentstore.server.logs.operations.Operation used on opensource projects

Example 31 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class ContainerMetadataUpdateTransactionTests method testWithAttributes.

private void testWithAttributes(Function<Collection<AttributeUpdate>, Operation> createOperation) throws Exception {
    final UUID attributeNoUpdate = UUID.randomUUID();
    final UUID attributeAccumulate = UUID.randomUUID();
    final UUID attributeReplace = UUID.randomUUID();
    final UUID attributeReplaceIfGreater = UUID.randomUUID();
    final UUID attributeReplaceIfEquals = UUID.randomUUID();
    UpdateableContainerMetadata metadata = createMetadata();
    val txn = createUpdateTransaction(metadata);
    // Update #1.
    Collection<AttributeUpdate> attributeUpdates = new ArrayList<>();
    // Initial add, so it's ok.
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 1));
    attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
    attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, 1));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 1));
    // Need to initialize to something.
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.Replace, 1));
    val expectedValues = attributeUpdates.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue));
    Operation op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    // Verify that the AttributeUpdates still have the same values (there was nothing there prior) and that the updater
    // has internalized the attribute updates.
    verifyAttributeUpdates("after acceptOperation (1)", txn, attributeUpdates, expectedValues);
    // Update #2: update all attributes that can be updated.
    attributeUpdates.clear();
    // 1 + 1 = 2
    attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
    attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 2));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, 2, 1));
    expectedValues.put(attributeAccumulate, 2L);
    expectedValues.put(attributeReplace, 2L);
    expectedValues.put(attributeReplaceIfGreater, 2L);
    expectedValues.put(attributeReplaceIfEquals, 2L);
    op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    // This is still in the transaction, so we need to add it for comparison sake.
    attributeUpdates.add(new AttributeUpdate(attributeNoUpdate, AttributeUpdateType.None, 1));
    verifyAttributeUpdates("after acceptOperation (2)", txn, attributeUpdates, expectedValues);
    // Update #3: after commit, verify that attributes are committed when they need to.
    val previousAcceptedValues = new HashMap<UUID, Long>(expectedValues);
    txn.commit(metadata);
    attributeUpdates.clear();
    // 2 + 1 = 3
    attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
    attributeUpdates.add(new AttributeUpdate(attributeReplace, AttributeUpdateType.Replace, 3));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfGreater, AttributeUpdateType.ReplaceIfGreater, 3));
    attributeUpdates.add(new AttributeUpdate(attributeReplaceIfEquals, AttributeUpdateType.ReplaceIfEquals, 3, 2));
    expectedValues.put(attributeAccumulate, 3L);
    expectedValues.put(attributeReplace, 3L);
    expectedValues.put(attributeReplaceIfGreater, 3L);
    expectedValues.put(attributeReplaceIfEquals, 3L);
    op = createOperation.apply(attributeUpdates);
    txn.preProcessOperation(op);
    txn.acceptOperation(op);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in segment metadata after commit+acceptOperation, but prior to second commit.", previousAcceptedValues, metadata.getStreamSegmentMetadata(SEGMENT_ID));
    verifyAttributeUpdates("after commit+acceptOperation", txn, attributeUpdates, expectedValues);
    // Final step: commit Append #3, and verify final segment metadata.
    txn.commit(metadata);
    SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in segment metadata after final commit.", expectedValues, metadata.getStreamSegmentMetadata(SEGMENT_ID));
}

Example 32 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class DurableLogTests method assertRecoveredOperationsMatch.

private void assertRecoveredOperationsMatch(List<Operation> expected, List<Operation> actual) {
    Assert.assertEquals("Recovered operations do not match original ones. Collections differ in size.", expected.size(), actual.size());
    for (int i = 0; i < expected.size(); i++) {
        Operation expectedItem = expected.get(i);
        Operation actualItem = actual.get(i);
        OperationComparer.DEFAULT.assertEquals(String.format("Recovered operations do not match original ones. Elements at index %d differ. Expected '%s', found '%s'.", i, expectedItem, actualItem), expectedItem, actualItem);
    }
}

Example 33 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class DurableLogTests method testMetadataCheckpoint.

/**
 * Tests the ability of the DurableLog to add MetadataCheckpointOperations.
 *
 * @param createDurableLogConfig     A Supplier that creates a DurableLogConfig object.
 * @param waitForProcessingFrequency The frequency at which to stop and wait for operations to be processed by the
 *                                   DurableLog before adding others.
 */
private void testMetadataCheckpoint(Supplier<DurableLogConfig> createDurableLogConfig, int waitForProcessingFrequency) throws Exception {
    int streamSegmentCount = 500;
    int appendsPerStreamSegment = 20;
    // Setup a DurableLog and start it.
    @Cleanup ContainerSetup setup = new ContainerSetup(executorService());
    DurableLogConfig durableLogConfig = createDurableLogConfig.get();
    setup.setDurableLogConfig(durableLogConfig);
    @Cleanup DurableLog durableLog = setup.createDurableLog();
    durableLog.startAsync().awaitRunning();
    // Verify that on a freshly created DurableLog, it auto-adds a MetadataCheckpoint as the first operation.
    verifyFirstItemIsMetadataCheckpoint(durableLog.read(-1L, 1, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS));
    // Generate some test data (we need to do this after we started the DurableLog because in the process of
    // recovery, it wipes away all existing metadata).
    HashSet<Long> streamSegmentIds = createStreamSegmentsInMetadata(streamSegmentCount, setup.metadata);
    List<Operation> operations = generateOperations(streamSegmentIds, Collections.emptyMap(), appendsPerStreamSegment, NO_METADATA_CHECKPOINT, false, false);
    // Process all generated operations.
    List<OperationWithCompletion> completionFutures = processOperations(operations, durableLog, waitForProcessingFrequency);
    // Wait for all such operations to complete. If any of them failed, this will fail too and report the exception.
    OperationWithCompletion.allOf(completionFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    List<Operation> readOperations = readUpToSequenceNumber(durableLog, setup.metadata.getOperationSequenceNumber());
    // Count the number of injected MetadataCheckpointOperations.
    int injectedOperationCount = 0;
    for (Operation o : readOperations) {
        if (o instanceof MetadataCheckpointOperation) {
            injectedOperationCount++;
        }
    }
    // Calculate how many we were expecting.
    int expectedCheckpoints = readOperations.size() - (int) operations.stream().filter(Operation::canSerialize).count();
    if (expectedCheckpoints != injectedOperationCount) {
        Assert.assertEquals("Unexpected operations were injected. Expected only MetadataCheckpointOperations.", expectedCheckpoints, injectedOperationCount);
    }
    // We expect at least 2 injected operations (one is the very first one (checked above), and then at least
    // one more based on written data.
    AssertExtensions.assertGreaterThan("Insufficient number of injected operations.", 1, injectedOperationCount);
    // Stop the processor.
    durableLog.stopAsync().awaitTerminated();
}

Example 34 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class DurableLogTests method verifyFirstItemIsMetadataCheckpoint.

private void verifyFirstItemIsMetadataCheckpoint(Iterator<Operation> logIterator) {
    Assert.assertTrue("DurableLog is empty even though a MetadataCheckpointOperation was expected.", logIterator.hasNext());
    Operation firstOp = logIterator.next();
    Assert.assertTrue("First operation in DurableLog is not a MetadataCheckpointOperation: " + firstOp, firstOp instanceof MetadataCheckpointOperation);
}

Example 35 with Operation

use of io.pravega.segmentstore.server.logs.operations.Operation in project pravega by pravega.

the class DurableLogTests method testAddWithDataLogWriterNotPrimaryException.

/**
 * Tests the ability of the DurableLog to handle a DataLogWriterNotPrimaryException.
 */
@Test
public void testAddWithDataLogWriterNotPrimaryException() throws Exception {
    int streamSegmentCount = 1;
    int appendsPerStreamSegment = 1;
    // Setup a DurableLog and start it.
    @Cleanup ContainerSetup setup = new ContainerSetup(executorService());
    @Cleanup DurableLog durableLog = setup.createDurableLog();
    durableLog.startAsync().awaitRunning();
    HashSet<Long> streamSegmentIds = createStreamSegmentsInMetadata(streamSegmentCount, setup.metadata);
    List<Operation> operations = generateOperations(streamSegmentIds, new HashMap<>(), appendsPerStreamSegment, METADATA_CHECKPOINT_EVERY, false, false);
    ErrorInjector<Exception> aSyncErrorInjector = new ErrorInjector<>(count -> true, () -> new CompletionException(new DataLogWriterNotPrimaryException("intentional")));
    setup.dataLog.get().setAppendErrorInjectors(null, aSyncErrorInjector);
    // Process all generated operations.
    List<OperationWithCompletion> completionFutures = processOperations(operations, durableLog);
    // Wait for all such operations to complete. We are expecting exceptions, so verify that we do.
    AssertExtensions.assertThrows("No operations failed.", OperationWithCompletion.allOf(completionFutures)::join, ex -> ex instanceof IOException || ex instanceof DataLogWriterNotPrimaryException);
    // Verify that the OperationProcessor automatically shuts down and that it has the right failure cause.
    ServiceListeners.awaitShutdown(durableLog, TIMEOUT, false);
    Assert.assertEquals("DurableLog is not in a failed state after fence-out detected.", Service.State.FAILED, durableLog.state());
    Assert.assertTrue("DurableLog did not fail with the correct exception.", Exceptions.unwrap(durableLog.failureCause()) instanceof DataLogWriterNotPrimaryException);
}