Examples with WriterFlushResult io.pravega.segmentstore.server.WriterFlushResult used on opensource projects

Example 11 with WriterFlushResult

use of io.pravega.segmentstore.server.WriterFlushResult in project pravega by pravega.

the class AttributeAggregatorTests method testRecovery.

/**
 * Tests the ability to resume operations after a recovery.
 */
@Test
public void testRecovery() throws Exception {
    final WriterConfig config = DEFAULT_CONFIG;
    final int attributesPerUpdate = Math.max(1, config.getFlushAttributesThreshold() / 5);
    final int updateCount = config.getFlushAttributesThreshold() * 10;
    @Cleanup TestContext context = new TestContext(config);
    // Generate some data.
    val operations = new ArrayList<AttributeUpdaterOperation>();
    for (int i = 0; i < updateCount; i++) {
        // Add another operation.
        AttributeUpdaterOperation op = i % 2 == 0 ? generateAppendAndUpdateMetadata(attributesPerUpdate, context) : generateUpdateAttributesAndUpdateMetadata(attributesPerUpdate, context);
        operations.add(op);
    }
    // include all operations with indices less than or equal to recoveryId and observe the results.
    for (int recoveryId = 0; recoveryId < operations.size(); recoveryId++) {
        long lastPersistedSeqNo = context.segmentMetadata.getAttributes().getOrDefault(Attributes.ATTRIBUTE_SEGMENT_PERSIST_SEQ_NO, Operation.NO_SEQUENCE_NUMBER);
        val outstandingAttributes = new HashSet<AttributeId>();
        val firstOutstandingSeqNo = new AtomicLong(Operation.NO_SEQUENCE_NUMBER);
        val lastOutstandingSeqNo = new AtomicLong(Operation.NO_SEQUENCE_NUMBER);
        @Cleanup val aggregator = context.createAggregator();
        val expectedAttributes = new HashMap<AttributeId, Long>();
        for (int i = 0; i <= recoveryId; i++) {
            AttributeUpdaterOperation op = operations.get(i);
            // Collect the latest values from this update.
            op.getAttributeUpdates().stream().filter(au -> !Attributes.isCoreAttribute(au.getAttributeId())).forEach(au -> expectedAttributes.put(au.getAttributeId(), au.getValue()));
            aggregator.add(op);
            // We only expect to process an op if its SeqNo is beyond the last one we committed.
            boolean expectedToProcess = op.getSequenceNumber() > lastPersistedSeqNo;
            if (expectedToProcess) {
                addExtendedAttributes(op, outstandingAttributes);
                firstOutstandingSeqNo.compareAndSet(Operation.NO_SEQUENCE_NUMBER, op.getSequenceNumber());
                lastOutstandingSeqNo.set(op.getSequenceNumber());
            }
            Assert.assertEquals("Unexpected LUSN.", firstOutstandingSeqNo.get(), aggregator.getLowestUncommittedSequenceNumber());
            boolean expectFlush = outstandingAttributes.size() >= config.getFlushAttributesThreshold();
            Assert.assertEquals("Unexpected value returned by mustFlush() (count threshold).", expectFlush, aggregator.mustFlush());
            if (expectFlush) {
                // Call flush() and inspect the result.
                WriterFlushResult flushResult = aggregator.flush(TIMEOUT).join();
                Assert.assertEquals("Not all attributes were flushed (count threshold).", outstandingAttributes.size(), flushResult.getFlushedAttributes());
                // We want to verify just those attributes that we flushed, not all of them (not all may be in yet).
                AssertExtensions.assertMapEquals("Unexpected attributes stored in AttributeIndex.", expectedAttributes, context.dataSource.getPersistedAttributes(SEGMENT_ID));
                checkAutoAttributesEventual(lastOutstandingSeqNo.get(), context);
                outstandingAttributes.clear();
                firstOutstandingSeqNo.set(Operation.NO_SEQUENCE_NUMBER);
                lastOutstandingSeqNo.set(Operation.NO_SEQUENCE_NUMBER);
            }
        }
        // We have reached the end. Flush the rest and perform a full check.
        if (recoveryId == operations.size() - 1) {
            aggregator.add(generateSealAndUpdateMetadata(context));
            aggregator.flush(TIMEOUT).join();
            checkAttributes(context);
            checkAutoAttributesEventual(lastOutstandingSeqNo.get(), context);
        }
    }
}

Example 12 with WriterFlushResult

use of io.pravega.segmentstore.server.WriterFlushResult in project pravega by pravega.

the class AttributeAggregatorTests method testFlushForce.

/**
 * Tests {@link AttributeAggregator#flush} with the force flag set.
 */
@Test
public void testFlushForce() throws Exception {
    final WriterConfig config = DEFAULT_CONFIG;
    final int attributesPerUpdate = Math.max(1, config.getFlushAttributesThreshold() / 5);
    final int updateCount = 10;
    @Cleanup TestContext context = new TestContext(config);
    val outstandingAttributes = new HashSet<AttributeId>();
    val firstOutstandingSeqNo = new AtomicLong(Operation.NO_SEQUENCE_NUMBER);
    val lastOutstandingSeqNo = new AtomicLong(Operation.NO_SEQUENCE_NUMBER);
    // Part 0: Empty operations.
    context.aggregator.add(generateUpdateAttributesAndUpdateMetadata(0, context));
    Assert.assertFalse("Unexpected value returned by mustFlush() after empty operation.", context.aggregator.mustFlush());
    Assert.assertEquals("Unexpected value returned by getLowestUncommittedSequenceNumber() after empty operation.", firstOutstandingSeqNo.get(), context.aggregator.getLowestUncommittedSequenceNumber());
    // attributes accumulated.
    for (int i = 0; i < config.getFlushAttributesThreshold(); i++) {
        // Add another operation.
        AttributeUpdaterOperation op = i % 2 == 0 ? generateAppendAndUpdateMetadata(attributesPerUpdate, context) : generateUpdateAttributesAndUpdateMetadata(attributesPerUpdate, context);
        addExtendedAttributes(op, outstandingAttributes);
        firstOutstandingSeqNo.compareAndSet(Operation.NO_SEQUENCE_NUMBER, op.getSequenceNumber());
        lastOutstandingSeqNo.set(op.getSequenceNumber());
        context.aggregator.add(op);
    }
    Assert.assertTrue("Unexpected value returned by mustFlush() after initial fill-up.", context.aggregator.mustFlush());
    for (int i = 0; i < updateCount; i++) {
        // Add another operation.
        AttributeUpdaterOperation op = i % 2 == 0 ? generateAppendAndUpdateMetadata(attributesPerUpdate, context) : generateUpdateAttributesAndUpdateMetadata(attributesPerUpdate, context);
        addExtendedAttributes(op, outstandingAttributes);
        firstOutstandingSeqNo.compareAndSet(Operation.NO_SEQUENCE_NUMBER, op.getSequenceNumber());
        lastOutstandingSeqNo.set(op.getSequenceNumber());
        context.aggregator.add(op);
        boolean expectFlush = outstandingAttributes.size() >= config.getFlushAttributesThreshold();
        Assert.assertEquals("Unexpected value returned by mustFlush() (count threshold).", expectFlush, context.aggregator.mustFlush());
        Assert.assertEquals("Unexpected value returned by getLowestUncommittedSequenceNumber() before flush (count threshold).", firstOutstandingSeqNo.get(), context.aggregator.getLowestUncommittedSequenceNumber());
        // Call flush() and inspect the result.
        WriterFlushResult flushResult = context.aggregator.flush(true, TIMEOUT).join();
        Assert.assertFalse("Unexpected value returned by mustFlush() after flush (count threshold).", context.aggregator.mustFlush());
        Assert.assertEquals("Not all attributes were flushed (count threshold).", outstandingAttributes.size(), flushResult.getFlushedAttributes());
        checkAttributes(context);
        checkAutoAttributesEventual(lastOutstandingSeqNo.get(), context);
        outstandingAttributes.clear();
        firstOutstandingSeqNo.set(Operation.NO_SEQUENCE_NUMBER);
        lastOutstandingSeqNo.set(Operation.NO_SEQUENCE_NUMBER);
        assertEventuallyEquals("Unexpected value returned by getLowestUncommittedSequenceNumber() after flush (count threshold).", firstOutstandingSeqNo.get(), context.aggregator::getLowestUncommittedSequenceNumber);
    }
}

Example 13 with WriterFlushResult

use of io.pravega.segmentstore.server.WriterFlushResult in project pravega by pravega.

the class AttributeAggregatorTests method testFlush.

/**
 * Tests {@link AttributeAggregator#flush}.
 */
@Test
public void testFlush() throws Exception {
    final WriterConfig config = DEFAULT_CONFIG;
    final int attributesPerUpdate = Math.max(1, config.getFlushAttributesThreshold() / 5);
    final int updateCount = config.getFlushAttributesThreshold() * 10;
    @Cleanup TestContext context = new TestContext(config);
    val outstandingAttributes = new HashSet<AttributeId>();
    val firstOutstandingSeqNo = new AtomicLong(Operation.NO_SEQUENCE_NUMBER);
    val lastOutstandingSeqNo = new AtomicLong(Operation.NO_SEQUENCE_NUMBER);
    // Part 0: Empty operations.
    context.aggregator.add(generateUpdateAttributesAndUpdateMetadata(0, context));
    Assert.assertFalse("Unexpected value returned by mustFlush() after empty operation.", context.aggregator.mustFlush());
    Assert.assertEquals("Unexpected value returned by getLowestUncommittedSequenceNumber() after empty operation.", firstOutstandingSeqNo.get(), context.aggregator.getLowestUncommittedSequenceNumber());
    // Part 1: flush triggered by accumulated counts.
    for (int i = 0; i < updateCount; i++) {
        // Add another operation.
        AttributeUpdaterOperation op = i % 2 == 0 ? generateAppendAndUpdateMetadata(attributesPerUpdate, context) : generateUpdateAttributesAndUpdateMetadata(attributesPerUpdate, context);
        addExtendedAttributes(op, outstandingAttributes);
        firstOutstandingSeqNo.compareAndSet(Operation.NO_SEQUENCE_NUMBER, op.getSequenceNumber());
        lastOutstandingSeqNo.set(op.getSequenceNumber());
        context.aggregator.add(op);
        boolean expectFlush = outstandingAttributes.size() >= config.getFlushAttributesThreshold();
        Assert.assertEquals("Unexpected value returned by mustFlush() (count threshold).", expectFlush, context.aggregator.mustFlush());
        Assert.assertEquals("Unexpected value returned by getLowestUncommittedSequenceNumber() before flush (count threshold).", firstOutstandingSeqNo.get(), context.aggregator.getLowestUncommittedSequenceNumber());
        // Call flush() and inspect the result.
        WriterFlushResult flushResult = context.aggregator.flush(TIMEOUT).join();
        if (expectFlush) {
            Assert.assertFalse("Unexpected value returned by mustFlush() after flush (count threshold).", context.aggregator.mustFlush());
            Assert.assertEquals("Not all attributes were flushed (count threshold).", outstandingAttributes.size(), flushResult.getFlushedAttributes());
            checkAttributes(context);
            checkAutoAttributesEventual(lastOutstandingSeqNo.get(), context);
            outstandingAttributes.clear();
            firstOutstandingSeqNo.set(Operation.NO_SEQUENCE_NUMBER);
            lastOutstandingSeqNo.set(Operation.NO_SEQUENCE_NUMBER);
            assertEventuallyEquals("Unexpected value returned by getLowestUncommittedSequenceNumber() after flush (count threshold).", firstOutstandingSeqNo.get(), context.aggregator::getLowestUncommittedSequenceNumber);
        } else {
            Assert.assertEquals(String.format("Not expecting a flush. OutstandingCount=%s, Threshold=%d", outstandingAttributes.size(), config.getFlushThresholdBytes()), 0, flushResult.getFlushedBytes());
        }
    }
    // Part 2: flush triggered by time.
    for (int i = 0; i < updateCount; i++) {
        // Add another operation.
        AttributeUpdaterOperation op = i % 2 == 0 ? generateAppendAndUpdateMetadata(attributesPerUpdate, context) : generateUpdateAttributesAndUpdateMetadata(attributesPerUpdate, context);
        addExtendedAttributes(op, outstandingAttributes);
        firstOutstandingSeqNo.compareAndSet(Operation.NO_SEQUENCE_NUMBER, op.getSequenceNumber());
        lastOutstandingSeqNo.set(op.getSequenceNumber());
        context.aggregator.add(op);
        // Call flush() and inspect the result.
        Assert.assertFalse("Unexpected value returned by mustFlush() before time elapsed (time threshold).", context.aggregator.mustFlush());
        Assert.assertEquals("Unexpected value returned by getLowestUncommittedSequenceNumber() before flush (time threshold).", firstOutstandingSeqNo.get(), context.aggregator.getLowestUncommittedSequenceNumber());
        val flushResult = forceTimeFlush(context);
        // We are always expecting a flush.
        Assert.assertEquals("Not all attributes were flushed (time threshold).", outstandingAttributes.size(), flushResult.getFlushedAttributes());
        checkAutoAttributesEventual(lastOutstandingSeqNo.get(), context);
        outstandingAttributes.clear();
        firstOutstandingSeqNo.set(Operation.NO_SEQUENCE_NUMBER);
        lastOutstandingSeqNo.set(Operation.NO_SEQUENCE_NUMBER);
        assertEventuallyEquals("Unexpected value returned by getLowestUncommittedSequenceNumber() after flush (time threshold).", firstOutstandingSeqNo.get(), context.aggregator::getLowestUncommittedSequenceNumber);
        Assert.assertEquals("Not expecting any merged bytes in this test.", 0, flushResult.getMergedBytes());
        checkAttributes(context);
    }
}

Example 14 with WriterFlushResult

use of io.pravega.segmentstore.server.WriterFlushResult in project pravega by pravega.

the class WriterStateTests method testForceFlush.

/**
 * Tests {@link WriterState#setForceFlush}.
 */
@Test
public void testForceFlush() throws Exception {
    val s = new WriterState();
    s.setLastReadSequenceNumber(10);
    // 1. When the desired sequence number has already been acknowledged.
    s.setLastTruncatedSequenceNumber(8);
    val r1 = s.setForceFlush(8);
    Assert.assertTrue("Expected an already completed result.", r1.isDone());
    Assert.assertFalse("Expected a result with isAnythingFlushed == false.", r1.join());
    Assert.assertFalse("Not expecting force flush flag to be set.", s.isForceFlush());
    // 2. With something flushed.
    val r2 = s.setForceFlush(100);
    Assert.assertFalse(r2.isDone());
    Assert.assertTrue("Expecting force flush flag to be set.", s.isForceFlush());
    AssertExtensions.assertThrows("", () -> s.setForceFlush(101), ex -> ex instanceof IllegalStateException);
    s.setLastReadSequenceNumber(90);
    s.recordFlushComplete(new WriterFlushResult());
    Assert.assertFalse(r2.isDone());
    s.setLastReadSequenceNumber(100);
    Assert.assertFalse(r2.isDone());
    s.recordFlushComplete(new WriterFlushResult().withFlushedBytes(1));
    TestUtils.await(r2::isDone, 5, 30000);
    Assert.assertTrue("Unexpected result when something was flushed.", r2.join());
    Assert.assertFalse("Expecting force flush flag to be cleared.", s.isForceFlush());
    // 3. Nothing flushed.
    val r3 = s.setForceFlush(200);
    Assert.assertFalse(r3.isDone());
    Assert.assertTrue("Expecting force flush flag to be set.", s.isForceFlush());
    s.setLastReadSequenceNumber(201);
    s.recordFlushComplete(new WriterFlushResult());
    TestUtils.await(r3::isDone, 5, 30000);
    Assert.assertFalse("Unexpected result when nothing was flushed.", r3.join());
    Assert.assertFalse("Expecting force flush flag to be cleared.", s.isForceFlush());
}

Example 15 with WriterFlushResult

use of io.pravega.segmentstore.server.WriterFlushResult in project pravega by pravega.

the class SegmentAggregatorTests method tryFlushSegment.

private <T extends Throwable> WriterFlushResult tryFlushSegment(SegmentAggregator aggregator, Supplier<T> exceptionProvider) {
    try {
        WriterFlushResult flushResult = aggregator.flush(TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        T expectedException = exceptionProvider.get();
        Assert.assertNull("Expected an exception but none got thrown.", expectedException);
        Assert.assertNotNull("Expected a FlushResult.", flushResult);
        return flushResult;
    } catch (Throwable ex) {
        ex = Exceptions.unwrap(ex);
        T expectedException = exceptionProvider.get();
        Assert.assertEquals("Unexpected exception or no exception got thrown.", expectedException, ex);
        return null;
    }
}