Search in sources :

Example 66 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class StreamSegmentContainerTests method checkReadIndex.

private static void checkReadIndex(Map<String, ByteArrayOutputStream> segmentContents, Map<String, Long> lengths, Map<String, Long> truncationOffsets, SegmentContainer container) throws Exception {
    waitForOperationsInReadIndex(container);
    for (String segmentName : segmentContents.keySet()) {
        long segmentLength = lengths.get(segmentName);
        long startOffset = truncationOffsets.getOrDefault(segmentName, 0L);
        val si = container.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        Assert.assertEquals("Unexpected Metadata StartOffset for segment " + segmentName, startOffset, si.getStartOffset());
        Assert.assertEquals("Unexpected Metadata Length for segment " + segmentName, segmentLength, si.getLength());
        byte[] expectedData = segmentContents.get(segmentName).toByteArray();
        long expectedCurrentOffset = startOffset;
        @Cleanup ReadResult readResult = container.read(segmentName, expectedCurrentOffset, (int) (segmentLength - startOffset), TIMEOUT).join();
        Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext());
        // A more thorough read check is done in testSegmentRegularOperations; here we just check if the data was merged correctly.
        while (readResult.hasNext()) {
            ReadResultEntry readEntry = readResult.next();
            AssertExtensions.assertGreaterThan("getRequestedReadLength should be a positive integer for segment " + segmentName, 0, readEntry.getRequestedReadLength());
            Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset());
            Assert.assertTrue("getContent() did not return a completed future for segment" + segmentName, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally());
            Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
            BufferView readEntryContents = readEntry.getContent().join();
            byte[] actualData = readEntryContents.getCopy();
            AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength());
            expectedCurrentOffset += readEntryContents.getLength();
        }
        Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
    }
}
Also used : lombok.val(lombok.val) BufferView(io.pravega.common.util.BufferView) ReadResultEntry(io.pravega.segmentstore.contracts.ReadResultEntry) ReadResult(io.pravega.segmentstore.contracts.ReadResult) Cleanup(lombok.Cleanup)

Example 67 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class StreamSegmentContainerTests method testSegmentSeal.

/**
 * Test the seal operation on StreamSegments. Also tests the behavior of Reads (non-tailing) when encountering
 * the end of a sealed StreamSegment.
 */
@Test
public void testSegmentSeal() throws Exception {
    final int appendsPerSegment = 1;
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    // 2. Add some appends.
    ArrayList<CompletableFuture<Long>> appendFutures = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    for (String segmentName : segmentNames) {
        ByteArrayOutputStream segmentStream = new ByteArrayOutputStream();
        segmentContents.put(segmentName, segmentStream);
        for (int i = 0; i < appendsPerSegment; i++) {
            ByteArraySegment appendData = getAppendData(segmentName, i);
            appendFutures.add(context.container.append(segmentName, appendData, null, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.getLength());
            appendData.copyTo(segmentStream);
        }
    }
    Futures.allOf(appendFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Seal first half of segments.
    ArrayList<CompletableFuture<Long>> sealFutures = new ArrayList<>();
    for (int i = 0; i < segmentNames.size() / 2; i++) {
        sealFutures.add(context.container.sealStreamSegment(segmentNames.get(i), TIMEOUT));
    }
    Futures.allOf(sealFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Check that the segments were properly sealed.
    for (int i = 0; i < segmentNames.size(); i++) {
        String segmentName = segmentNames.get(i);
        boolean expectedSealed = i < segmentNames.size() / 2;
        SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, TIMEOUT).join();
        if (expectedSealed) {
            Assert.assertTrue("Segment is not sealed when it should be " + segmentName, sp.isSealed());
            Assert.assertEquals("Unexpected result from seal() future for segment " + segmentName, sp.getLength(), (long) sealFutures.get(i).join());
            AssertExtensions.assertThrows("Container allowed appending to a sealed segment " + segmentName, context.container.append(segmentName, new ByteArraySegment("foo".getBytes()), null, TIMEOUT)::join, ex -> ex instanceof StreamSegmentSealedException);
        } else {
            Assert.assertFalse("Segment is sealed when it shouldn't be " + segmentName, sp.isSealed());
            // Verify we can still append to these segments.
            byte[] appendData = "foo".getBytes();
            context.container.append(segmentName, new ByteArraySegment(appendData), null, TIMEOUT).join();
            segmentContents.get(segmentName).write(appendData);
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
        }
    }
    // 4. Reads (regular reads, not tail reads, and only for the sealed segments).
    waitForOperationsInReadIndex(context.container);
    for (int i = 0; i < segmentNames.size() / 2; i++) {
        String segmentName = segmentNames.get(i);
        long segmentLength = context.container.getStreamSegmentInfo(segmentName, TIMEOUT).join().getLength();
        // Read starting 1 byte from the end - make sure it wont hang at the end by turning into a future read.
        final int totalReadLength = 1;
        long expectedCurrentOffset = segmentLength - totalReadLength;
        @Cleanup ReadResult readResult = context.container.read(segmentName, expectedCurrentOffset, Integer.MAX_VALUE, TIMEOUT).join();
        int readLength = 0;
        while (readResult.hasNext()) {
            ReadResultEntry readEntry = readResult.next();
            if (readEntry.getStreamSegmentOffset() >= segmentLength) {
                Assert.assertEquals("Unexpected value for isEndOfStreamSegment when reaching the end of sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                AssertExtensions.assertSuppliedFutureThrows("ReadResultEntry.getContent() returned a result when reached the end of sealed segment " + segmentName, readEntry::getContent, ex -> ex instanceof IllegalStateException);
            } else {
                Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment before reaching end of sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                Assert.assertTrue("getContent() did not return a completed future for segment" + segmentName, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally());
                BufferView readEntryContents = readEntry.getContent().join();
                expectedCurrentOffset += readEntryContents.getLength();
                readLength += readEntryContents.getLength();
            }
        }
        Assert.assertEquals("Unexpected number of bytes read.", totalReadLength, readLength);
        Assert.assertTrue("ReadResult was not closed when reaching the end of sealed segment" + segmentName, readResult.isClosed());
    }
    // 5. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}
Also used : ByteArraySegment(io.pravega.common.util.ByteArraySegment) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ReadResult(io.pravega.segmentstore.contracts.ReadResult) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) BufferView(io.pravega.common.util.BufferView) StreamSegmentSealedException(io.pravega.segmentstore.contracts.StreamSegmentSealedException) ReadResultEntry(io.pravega.segmentstore.contracts.ReadResultEntry) AtomicLong(java.util.concurrent.atomic.AtomicLong) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) Test(org.junit.Test)

Example 68 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class StreamSegmentContainerTests method testConcurrentSegmentActivation.

/**
 * Tests the ability for the StreamSegmentContainer to handle concurrent actions on a Segment that it does not know
 * anything about, and handling the resulting concurrency.
 * Note: this is tested with a single segment. It could be tested with multiple segments, but different segments
 * are mostly independent of each other, so we would not be gaining much by doing so.
 */
@Test
public void testConcurrentSegmentActivation() throws Exception {
    final AttributeId attributeAccumulate = AttributeId.randomUUID();
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    final int appendLength = 10;
    @Cleanup TestContext context = createContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    String segmentName = createSegments(context).get(0);
    // 2. Add some appends.
    List<CompletableFuture<Void>> opFutures = Collections.synchronizedList(new ArrayList<>());
    AtomicLong expectedLength = new AtomicLong();
    @Cleanup("shutdown") ExecutorService testExecutor = newScheduledThreadPool(Math.min(20, APPENDS_PER_SEGMENT), "testConcurrentSegmentActivation");
    val submitFutures = new ArrayList<Future<?>>();
    for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
        final byte fillValue = (byte) i;
        submitFutures.add(testExecutor.submit(() -> {
            val attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            byte[] appendData = new byte[appendLength];
            Arrays.fill(appendData, (byte) (fillValue + 1));
            opFutures.add(Futures.toVoid(context.container.append(segmentName, new ByteArraySegment(appendData), attributeUpdates, TIMEOUT)));
            expectedLength.addAndGet(appendData.length);
        }));
    }
    // 2.1 Update the attribute.
    for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
        submitFutures.add(testExecutor.submit(() -> {
            AttributeUpdateCollection attributeUpdates = AttributeUpdateCollection.from(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            opFutures.add(context.container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }));
    }
    // Wait for the submittal of tasks to complete.
    submitFutures.forEach(this::await);
    // Now wait for all the appends to finish.
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. getSegmentInfo: verify final state of the attribute.
    SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, TIMEOUT).join();
    Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength.get(), sp.getLength());
    Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
    Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted());
    Assert.assertEquals("Unexpected Segment Type.", getSegmentType(segmentName), SegmentType.fromAttributes(sp.getAttributes()));
    // Verify all attribute values.
    Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeAccumulate, Attributes.NULL_ATTRIBUTE_VALUE));
    checkActiveSegments(context.container, 1);
    // 4. Written data.
    waitForOperationsInReadIndex(context.container);
    byte[] actualData = new byte[(int) expectedLength.get()];
    int offset = 0;
    @Cleanup ReadResult readResult = context.container.read(segmentName, 0, actualData.length, TIMEOUT).join();
    while (readResult.hasNext()) {
        ReadResultEntry readEntry = readResult.next();
        BufferView readEntryContents = readEntry.getContent().join();
        AssertExtensions.assertLessThanOrEqual("Too much to read.", actualData.length, offset + actualData.length);
        readEntryContents.copyTo(ByteBuffer.wrap(actualData, offset, actualData.length));
        offset += actualData.length;
    }
    Assert.assertEquals("Unexpected number of bytes read.", actualData.length, offset);
    Assert.assertTrue("Unexpected number of bytes read (multiple of appendLength).", actualData.length % appendLength == 0);
    boolean[] observedValues = new boolean[APPENDS_PER_SEGMENT + 1];
    for (int i = 0; i < actualData.length; i += appendLength) {
        byte value = actualData[i];
        Assert.assertFalse("Append with value " + value + " was written multiple times.", observedValues[value]);
        observedValues[value] = true;
        for (int j = 1; j < appendLength; j++) {
            Assert.assertEquals("Append was not written atomically at offset " + (i + j), value, actualData[i + j]);
        }
    }
    // Verify all the appends made it (we purposefully did not write 0, since that's the default fill value in an array).
    Assert.assertFalse("Not expecting 0 as a value.", observedValues[0]);
    for (int i = 1; i < observedValues.length; i++) {
        Assert.assertTrue("Append with value " + i + " was not written.", observedValues[i]);
    }
    context.container.stopAsync().awaitTerminated();
}
Also used : lombok.val(lombok.val) AttributeUpdateCollection(io.pravega.segmentstore.contracts.AttributeUpdateCollection) AttributeUpdate(io.pravega.segmentstore.contracts.AttributeUpdate) DynamicAttributeUpdate(io.pravega.segmentstore.contracts.DynamicAttributeUpdate) ByteArraySegment(io.pravega.common.util.ByteArraySegment) AttributeId(io.pravega.segmentstore.contracts.AttributeId) ArrayList(java.util.ArrayList) ReadResult(io.pravega.segmentstore.contracts.ReadResult) Cleanup(lombok.Cleanup) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicLong(java.util.concurrent.atomic.AtomicLong) BufferView(io.pravega.common.util.BufferView) ReadResultEntry(io.pravega.segmentstore.contracts.ReadResultEntry) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExecutorService(java.util.concurrent.ExecutorService) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) Test(org.junit.Test)

Example 69 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class ContainerEventProcessorTests method testReadWithFailingSegment.

/**
 * Test the behavior of the EventProcessor when internal Segment reads fail.
 *
 * @throws Exception
 */
@Test(timeout = 10000)
public void testReadWithFailingSegment() throws Exception {
    DirectSegmentAccess faultySegment = spy(new SegmentMock(this.executorService()));
    Function<String, CompletableFuture<DirectSegmentAccess>> faultySegmentSupplier = s -> CompletableFuture.completedFuture(faultySegment);
    @Cleanup ContainerEventProcessor eventProcessorService = new ContainerEventProcessorImpl(0, faultySegmentSupplier, ITERATION_DELAY, CONTAINER_OPERATION_TIMEOUT, this.executorService());
    int maxItemsProcessed = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int truncationDataSize = 500;
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsProcessed, maxOutstandingBytes, truncationDataSize);
    ReusableLatch latch = new ReusableLatch();
    Function<List<BufferView>, CompletableFuture<Void>> doNothing = l -> {
        latch.release();
        return CompletableFuture.completedFuture(null);
    };
    // Make the internal Segment of the processor to fail upon a read.
    when(faultySegment.read(anyLong(), anyInt(), any(Duration.class))).thenThrow(IntentionalException.class).thenCallRealMethod();
    @Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testSegmentMax", doNothing, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Write an event to make sure that the processor is running and await for it to be processed.
    BufferView event = new ByteArraySegment("Test".getBytes());
    processor.add(event, TIMEOUT_FUTURE).join();
    latch.await();
}
Also used : IntStream(java.util.stream.IntStream) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) ContainerEventProcessor(io.pravega.segmentstore.server.ContainerEventProcessor) AssertExtensions(io.pravega.test.common.AssertExtensions) Exceptions(io.pravega.common.Exceptions) Cleanup(lombok.Cleanup) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) ByteBuffer(java.nio.ByteBuffer) ArrayList(java.util.ArrayList) SegmentMetadata(io.pravega.segmentstore.server.SegmentMetadata) BufferView(io.pravega.common.util.BufferView) Duration(java.time.Duration) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Timeout(org.junit.rules.Timeout) SerializationException(io.pravega.common.io.SerializationException) Mockito.anyLong(org.mockito.Mockito.anyLong) ReusableLatch(io.pravega.common.util.ReusableLatch) IntentionalException(io.pravega.test.common.IntentionalException) Test(org.junit.Test) IOException(java.io.IOException) CompletionException(java.util.concurrent.CompletionException) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) SegmentMock(io.pravega.segmentstore.server.SegmentMock) AtomicLong(java.util.concurrent.atomic.AtomicLong) DirectSegmentAccess(io.pravega.segmentstore.server.DirectSegmentAccess) List(java.util.List) Rule(org.junit.Rule) ByteArraySegment(io.pravega.common.util.ByteArraySegment) ThreadPooledTestSuite(io.pravega.test.common.ThreadPooledTestSuite) Mockito.anyInt(org.mockito.Mockito.anyInt) Assert(org.junit.Assert) Mockito.mock(org.mockito.Mockito.mock) ByteArraySegment(io.pravega.common.util.ByteArraySegment) SegmentMock(io.pravega.segmentstore.server.SegmentMock) Cleanup(lombok.Cleanup) IntentionalException(io.pravega.test.common.IntentionalException) ContainerEventProcessor(io.pravega.segmentstore.server.ContainerEventProcessor) CompletableFuture(java.util.concurrent.CompletableFuture) DirectSegmentAccess(io.pravega.segmentstore.server.DirectSegmentAccess) ReusableLatch(io.pravega.common.util.ReusableLatch) BufferView(io.pravega.common.util.BufferView) ArrayList(java.util.ArrayList) List(java.util.List) Test(org.junit.Test)

Example 70 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class ContainerEventProcessorTests method testContainerMaxItemsPerBatchRespected.

public static void testContainerMaxItemsPerBatchRespected(ContainerEventProcessor eventProcessorService) throws Exception {
    int maxItemsPerBatch = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int allEventsToProcess = 1000;
    int truncationDataSize = 500;
    List<Integer> processedItems = new ArrayList<>();
    Function<List<BufferView>, CompletableFuture<Void>> handler = getNumberSequenceHandler(processedItems, maxItemsPerBatch);
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsPerBatch, maxOutstandingBytes, truncationDataSize);
    @Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testSegment", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Write all the events as fast as possible.
    for (int i = 0; i < allEventsToProcess; i++) {
        BufferView event = new ByteArraySegment(ByteBuffer.allocate(Integer.BYTES).putInt(i).array());
        processor.add(event, TIMEOUT_FUTURE).join();
    }
    // Perform basic validation on this processor.
    validateProcessorResults(processor, processedItems, allEventsToProcess);
}
Also used : ByteArraySegment(io.pravega.common.util.ByteArraySegment) ArrayList(java.util.ArrayList) Cleanup(lombok.Cleanup) ContainerEventProcessor(io.pravega.segmentstore.server.ContainerEventProcessor) CompletableFuture(java.util.concurrent.CompletableFuture) BufferView(io.pravega.common.util.BufferView) ArrayList(java.util.ArrayList) List(java.util.List)

Aggregations

BufferView (io.pravega.common.util.BufferView)77 ArrayList (java.util.ArrayList)49 lombok.val (lombok.val)49 ByteArraySegment (io.pravega.common.util.ByteArraySegment)44 Cleanup (lombok.Cleanup)42 Duration (java.time.Duration)39 Test (org.junit.Test)39 List (java.util.List)37 CompletableFuture (java.util.concurrent.CompletableFuture)34 AssertExtensions (io.pravega.test.common.AssertExtensions)29 HashMap (java.util.HashMap)29 Assert (org.junit.Assert)29 ThreadPooledTestSuite (io.pravega.test.common.ThreadPooledTestSuite)28 TimeUnit (java.util.concurrent.TimeUnit)28 AtomicReference (java.util.concurrent.atomic.AtomicReference)26 Collectors (java.util.stream.Collectors)26 AtomicLong (java.util.concurrent.atomic.AtomicLong)25 Exceptions (io.pravega.common.Exceptions)24 TableEntry (io.pravega.segmentstore.contracts.tables.TableEntry)24 Map (java.util.Map)22