Examples with ReadResult io.pravega.segmentstore.contracts.ReadResult used on opensource projects

Example 11 with ReadResult

use of io.pravega.segmentstore.contracts.ReadResult in project pravega by pravega.

the class ReadTest method testReadDirectlyFromStore.

@Test
public void testReadDirectlyFromStore() throws InterruptedException, ExecutionException, IOException {
    String segmentName = "testReadFromStore";
    final int entries = 10;
    final byte[] data = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    UUID clientId = UUID.randomUUID();
    StreamSegmentStore segmentStore = serviceBuilder.createStreamSegmentService();
    fillStoreForSegment(segmentName, clientId, data, entries, segmentStore);
    ReadResult result = segmentStore.read(segmentName, 0, entries * data.length, Duration.ZERO).get();
    int index = 0;
    while (result.hasNext()) {
        ReadResultEntry entry = result.next();
        ReadResultEntryType type = entry.getType();
        assertEquals(ReadResultEntryType.Cache, type);
        // Each ReadResultEntryContents may be of an arbitrary length - we should make no assumptions.
        ReadResultEntryContents contents = entry.getContent().get();
        byte next;
        while ((next = (byte) contents.getData().read()) != -1) {
            byte expected = data[index % data.length];
            assertEquals(expected, next);
            index++;
        }
    }
    assertEquals(entries * data.length, index);
}

Example 12 with ReadResult

use of io.pravega.segmentstore.contracts.ReadResult in project pravega by pravega.

the class StreamSegmentContainerTests method testConcurrentSegmentActivation.

/**
 * Tests the ability for the StreamSegmentContainer to handle concurrent actions on a Segment that it does not know
 * anything about, and handling the resulting concurrency.
 * Note: this is tested with a single segment. It could be tested with multiple segments, but different segments
 * are mostly independent of each other, so we would not be gaining much by doing so.
 */
@Test
public void testConcurrentSegmentActivation() throws Exception {
    final UUID attributeAccumulate = UUID.randomUUID();
    final long expectedAttributeValue = APPENDS_PER_SEGMENT + ATTRIBUTE_UPDATES_PER_SEGMENT;
    final int appendLength = 10;
    @Cleanup TestContext context = new TestContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    String segmentName = createSegments(context).get(0);
    // 2. Add some appends.
    List<CompletableFuture<Void>> opFutures = Collections.synchronizedList(new ArrayList<>());
    AtomicLong expectedLength = new AtomicLong();
    @Cleanup("shutdown") ExecutorService testExecutor = newScheduledThreadPool(Math.min(20, APPENDS_PER_SEGMENT), "testConcurrentSegmentActivation");
    val submitFutures = new ArrayList<Future<?>>();
    for (int i = 0; i < APPENDS_PER_SEGMENT; i++) {
        final byte fillValue = (byte) i;
        submitFutures.add(testExecutor.submit(() -> {
            Collection<AttributeUpdate> attributeUpdates = Collections.singleton(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            byte[] appendData = new byte[appendLength];
            Arrays.fill(appendData, (byte) (fillValue + 1));
            opFutures.add(context.container.append(segmentName, appendData, attributeUpdates, TIMEOUT));
            expectedLength.addAndGet(appendData.length);
        }));
    }
    // 2.1 Update the attribute.
    for (int i = 0; i < ATTRIBUTE_UPDATES_PER_SEGMENT; i++) {
        submitFutures.add(testExecutor.submit(() -> {
            Collection<AttributeUpdate> attributeUpdates = new ArrayList<>();
            attributeUpdates.add(new AttributeUpdate(attributeAccumulate, AttributeUpdateType.Accumulate, 1));
            opFutures.add(context.container.updateAttributes(segmentName, attributeUpdates, TIMEOUT));
        }));
    }
    // Wait for the submittal of tasks to complete.
    submitFutures.forEach(this::await);
    // Now wait for all the appends to finish.
    Futures.allOf(opFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. getSegmentInfo: verify final state of the attribute.
    SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
    Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength.get(), sp.getLength());
    Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
    Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted());
    // Verify all attribute values.
    Assert.assertEquals("Unexpected value for attribute " + attributeAccumulate + " for segment " + segmentName, expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeAccumulate, SegmentMetadata.NULL_ATTRIBUTE_VALUE));
    checkActiveSegments(context.container, 1);
    // 4. Written data.
    waitForOperationsInReadIndex(context.container);
    byte[] actualData = new byte[(int) expectedLength.get()];
    int offset = 0;
    @Cleanup ReadResult readResult = context.container.read(segmentName, 0, actualData.length, TIMEOUT).join();
    while (readResult.hasNext()) {
        ReadResultEntry readEntry = readResult.next();
        ReadResultEntryContents readEntryContents = readEntry.getContent().join();
        AssertExtensions.assertLessThanOrEqual("Too much to read.", actualData.length, offset + actualData.length);
        StreamHelpers.readAll(readEntryContents.getData(), actualData, offset, actualData.length);
        offset += actualData.length;
    }
    Assert.assertEquals("Unexpected number of bytes read.", actualData.length, offset);
    Assert.assertTrue("Unexpected number of bytes read (multiple of appendLength).", actualData.length % appendLength == 0);
    boolean[] observedValues = new boolean[APPENDS_PER_SEGMENT + 1];
    for (int i = 0; i < actualData.length; i += appendLength) {
        byte value = actualData[i];
        Assert.assertFalse("Append with value " + value + " was written multiple times.", observedValues[value]);
        observedValues[value] = true;
        for (int j = 1; j < appendLength; j++) {
            Assert.assertEquals("Append was not written atomically at offset " + (i + j), value, actualData[i + j]);
        }
    }
    // Verify all the appends made it (we purposefully did not write 0, since that's the default fill value in an array).
    Assert.assertFalse("Not expecting 0 as a value.", observedValues[0]);
    for (int i = 1; i < observedValues.length; i++) {
        Assert.assertTrue("Append with value " + i + " was not written.", observedValues[i]);
    }
    context.container.stopAsync().awaitTerminated();
}

Example 13 with ReadResult

use of io.pravega.segmentstore.contracts.ReadResult in project pravega by pravega.

the class StreamSegmentContainerTests method testSegmentSeal.

/**
 * Test the seal operation on StreamSegments. Also tests the behavior of Reads (non-tailing) when encountering
 * the end of a sealed StreamSegment.
 */
@Test
public void testSegmentSeal() throws Exception {
    final int appendsPerSegment = 1;
    @Cleanup TestContext context = new TestContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    // 2. Add some appends.
    ArrayList<CompletableFuture<Void>> appendFutures = new ArrayList<>();
    HashMap<String, Long> lengths = new HashMap<>();
    for (String segmentName : segmentNames) {
        ByteArrayOutputStream segmentStream = new ByteArrayOutputStream();
        segmentContents.put(segmentName, segmentStream);
        for (int i = 0; i < appendsPerSegment; i++) {
            byte[] appendData = getAppendData(segmentName, i);
            appendFutures.add(context.container.append(segmentName, appendData, null, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
            segmentStream.write(appendData);
        }
    }
    Futures.allOf(appendFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // 3. Seal first half of segments.
    ArrayList<CompletableFuture<Long>> sealFutures = new ArrayList<>();
    for (int i = 0; i < segmentNames.size() / 2; i++) {
        sealFutures.add(context.container.sealStreamSegment(segmentNames.get(i), TIMEOUT));
    }
    Futures.allOf(sealFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Check that the segments were properly sealed.
    for (int i = 0; i < segmentNames.size(); i++) {
        String segmentName = segmentNames.get(i);
        boolean expectedSealed = i < segmentNames.size() / 2;
        SegmentProperties sp = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        if (expectedSealed) {
            Assert.assertTrue("Segment is not sealed when it should be " + segmentName, sp.isSealed());
            Assert.assertEquals("Unexpected result from seal() future for segment " + segmentName, sp.getLength(), (long) sealFutures.get(i).join());
            AssertExtensions.assertThrows("Container allowed appending to a sealed segment " + segmentName, context.container.append(segmentName, "foo".getBytes(), null, TIMEOUT)::join, ex -> ex instanceof StreamSegmentSealedException);
        } else {
            Assert.assertFalse("Segment is sealed when it shouldn't be " + segmentName, sp.isSealed());
            // Verify we can still append to these segments.
            byte[] appendData = "foo".getBytes();
            context.container.append(segmentName, appendData, null, TIMEOUT).join();
            segmentContents.get(segmentName).write(appendData);
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
        }
    }
    // 4. Reads (regular reads, not tail reads, and only for the sealed segments).
    waitForOperationsInReadIndex(context.container);
    for (int i = 0; i < segmentNames.size() / 2; i++) {
        String segmentName = segmentNames.get(i);
        long segmentLength = context.container.getStreamSegmentInfo(segmentName, false, TIMEOUT).join().getLength();
        // Read starting 1 byte from the end - make sure it wont hang at the end by turning into a future read.
        final int totalReadLength = 1;
        long expectedCurrentOffset = segmentLength - totalReadLength;
        @Cleanup ReadResult readResult = context.container.read(segmentName, expectedCurrentOffset, Integer.MAX_VALUE, TIMEOUT).join();
        int readLength = 0;
        while (readResult.hasNext()) {
            ReadResultEntry readEntry = readResult.next();
            if (readEntry.getStreamSegmentOffset() >= segmentLength) {
                Assert.assertEquals("Unexpected value for isEndOfStreamSegment when reaching the end of sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                AssertExtensions.assertThrows("ReadResultEntry.getContent() returned a result when reached the end of sealed segment " + segmentName, readEntry::getContent, ex -> ex instanceof IllegalStateException);
            } else {
                Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment before reaching end of sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType());
                Assert.assertTrue("getContent() did not return a completed future for segment" + segmentName, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally());
                ReadResultEntryContents readEntryContents = readEntry.getContent().join();
                expectedCurrentOffset += readEntryContents.getLength();
                readLength += readEntryContents.getLength();
            }
        }
        Assert.assertEquals("Unexpected number of bytes read.", totalReadLength, readLength);
        Assert.assertTrue("ReadResult was not closed when reaching the end of sealed segment" + segmentName, readResult.isClosed());
    }
    // 5. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
    context.container.stopAsync().awaitTerminated();
}

Example 14 with ReadResult

use of io.pravega.segmentstore.contracts.ReadResult in project pravega by pravega.

the class StreamSegmentContainerTests method testFutureReads.

/**
 * Tests the ability to perform future (tail) reads. Scenarios tested include:
 * * Regular appends
 * * Segment sealing
 * * Transaction merging.
 */
@Test
public void testFutureReads() throws Exception {
    final int nonSealReadLimit = 100;
    @Cleanup TestContext context = new TestContext();
    context.container.startAsync().awaitRunning();
    // 1. Create the StreamSegments.
    ArrayList<String> segmentNames = createSegments(context);
    HashMap<String, ArrayList<String>> transactionsBySegment = createTransactions(segmentNames, context);
    activateAllSegments(segmentNames, context);
    transactionsBySegment.values().forEach(s -> activateAllSegments(s, context));
    HashMap<String, ReadResult> readsBySegment = new HashMap<>();
    ArrayList<AsyncReadResultProcessor> readProcessors = new ArrayList<>();
    HashSet<String> segmentsToSeal = new HashSet<>();
    HashMap<String, ByteArrayOutputStream> readContents = new HashMap<>();
    HashMap<String, TestReadResultHandler> entryHandlers = new HashMap<>();
    // should stop upon reaching the limit).
    for (int i = 0; i < segmentNames.size(); i++) {
        String segmentName = segmentNames.get(i);
        ByteArrayOutputStream readContentsStream = new ByteArrayOutputStream();
        readContents.put(segmentName, readContentsStream);
        ReadResult readResult;
        if (i < segmentNames.size() / 2) {
            // We're going to seal this one at one point.
            segmentsToSeal.add(segmentName);
            readResult = context.container.read(segmentName, 0, Integer.MAX_VALUE, TIMEOUT).join();
        } else {
            // Just a regular one, nothing special.
            readResult = context.container.read(segmentName, 0, nonSealReadLimit, TIMEOUT).join();
        }
        // The Read callback is only accumulating data in this test; we will then compare it against the real data.
        TestReadResultHandler entryHandler = new TestReadResultHandler(readContentsStream, TIMEOUT);
        entryHandlers.put(segmentName, entryHandler);
        readsBySegment.put(segmentName, readResult);
        readProcessors.add(AsyncReadResultProcessor.process(readResult, entryHandler, executorService()));
    }
    // 3. Add some appends.
    HashMap<String, Long> lengths = new HashMap<>();
    HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>();
    appendToParentsAndTransactions(segmentNames, transactionsBySegment, lengths, segmentContents, context);
    // 4. Merge all the Transactions.
    mergeTransactions(transactionsBySegment, lengths, segmentContents, context);
    // 5. Add more appends (to the parent segments)
    ArrayList<CompletableFuture<Void>> operationFutures = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        for (String segmentName : segmentNames) {
            byte[] appendData = getAppendData(segmentName, APPENDS_PER_SEGMENT + i);
            operationFutures.add(context.container.append(segmentName, appendData, null, TIMEOUT));
            lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length);
            recordAppend(segmentName, appendData, segmentContents);
        }
    }
    segmentsToSeal.forEach(segmentName -> operationFutures.add(Futures.toVoid(context.container.sealStreamSegment(segmentName, TIMEOUT))));
    Futures.allOf(operationFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    // Now wait for all the reads to complete, and verify their results against the expected output.
    Futures.allOf(entryHandlers.values().stream().map(h -> h.getCompleted()).collect(Collectors.toList())).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    readProcessors.forEach(AsyncReadResultProcessor::close);
    // Check to see if any errors got thrown (and caught) during the reading process).
    for (Map.Entry<String, TestReadResultHandler> e : entryHandlers.entrySet()) {
        Throwable err = e.getValue().getError().get();
        if (err != null) {
            // The next check (see below) will verify if the segments were properly read).
            if (!(err instanceof StreamSegmentSealedException && segmentsToSeal.contains(e.getKey()))) {
                Assert.fail("Unexpected error happened while processing Segment " + e.getKey() + ": " + e.getValue().getError().get());
            }
        }
    }
    // Check that all the ReadResults are closed
    for (Map.Entry<String, ReadResult> e : readsBySegment.entrySet()) {
        Assert.assertTrue("Read result is not closed for segment " + e.getKey(), e.getValue().isClosed());
    }
    // Compare, byte-by-byte, the outcome of the tail reads.
    Assert.assertEquals("Unexpected number of segments were read.", segmentContents.size(), readContents.size());
    for (String segmentName : segmentNames) {
        boolean isSealed = segmentsToSeal.contains(segmentName);
        byte[] expectedData = segmentContents.get(segmentName).toByteArray();
        byte[] actualData = readContents.get(segmentName).toByteArray();
        int expectedLength = isSealed ? (int) (long) lengths.get(segmentName) : nonSealReadLimit;
        Assert.assertEquals("Unexpected read length for segment " + segmentName, expectedLength, actualData.length);
        AssertExtensions.assertArrayEquals("Unexpected read contents for segment " + segmentName, expectedData, 0, actualData, 0, actualData.length);
    }
    // 6. Writer moving data to Storage.
    waitForSegmentsInStorage(segmentNames, context).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    checkStorage(segmentContents, lengths, context);
}

Example 15 with ReadResult

use of io.pravega.segmentstore.contracts.ReadResult in project pravega by pravega.

the class StreamSegmentStoreTestBase method checkStorage.

private static void checkStorage(HashMap<String, ByteArrayOutputStream> segmentContents, StreamSegmentStore baseStore, StreamSegmentStore readOnlySegmentStore) throws Exception {
    for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) {
        String segmentName = e.getKey();
        byte[] expectedData = e.getValue().toByteArray();
        // 1. Deletion status
        SegmentProperties sp = null;
        try {
            sp = baseStore.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        } catch (Exception ex) {
            if (!(Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException)) {
                throw ex;
            }
        }
        if (sp == null) {
            AssertExtensions.assertThrows("Segment is marked as deleted in SegmentStore but was not deleted in Storage " + segmentName, () -> readOnlySegmentStore.getStreamSegmentInfo(segmentName, false, TIMEOUT), ex -> ex instanceof StreamSegmentNotExistsException);
            // No need to do other checks.
            continue;
        }
        // 2. Seal Status
        SegmentProperties storageProps = readOnlySegmentStore.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        Assert.assertEquals("Segment seal status disagree between Store and Storage for segment " + segmentName, sp.isSealed(), storageProps.isSealed());
        // 3. Contents.
        SegmentProperties metadataProps = baseStore.getStreamSegmentInfo(segmentName, false, TIMEOUT).join();
        Assert.assertEquals("Unexpected Storage length for segment " + segmentName, expectedData.length, storageProps.getLength());
        byte[] actualData = new byte[expectedData.length];
        int actualLength = 0;
        int expectedLength = actualData.length;
        try {
            @Cleanup ReadResult readResult = readOnlySegmentStore.read(segmentName, 0, actualData.length, TIMEOUT).join();
            actualLength = readResult.readRemaining(actualData, TIMEOUT);
        } catch (Exception ex) {
            ex = (Exception) Exceptions.unwrap(ex);
            if (!(ex instanceof StreamSegmentTruncatedException) || metadataProps.getStartOffset() == 0) {
                // We encountered an unexpected Exception, or a Truncated Segment which was not expected to be truncated.
                throw ex;
            }
            // Read from the truncated point, except if the whole segment got truncated.
            expectedLength = (int) (storageProps.getLength() - metadataProps.getStartOffset());
            if (metadataProps.getStartOffset() < storageProps.getLength()) {
                @Cleanup ReadResult readResult = readOnlySegmentStore.read(segmentName, metadataProps.getStartOffset(), expectedLength, TIMEOUT).join();
                actualLength = readResult.readRemaining(actualData, TIMEOUT);
            }
        }
        Assert.assertEquals("Unexpected number of bytes read from Storage for segment " + segmentName, expectedLength, actualLength);
        AssertExtensions.assertArrayEquals("Unexpected data written to storage for segment " + segmentName, expectedData, expectedData.length - expectedLength, actualData, 0, expectedLength);
    }
}