Examples with SegmentHandle io.pravega.segmentstore.storage.SegmentHandle used on opensource projects

Example 1 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class IdempotentStorageTestBase method testParallelWriteTwoHosts.

// endregion
// region synchronization unit tests
/**
 * This test case simulates two hosts writing at the same offset at the same time.
 */
@Test(timeout = 30000)
public void testParallelWriteTwoHosts() {
    String segmentName = "foo_write";
    int appendCount = 5;
    try (Storage s1 = createStorage();
        Storage s2 = createStorage()) {
        s1.initialize(DEFAULT_EPOCH);
        s1.create(segmentName, TIMEOUT).join();
        SegmentHandle writeHandle1 = s1.openWrite(segmentName).join();
        SegmentHandle writeHandle2 = s2.openWrite(segmentName).join();
        long offset = 0;
        byte[] writeData = String.format("Segment_%s_Append", segmentName).getBytes();
        for (int j = 0; j < appendCount; j++) {
            ByteArrayInputStream dataStream1 = new ByteArrayInputStream(writeData);
            ByteArrayInputStream dataStream2 = new ByteArrayInputStream(writeData);
            CompletableFuture f1 = s1.write(writeHandle1, offset, dataStream1, writeData.length, TIMEOUT);
            CompletableFuture f2 = s2.write(writeHandle2, offset, dataStream2, writeData.length, TIMEOUT);
            assertMayThrow("Write expected to complete OR throw BadOffsetException." + "threw an unexpected exception.", () -> CompletableFuture.allOf(f1, f2), ex -> ex instanceof BadOffsetException);
            // Make sure at least one operation is success.
            Assert.assertTrue("At least one of the two parallel writes should succeed.", !f1.isCompletedExceptionally() || !f2.isCompletedExceptionally());
            offset += writeData.length;
        }
        Assert.assertTrue("Writes at the same offset are expected to be idempotent.", s1.getStreamSegmentInfo(segmentName, TIMEOUT).join().getLength() == offset);
        offset = 0;
        byte[] readBuffer = new byte[writeData.length];
        for (int j = 0; j < appendCount; j++) {
            int bytesRead = s1.read(writeHandle1, j * readBuffer.length, readBuffer, 0, readBuffer.length, TIMEOUT).join();
            Assert.assertEquals(String.format("Unexpected number of bytes read from offset %d.", offset), readBuffer.length, bytesRead);
            AssertExtensions.assertArrayEquals(String.format("Unexpected read result from offset %d.", offset), readBuffer, 0, readBuffer, 0, bytesRead);
        }
        s1.delete(writeHandle1, TIMEOUT).join();
    }
}

Example 2 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class IdempotentStorageTestBase method testFencing.

// region Fencing tests
/**
 * Tests fencing abilities. We create two different Storage objects with different owner ids.
 * * We create the Segment on Storage1:
 * ** We verify that Storage1 can execute all operations.
 * ** We verify that Storage2 can execute only read-only operations.
 * ** We open the Segment on Storage2:
 * ** We verify that Storage1 can execute only read-only operations.
 * ** We verify that Storage2 can execute all operations.
 */
@Override
@Test(timeout = 30000)
public void testFencing() throws Exception {
    final long epoch1 = 1;
    final long epoch2 = 2;
    final String segmentName = "segment";
    try (val storage1 = createStorage();
        val storage2 = createStorage()) {
        storage1.initialize(epoch1);
        storage2.initialize(epoch2);
        // Create segment in Storage1 (thus Storage1 owns it for now).
        storage1.create(segmentName, TIMEOUT).join();
        // Storage1 should be able to execute all operations.
        SegmentHandle handle1 = storage1.openWrite(segmentName).join();
        verifyWriteOperationsSucceed(handle1, storage1);
        verifyReadOnlyOperationsSucceed(handle1, storage1);
        // Open the segment in Storage2 (thus Storage2 owns it for now).
        SegmentHandle handle2 = storage2.openWrite(segmentName).join();
        // Storage1 should be able to execute read-only operations.
        verifyReadOnlyOperationsSucceed(handle1, storage1);
        // Storage2 should be able to execute all operations.
        verifyReadOnlyOperationsSucceed(handle2, storage2);
        verifyWriteOperationsSucceed(handle2, storage2);
        // Seal and Delete (these should be run last, otherwise we can't run our test).
        verifyFinalWriteOperationsSucceed(handle2, storage2);
    }
}

Example 3 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class IdempotentStorageTestBase method testPartialConcat.

/**
 * This test case simulates host crashing during concat and retrying the operation.
 */
@Test(timeout = 30000)
public void testPartialConcat() {
    String segmentName = "foo_write";
    String concatSegmentName = "foo_concat";
    String newConcatSegmentName = "foo_concat0";
    int offset = 0;
    try (Storage s1 = createStorage()) {
        s1.initialize(DEFAULT_EPOCH);
        s1.create(segmentName, TIMEOUT).join();
        s1.create(concatSegmentName, TIMEOUT).join();
        SegmentHandle writeHandle1 = s1.openWrite(segmentName).join();
        SegmentHandle writeHandle2 = s1.openWrite(concatSegmentName).join();
        byte[] writeData = String.format("Segment_%s_Append", segmentName).getBytes();
        ByteArrayInputStream dataStream1 = new ByteArrayInputStream(writeData);
        ByteArrayInputStream dataStream2 = new ByteArrayInputStream(writeData);
        s1.write(writeHandle1, offset, dataStream1, writeData.length, TIMEOUT).join();
        s1.write(writeHandle2, offset, dataStream2, writeData.length, TIMEOUT).join();
        s1.seal(writeHandle2, TIMEOUT).join();
        // This will append the segments and delete the concat segment.
        s1.concat(writeHandle1, writeData.length, concatSegmentName, TIMEOUT).join();
        long lengthBeforeRetry = s1.getStreamSegmentInfo(segmentName, TIMEOUT).join().getLength();
        // Create the segment again.
        s1.create(newConcatSegmentName, TIMEOUT).join();
        writeHandle2 = s1.openWrite(newConcatSegmentName).join();
        dataStream2 = new ByteArrayInputStream(writeData);
        s1.write(writeHandle2, offset, dataStream2, writeData.length, TIMEOUT).join();
        s1.seal(writeHandle2, TIMEOUT).join();
        // Concat at the same offset again
        s1.concat(writeHandle1, writeData.length, newConcatSegmentName, TIMEOUT).join();
        long lengthAfterRetry = s1.getStreamSegmentInfo(segmentName, TIMEOUT).join().getLength();
        Assert.assertTrue(String.format("Concatenation of same segment at the same offset(%d) should result in " + "same segment size(%d), but is (%d)", writeData.length, lengthBeforeRetry, lengthAfterRetry), lengthBeforeRetry == lengthAfterRetry);
        // Verify the data
        byte[] readBuffer = new byte[writeData.length];
        for (int j = 0; j < 2; j++) {
            int bytesRead = s1.read(writeHandle1, j * readBuffer.length, readBuffer, 0, readBuffer.length, TIMEOUT).join();
            Assert.assertEquals(String.format("Unexpected number of bytes read from offset %d.", offset), readBuffer.length, bytesRead);
            AssertExtensions.assertArrayEquals(String.format("Unexpected read result from offset %d.", offset), readBuffer, (int) offset, readBuffer, 0, bytesRead);
        }
        s1.delete(writeHandle1, TIMEOUT).join();
    }
}

Example 4 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class ExtendedS3StorageTest method testMetrics.

@Test
public void testMetrics() throws Exception {
    assertEquals(0, ExtendedS3Metrics.CREATE_COUNT.get());
    assertEquals(0, ExtendedS3Metrics.CONCAT_COUNT.get());
    assertEquals(0, ExtendedS3Metrics.DELETE_COUNT.get());
    try (val storage = createStorage()) {
        // Create segment A
        storage.create("a", null).join();
        assertEquals(1, ExtendedS3Metrics.CREATE_COUNT.get());
        assertEquals(1, ExtendedS3Metrics.CREATE_LATENCY.toOpStatsData().getNumSuccessfulEvents());
        assertTrue(0 <= ExtendedS3Metrics.CREATE_LATENCY.toOpStatsData().getAvgLatencyMillis());
        // Create segment B
        storage.create("b", null).join();
        assertEquals(2, ExtendedS3Metrics.CREATE_COUNT.get());
        assertEquals(2, ExtendedS3Metrics.CREATE_LATENCY.toOpStatsData().getNumSuccessfulEvents());
        assertTrue(0 <= ExtendedS3Metrics.CREATE_LATENCY.toOpStatsData().getAvgLatencyMillis());
        SegmentHandle handleA = storage.openWrite("a").get();
        SegmentHandle handleB = storage.openWrite("b").get();
        // Write some data to A
        String str = "0123456789";
        int totalBytesWritten = 0;
        for (int i = 1; i < 5; i++) {
            try (BoundedInputStream bis = new BoundedInputStream(new ByteArrayInputStream(str.getBytes()), i)) {
                storage.write(handleA, totalBytesWritten, bis, i, null).join();
            }
            totalBytesWritten += i;
            assertEquals(totalBytesWritten, ExtendedS3Metrics.WRITE_BYTES.get());
            assertEquals(i, ExtendedS3Metrics.WRITE_LATENCY.toOpStatsData().getNumSuccessfulEvents());
            assertTrue(0 <= ExtendedS3Metrics.WRITE_LATENCY.toOpStatsData().getAvgLatencyMillis());
        }
        // Write some data to segment B
        storage.write(handleB, 0, new ByteArrayInputStream(str.getBytes()), str.length(), null).join();
        totalBytesWritten += str.length();
        assertEquals(totalBytesWritten, ExtendedS3Metrics.WRITE_BYTES.get());
        assertEquals(5, ExtendedS3Metrics.WRITE_LATENCY.toOpStatsData().getNumSuccessfulEvents());
        assertTrue(0 <= ExtendedS3Metrics.WRITE_LATENCY.toOpStatsData().getAvgLatencyMillis());
        // Read some data
        int totalBytesRead = 0;
        byte[] buffer = new byte[10];
        for (int i = 1; i < 5; i++) {
            totalBytesRead += storage.read(handleA, totalBytesRead, buffer, 0, i, null).join();
            assertEquals(totalBytesRead, ExtendedS3Metrics.READ_BYTES.get());
            assertEquals(i, ExtendedS3Metrics.READ_LATENCY.toOpStatsData().getNumSuccessfulEvents());
            assertTrue(0 <= ExtendedS3Metrics.READ_LATENCY.toOpStatsData().getAvgLatencyMillis());
        }
        // Concat
        SegmentProperties info = storage.getStreamSegmentInfo(handleA.getSegmentName(), null).join();
        storage.seal(handleB, null).join();
        storage.concat(handleA, info.getLength(), handleB.getSegmentName(), null).get();
        assertEquals(str.length(), ExtendedS3Metrics.CONCAT_BYTES.get());
        assertEquals(1, ExtendedS3Metrics.CONCAT_LATENCY.toOpStatsData().getNumSuccessfulEvents());
        assertTrue(0 <= ExtendedS3Metrics.CONCAT_LATENCY.toOpStatsData().getAvgLatencyMillis());
        // delete
        storage.delete(handleA, null).join();
        assertEquals(1, ExtendedS3Metrics.DELETE_COUNT.get());
        assertEquals(1, ExtendedS3Metrics.DELETE_LATENCY.toOpStatsData().getNumSuccessfulEvents());
        assertTrue(0 <= ExtendedS3Metrics.DELETE_LATENCY.toOpStatsData().getAvgLatencyMillis());
    }
}

Example 5 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class FileSystemMockTests method doReadTest.

private void doReadTest(int index, int bufferSize) throws Exception {
    // Set up mocks.
    FileChannel channel = mock(FileChannel.class);
    fixChannelMock(channel);
    String segmentName = "test";
    @Cleanup TestFileSystemStorage testStorage = new TestFileSystemStorage(storageConfig, channel);
    testStorage.setSizeToReturn(2L * bufferSize);
    SegmentHandle handle = FileSystemSegmentHandle.readHandle(segmentName);
    // Force two reads.
    ArgumentCaptor<Long> expectedArgs = ArgumentCaptor.forClass(Long.class);
    when(channel.read(any(), anyLong())).thenReturn(index, bufferSize - index);
    // Call method.
    byte[] buffer = new byte[bufferSize];
    testStorage.read(handle, 0, buffer, 0, bufferSize);
    // Verify.
    verify(channel, times(2)).read(any(), expectedArgs.capture());
    List<Long> actualArgs = expectedArgs.getAllValues();
    assertEquals(2, actualArgs.size());
    assertEquals(0, actualArgs.get(0).longValue());
    assertEquals(index, actualArgs.get(1).longValue());
}