Examples with SegmentHandle io.pravega.segmentstore.storage.SegmentHandle used on opensource projects

Example 16 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class SimpleStorageTests method testGetInfo.

/**
 * Tests general GetInfoOperation behavior.
 */
@Test
public void testGetInfo() throws Exception {
    String segmentName = "foo_open";
    try (Storage s = createStorage()) {
        s.initialize(DEFAULT_EPOCH);
        createSegment(segmentName, s);
        SegmentHandle handle = s.openWrite(segmentName).join();
        long expectedLength = 0;
        for (int i = 0; i < WRITE_COUNT; i++) {
            byte[] data = new byte[i + 1];
            s.write(handle, expectedLength, new ByteArrayInputStream(data), data.length, null).join();
            expectedLength += data.length;
        }
        SegmentProperties result = s.getStreamSegmentInfo(segmentName, null).join();
        validateProperties("pre-seal", segmentName, result, expectedLength, false);
        // Seal.
        s.seal(handle, null).join();
        result = s.getStreamSegmentInfo(segmentName, null).join();
        validateProperties("post-seal", segmentName, result, expectedLength, true);
        // Inexistent segment.
        AssertExtensions.assertFutureThrows("GetInfo succeeded on missing segment.", s.getStreamSegmentInfo("non-existent", null), ex -> ex instanceof StreamSegmentNotExistsException);
    }
}

Example 17 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class SimpleStorageTests method testConsecutiveReads.

@Test
public void testConsecutiveReads() throws Exception {
    String segmentName = createSegmentName("testConsecutiveReads");
    try (Storage s = createStorage()) {
        s.initialize(DEFAULT_EPOCH);
        createSegment(segmentName, s);
        Assert.assertTrue("Expected the segment to exist.", s.exists(segmentName, null).join());
        SegmentHandle writeHandle = s.openWrite(segmentName).join();
        Assert.assertEquals(segmentName, writeHandle.getSegmentName());
        Assert.assertEquals(false, writeHandle.isReadOnly());
        byte[] writeBuffer = new byte[15];
        populate(writeBuffer);
        s.write(writeHandle, 0, new ByteArrayInputStream(writeBuffer), writeBuffer.length, null).join();
        SegmentHandle readHandle = s.openRead(segmentName).join();
        Assert.assertEquals(segmentName, readHandle.getSegmentName());
        Assert.assertEquals(true, readHandle.isReadOnly());
        byte[] readBuffer = new byte[writeBuffer.length];
        int totalBytesRead = 0;
        for (int i = 1; i <= 5; i++) {
            int remaining = i;
            while (remaining > 0) {
                int bytesRead = s.read(readHandle, totalBytesRead, readBuffer, totalBytesRead, remaining, null).get();
                remaining -= bytesRead;
                totalBytesRead += bytesRead;
            }
        }
        Assert.assertEquals(writeBuffer.length, totalBytesRead);
        Assert.assertArrayEquals(writeBuffer, readBuffer);
    }
}

Example 18 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class SystemJournalTests method testSimpleBootstrapWithMultipleFailoversWithTruncate.

/**
 * Tests a scenario when there are multiple fail overs overs.
 * The test adds a few chunks to the system segments and then fails over.
 * After fail over the zombie instances continue to write junk data to both system segment and journal file.
 * The new instance should read the journal log file and recreate the layout of system segments.
 *
 * @throws Exception Throws exception in case of any error.
 */
@Test
public void testSimpleBootstrapWithMultipleFailoversWithTruncate() throws Exception {
    @Cleanup CleanupHelper cleanupHelper = new CleanupHelper();
    @Cleanup ChunkStorage chunkStorage = getChunkStorage();
    int containerId = 42;
    String systemSegmentName = SystemJournal.getChunkStorageSystemSegments(containerId)[0];
    long epoch = 0;
    val policy = new SegmentRollingPolicy(1024);
    val config = getDefaultConfigBuilder(policy).build();
    val data = new InMemorySnapshotInfoStore();
    val snapshotInfoStore = new SnapshotInfoStore(containerId, snapshotId -> data.setSnapshotId(containerId, snapshotId), () -> data.getSnapshotId(containerId));
    long offset = 0;
    long offsetToTruncateAt = 0;
    ChunkedSegmentStorage oldChunkedSegmentStorage = null;
    SegmentHandle oldHandle = null;
    long oldOffset = 0;
    for (int i = 1; i < 10; i++) {
        // Epoch 2
        epoch++;
        ChunkMetadataStore metadataStoreAfterCrash = getMetadataStore();
        cleanupHelper.add(metadataStoreAfterCrash);
        ChunkedSegmentStorage segmentStorageInLoop = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreAfterCrash, executorService(), config);
        cleanupHelper.add(segmentStorageInLoop);
        segmentStorageInLoop.initialize(epoch);
        segmentStorageInLoop.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
        segmentStorageInLoop.bootstrap(snapshotInfoStore, null).join();
        deleteGarbage(segmentStorageInLoop);
        checkSystemSegmentsLayout(segmentStorageInLoop);
        val h = segmentStorageInLoop.openWrite(systemSegmentName).join();
        if (null != oldChunkedSegmentStorage) {
            // Add some junk to previous instance after failover
            oldChunkedSegmentStorage.write(oldHandle, oldOffset, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
        } else {
            // Only first time.
            for (int j = 1; j < 10; j++) {
                val b0 = "JUNK".getBytes();
                segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b0), b0.length, null).join();
                offset += b0.length;
            }
        }
        offsetToTruncateAt += 4;
        val b1 = "Test".getBytes();
        segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b1), b1.length, null).join();
        offset += b1.length;
        val b2 = Integer.toString(i).getBytes();
        segmentStorageInLoop.write(h, offset, new ByteArrayInputStream(b2), b2.length, null).join();
        offset += b2.length;
        segmentStorageInLoop.truncate(h, offsetToTruncateAt, null).join();
        TestUtils.checkSegmentBounds(metadataStoreAfterCrash, h.getSegmentName(), offsetToTruncateAt, offset);
        val length = Math.toIntExact(offset - offsetToTruncateAt);
        byte[] readBytes = new byte[length];
        val bytesRead = segmentStorageInLoop.read(h, offsetToTruncateAt, readBytes, 0, length, null).get();
        Assert.assertEquals(length, readBytes.length);
        String s = new String(readBytes);
        // Add some garbage
        if (null != oldChunkedSegmentStorage) {
            oldChunkedSegmentStorage.write(oldHandle, oldOffset + 4, new ByteArrayInputStream("junk".getBytes()), 4, null).join();
        }
        // Save these instances so that you can write some junk after bootstrap.
        oldChunkedSegmentStorage = segmentStorageInLoop;
        oldHandle = h;
        oldOffset = offset;
    }
    epoch++;
    @Cleanup ChunkMetadataStore metadataStoreFinal = getMetadataStore();
    ChunkedSegmentStorage segmentStorageFinal = new ChunkedSegmentStorage(containerId, chunkStorage, metadataStoreFinal, executorService(), config);
    cleanupHelper.add(segmentStorageFinal);
    segmentStorageFinal.initialize(epoch);
    segmentStorageFinal.getGarbageCollector().initialize(new InMemoryTaskQueueManager()).join();
    segmentStorageFinal.bootstrap(snapshotInfoStore, null).join();
    deleteGarbage(segmentStorageFinal);
    checkSystemSegmentsLayout(segmentStorageFinal);
    val info = segmentStorageFinal.getStreamSegmentInfo(systemSegmentName, null).join();
    Assert.assertEquals(offset, info.getLength());
    Assert.assertEquals(offsetToTruncateAt, info.getStartOffset());
    byte[] out = new byte[Math.toIntExact(offset - offsetToTruncateAt)];
    val hr = segmentStorageFinal.openRead(systemSegmentName).join();
    segmentStorageFinal.read(hr, offsetToTruncateAt, out, 0, Math.toIntExact(offset - offsetToTruncateAt), null).join();
    val expected = "Test1Test2Test3Test4Test5Test6Test7Test8Test9";
    val actual = new String(out);
    Assert.assertEquals(expected, actual);
}

Example 19 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class ChunkedSegmentStorage method delete.

@Override
public CompletableFuture<Void> delete(SegmentHandle handle, Duration timeout) {
    checkInitialized();
    if (null == handle) {
        return CompletableFuture.failedFuture(new IllegalArgumentException("handle must not be null"));
    }
    return executeSerialized(() -> {
        val traceId = LoggerHelpers.traceEnter(log, "delete", handle);
        log.debug("{} delete - started segment={}.", logPrefix, handle.getSegmentName());
        val timer = new Timer();
        val streamSegmentName = handle.getSegmentName();
        return tryWith(metadataStore.beginTransaction(false, streamSegmentName), txn -> txn.get(streamSegmentName).thenComposeAsync(storageMetadata -> {
            val segmentMetadata = (SegmentMetadata) storageMetadata;
            // Check preconditions
            checkSegmentExists(streamSegmentName, segmentMetadata);
            checkOwnership(streamSegmentName, segmentMetadata);
            segmentMetadata.setActive(false);
            txn.update(segmentMetadata);
            // Collect garbage
            return garbageCollector.addSegmentToGarbage(txn.getVersion(), streamSegmentName).thenComposeAsync(vv -> {
                // Commit metadata.
                return txn.commit().thenRunAsync(() -> {
                    // Update the read index.
                    readIndexCache.remove(streamSegmentName);
                    val elapsed = timer.getElapsed();
                    SLTS_DELETE_LATENCY.reportSuccessEvent(elapsed);
                    SLTS_DELETE_COUNT.inc();
                    log.debug("{} delete - finished segment={}, latency={}.", logPrefix, handle.getSegmentName(), elapsed.toMillis());
                    LoggerHelpers.traceLeave(log, "delete", traceId, handle);
                }, executor);
            }, executor);
        }, executor), executor).exceptionally(ex -> {
            log.warn("{} delete - exception segment={}, latency={}.", logPrefix, handle.getSegmentName(), timer.getElapsedMillis(), ex);
            handleException(streamSegmentName, ex);
            return null;
        });
    }, handle.getSegmentName());
}

Example 20 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class InMemoryStorageTests method testFencing.

@Test
@Override
public void testFencing() throws Exception {
    final String segment1 = "segment1";
    final String segment2 = "segment2";
    @Cleanup val baseStorage = new InMemoryStorage();
    @Cleanup val storage = new AsyncStorageWrapper(baseStorage, executorService());
    storage.initialize(DEFAULT_EPOCH);
    // Part 1: Create a segment and verify all operations are allowed.
    storage.create(segment1, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentHandle handle1 = storage.openWrite(segment1).join();
    verifyAllOperationsSucceed(handle1, storage);
    // Part 2: Change owner, verify segment operations are not allowed until a call to open() is made.
    baseStorage.changeOwner();
    verifyWriteOperationsFail(handle1, storage);
    handle1 = storage.openWrite(segment1).join();
    verifyAllOperationsSucceed(handle1, storage);
    // Part 3: Create new segment and verify all operations are allowed.
    storage.create(segment2, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    SegmentHandle handle2 = storage.openWrite(segment2).join();
    verifyAllOperationsSucceed(handle2, storage);
    // Cleanup.
    storage.delete(handle1, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    storage.delete(handle2, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
}