Examples with SegmentHandle io.pravega.segmentstore.storage.SegmentHandle used on opensource projects

Example 41 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class StreamSegmentContainerTests method testTableSegmentReadAfterCompactionAndRecovery.

/**
 * Tests a non-trivial scenario in which ContainerKeyIndex may be tail-caching a stale version of a key if the
 * following conditions occur:
 * 1. StorageWriter processes values v0...vn for k1 and {@link WriterTableProcessor} indexes them.
 * 2. As a result of {@link WriterTableProcessor} activity, the last value vn for k1 is moved to the tail of the Segment.
 * 3. While TableCompactor works, a new PUT operation is appended to the Segment with new value vn+1 for k1.
 * 4. At this point, the StorageWriter stops its progress and the container restarts without processing neither the
 *    new value vn+1 nor the compacted value vn for k1.
 * 5. A subsequent restart will trigger the tail-caching from the last indexed offset, which points to vn+1.
 * 6. The bug, which consists of the tail-caching process not taking care of table entry versions, would overwrite
 *    vn+1 with vn, just because it has a higher offset as it was written later in the Segment.
 */
@Test
public void testTableSegmentReadAfterCompactionAndRecovery() throws Exception {
    @Cleanup TestContext context = new TestContext(DEFAULT_CONFIG, NO_TRUNCATIONS_DURABLE_LOG_CONFIG, DEFAULT_WRITER_CONFIG, null);
    val durableLog = new AtomicReference<OperationLog>();
    val durableLogFactory = new WatchableOperationLogFactory(context.operationLogFactory, durableLog::set);
    // Data size and count to be written in this test.
    int serializedEntryLength = 28;
    int writtenEntries = 7;
    @Cleanup StreamSegmentContainer container = new StreamSegmentContainer(CONTAINER_ID, DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    container.startAsync().awaitRunning();
    Assert.assertNotNull(durableLog.get());
    val tableStore = container.getExtension(ContainerTableExtension.class);
    // 1. Create the Table Segment and get a DirectSegmentAccess to it to monitor its size.
    String tableSegmentName = getSegmentName(0) + "_Table";
    val type = SegmentType.builder(getSegmentType(tableSegmentName)).tableSegment().build();
    tableStore.createSegment(tableSegmentName, type, TIMEOUT).join();
    DirectSegmentAccess directTableSegment = container.forSegment(tableSegmentName, TIMEOUT).join();
    // 2. Add some entries to the table segments. Note tha we write multiple values to each key, so the TableCompactor
    // can find entries to move to the tail.
    final BiFunction<String, Integer, TableEntry> createTableEntry = (key, value) -> TableEntry.unversioned(new ByteArraySegment(key.getBytes()), new ByteArraySegment(String.format("Value_%s", value).getBytes()));
    // 3. This callback will run when the StorageWriter writes data to Storage. At this point, StorageWriter would
    // have completed its first iteration, so it is the time to add a new value for key1 while TableCompactor is working.
    val compactedEntry = List.of(TableEntry.versioned(new ByteArraySegment("key1".getBytes(StandardCharsets.UTF_8)), new ByteArraySegment("3".getBytes(StandardCharsets.UTF_8)), serializedEntryLength * 2L));
    // Simulate that Table Compactor moves [k1, 3] to the tail of the Segment as a result of compacting the first 4 entries.
    val compactedEntryUpdate = EntrySerializerTests.generateUpdateWithExplicitVersion(compactedEntry);
    CompletableFuture<Void> callbackExecuted = new CompletableFuture<>();
    context.storageFactory.getPostWriteCallback().set((segmentHandle, offset) -> {
        if (segmentHandle.getSegmentName().contains("Segment_0_Table$attributes.index") && !callbackExecuted.isDone()) {
            // New PUT with the newest value.
            Futures.toVoid(tableStore.put(tableSegmentName, Collections.singletonList(createTableEntry.apply("key1", 4)), TIMEOUT)).join();
            // Simulates a compacted entry append performed by Table Compactor.
            directTableSegment.append(compactedEntryUpdate, null, TIMEOUT).join();
            callbackExecuted.complete(null);
        }
    });
    // Do the actual puts.
    Futures.toVoid(tableStore.put(tableSegmentName, Collections.singletonList(createTableEntry.apply("key1", 1)), TIMEOUT)).join();
    Futures.toVoid(tableStore.put(tableSegmentName, Collections.singletonList(createTableEntry.apply("key1", 2)), TIMEOUT)).join();
    Futures.toVoid(tableStore.put(tableSegmentName, Collections.singletonList(createTableEntry.apply("key1", 3)), TIMEOUT)).join();
    Futures.toVoid(tableStore.put(tableSegmentName, Collections.singletonList(createTableEntry.apply("key2", 1)), TIMEOUT)).join();
    Futures.toVoid(tableStore.put(tableSegmentName, Collections.singletonList(createTableEntry.apply("key2", 2)), TIMEOUT)).join();
    Futures.toVoid(tableStore.put(tableSegmentName, Collections.singletonList(createTableEntry.apply("key2", 3)), TIMEOUT)).join();
    // 4. Above, the test does 7 puts, each one 28 bytes in size (6 entries directly, 1 via callback). Now, we need
    // to wait for the TableCompactor writing the entry (key1, 3) to the tail of the Segment.
    callbackExecuted.join();
    AssertExtensions.assertEventuallyEquals(true, () -> directTableSegment.getInfo().getLength() > (long) serializedEntryLength * writtenEntries, 5000);
    // 5. The TableCompactor has moved the entry, so we immediately stop the container to prevent StorageWriter from
    // making more progress.
    container.close();
    // 6. Create a new container instance that will recover from existing data.
    @Cleanup val container2 = new StreamSegmentContainer(CONTAINER_ID, DEFAULT_CONFIG, durableLogFactory, context.readIndexFactory, context.attributeIndexFactory, context.writerFactory, context.storageFactory, context.getDefaultExtensions(), executorService());
    container2.startAsync().awaitRunning();
    // 7. Verify that (key1, 4) is the actual value after performing the tail-caching process, which now takes care
    // of entry versions.
    val expected = createTableEntry.apply("key1", 4);
    val tableStore2 = container2.getExtension(ContainerTableExtension.class);
    val actual = tableStore2.get(tableSegmentName, Collections.singletonList(expected.getKey().getKey()), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).get(0);
    Assert.assertEquals(actual.getKey().getKey(), expected.getKey().getKey());
    Assert.assertEquals(actual.getValue(), expected.getValue());
}

Example 42 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class RollingStorage method truncate.

@Override
public void truncate(SegmentHandle handle, long truncationOffset) throws StreamSegmentException {
    // Delete all SegmentChunks which are entirely before the truncation offset.
    RollingSegmentHandle h = getHandle(handle);
    ensureNotDeleted(h);
    // The only acceptable case where we allow a read-only handle is if the Segment is sealed, since openWrite() will
    // only return a read-only handle in that case.
    Preconditions.checkArgument(h.isSealed() || !h.isReadOnly(), "Can only truncate with a read-only handle if the Segment is Sealed.");
    if (h.getHeaderHandle() == null) {
        // No header means the Segment is made up of a single SegmentChunk. We can't do anything.
        return;
    }
    long traceId = LoggerHelpers.traceEnter(log, "truncate", h, truncationOffset);
    Preconditions.checkArgument(truncationOffset >= 0 && truncationOffset <= h.length(), "truncationOffset must be non-negative and at most the length of the Segment.");
    val last = h.lastChunk();
    boolean chunksDeleted;
    if (last != null && canTruncate(last, truncationOffset) && !h.isSealed()) {
        // If we were asked to truncate the entire (non-sealed) Segment, then rollover at this point so we can delete
        // all existing data.
        rollover(h);
        // We are free to delete all chunks.
        chunksDeleted = deleteChunks(h, s -> canTruncate(s, truncationOffset));
    } else {
        // Either we were asked not to truncate the whole segment, or we were, and the Segment is sealed. If the latter,
        // then the Header is also sealed, we could not have done a quick rollover; as such we have no option but to
        // preserve the last chunk so that we can recalculate the length of the Segment if we need it again.
        chunksDeleted = deleteChunks(h, s -> canTruncate(s, truncationOffset) && s.getLastOffset() < h.length());
    }
    // Try to truncate the handle if we can.
    if (chunksDeleted && this.headerStorage.supportsReplace()) {
        truncateHandle(h);
    }
    LoggerHelpers.traceLeave(log, "truncate", traceId, h, truncationOffset);
}

Example 43 with SegmentHandle

use of io.pravega.segmentstore.storage.SegmentHandle in project pravega by pravega.

the class RollingStorage method delete.

@Override
public void delete(SegmentHandle handle) throws StreamSegmentException {
    val h = getHandle(handle);
    long traceId = LoggerHelpers.traceEnter(log, "delete", handle);
    SegmentHandle headerHandle = h.getHeaderHandle();
    if (headerHandle == null) {
        // Directly delete the only SegmentChunk, and bubble up any exceptions if it doesn't exist.
        val subHandle = this.baseStorage.openWrite(h.lastChunk().getName());
        try {
            this.baseStorage.delete(subHandle);
            h.lastChunk().markInexistent();
            h.markDeleted();
        } catch (StreamSegmentNotExistsException ex) {
            h.lastChunk().markInexistent();
            h.markDeleted();
            throw ex;
        }
    } else {
        // them, after which we delete all SegmentChunks and finally the header file.
        if (!h.isSealed()) {
            val writeHandle = h.isReadOnly() ? (RollingSegmentHandle) openWrite(handle.getSegmentName()) : h;
            seal(writeHandle);
        }
        deleteChunks(h, s -> true);
        try {
            this.headerStorage.delete(headerHandle);
            h.markDeleted();
        } catch (StreamSegmentNotExistsException ex) {
            h.markDeleted();
            throw ex;
        }
    }
    LoggerHelpers.traceLeave(log, "delete", traceId, handle);
}