Examples with BufferView io.pravega.common.util.BufferView used on opensource projects

Example 31 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class StreamSegmentReadIndex method insertEntriesToCacheAndIndex.

/**
 * Inserts data in the index.
 *
 * @param data          A {@link BufferView} representing the data to insert.
 * @param segmentOffset The segment offset that maps to the first byte in the given {@link BufferView}.
 * @return A {@link CacheIndexEntry} representing the index entry added. If the given {@link BufferView} spanned
 * multiple entries (due to index fragmentation), only the last {@link CacheIndexEntry} is added.
 */
private CacheIndexEntry insertEntriesToCacheAndIndex(BufferView data, long segmentOffset) {
    CacheIndexEntry lastInsertedEntry = null;
    synchronized (this.lock) {
        // Do not insert after we have closed the index, otherwise we will leak cache entries.
        Exceptions.checkNotClosed(this.closed, this);
        while (data != null && data.getLength() > 0) {
            // Figure out if the first byte in the buffer is already cached.
            ReadIndexEntry existingEntry = this.indexEntries.getFloor(segmentOffset);
            long overlapLength;
            if (existingEntry != null && existingEntry.getLastStreamSegmentOffset() >= segmentOffset) {
                // First offset exists already. We need to skip over to the end of this entry.
                overlapLength = existingEntry.getStreamSegmentOffset() + existingEntry.getLength() - segmentOffset;
            } else {
                // First offset does not exist. Let's find out how much we can insert.
                existingEntry = this.indexEntries.getCeiling(segmentOffset);
                overlapLength = existingEntry == null ? data.getLength() : existingEntry.getStreamSegmentOffset() - segmentOffset;
                assert overlapLength > 0 : "indexEntries.getFloor(offset) == null != indexEntries.getCeiling(offset)";
                // Slice the data that we need to insert. We may be able to insert the whole buffer at once.
                BufferView dataToInsert = overlapLength >= data.getLength() ? data : data.slice(0, (int) overlapLength);
                CacheIndexEntry newEntry;
                // Null address pointer.
                int dataAddress = CacheStorage.NO_ADDRESS;
                try {
                    dataAddress = this.cacheStorage.insert(dataToInsert);
                    newEntry = new CacheIndexEntry(segmentOffset, dataToInsert.getLength(), dataAddress);
                    ReadIndexEntry overriddenEntry = addToIndex(newEntry);
                    assert overriddenEntry == null : "Insert overrode existing entry; " + segmentOffset + ":" + dataToInsert.getLength();
                    lastInsertedEntry = newEntry;
                } catch (Throwable ex) {
                    // Clean up the data we might have inserted if we were unable to add it to the index.
                    this.cacheStorage.delete(dataAddress);
                    throw ex;
                }
            }
            // Slice the remainder of the buffer, or set it to null if we processed everything.
            assert overlapLength != 0 : "unable to make any progress";
            data = overlapLength >= data.getLength() ? null : data.slice((int) overlapLength, data.getLength() - (int) overlapLength);
            segmentOffset += overlapLength;
        }
    }
    return lastInsertedEntry;
}

Example 32 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class StreamSegmentReadIndex method readDirect.

/**
 * Reads a contiguous sequence of bytes of the given length starting at the given offset. Every byte in the range
 * must meet the following conditions:
 * <ul>
 * <li> It must exist in this segment. This excludes bytes from merged transactions and future reads.
 * <li> It must be part of data that is not yet committed to Storage (tail part) - as such, it must be fully in the cache.
 * </ul>
 * Note: This method will not cause cache statistics to be updated. As such, Cache entry generations will not be
 * updated for those entries that are touched.
 *
 * @param startOffset The offset in the StreamSegment where to start reading.
 * @param length      The number of bytes to read.
 * @return An InputStream containing the requested data, or null if all of the conditions of this read cannot be met.
 * @throws IllegalStateException    If the read index is in recovery mode.
 * @throws IllegalArgumentException If the parameters are invalid (offset, length or offset+length are not in the Segment's range).
 */
BufferView readDirect(long startOffset, int length) {
    Exceptions.checkNotClosed(this.closed, this);
    Preconditions.checkState(!this.recoveryMode, "StreamSegmentReadIndex is in Recovery Mode.");
    Preconditions.checkArgument(length >= 0, "length must be a non-negative number");
    Preconditions.checkArgument(startOffset >= this.metadata.getStorageLength(), "[%s]: startOffset (%s) must refer to an offset beyond the Segment's StorageLength offset(%s).", this.traceObjectId, startOffset, this.metadata.getStorageLength());
    Preconditions.checkArgument(startOffset + length <= this.metadata.getLength(), "startOffset+length must be less than the length of the Segment.");
    Preconditions.checkArgument(startOffset >= Math.min(this.metadata.getStartOffset(), this.metadata.getStorageLength()), "startOffset is before the Segment's StartOffset.");
    // Get the first entry. This one is trickier because the requested start offset may not fall on an entry boundary.
    CompletableReadResultEntry nextEntry;
    synchronized (this.lock) {
        ReadIndexEntry indexEntry = this.indexEntries.getFloor(startOffset);
        if (indexEntry == null || startOffset > indexEntry.getLastStreamSegmentOffset() || !indexEntry.isDataEntry()) {
            // Data not available or data exist in a partially merged transaction.
            return null;
        } else {
            // Fetch data from the cache for the first entry, but do not update the cache hit stats.
            nextEntry = createMemoryRead(indexEntry, startOffset, length, false, false);
        }
    }
    // Since we know all entries should be in the cache and are contiguous, there is no need
    assert Futures.isSuccessful(nextEntry.getContent()) : "Found CacheReadResultEntry that is not completed yet: " + nextEntry;
    val entryContents = nextEntry.getContent().join();
    ArrayList<BufferView> contents = new ArrayList<>();
    contents.add(entryContents);
    int readLength = entryContents.getLength();
    while (readLength < length) {
        BufferView entryData;
        synchronized (this.lock) {
            ReadIndexEntry indexEntry = this.indexEntries.get(startOffset + readLength);
            if (!indexEntry.isDataEntry()) {
                // This cache entry refers to a merged segment (into this one). As per the contract, we shouldn't return it.
                return null;
            }
            entryData = this.cacheStorage.get(indexEntry.getCacheAddress());
        }
        if (entryData == null) {
            // Could not find the 'next' cache entry: this means the requested range is not fully cached.
            return null;
        }
        int entryReadLength = Math.min(entryData.getLength(), length - readLength);
        assert entryReadLength > 0 : "about to have fetched zero bytes from a cache entry";
        contents.add(entryData.slice(0, entryReadLength));
        readLength += entryReadLength;
    }
    return BufferView.wrap(contents);
}

Example 33 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class StreamSegmentReadIndex method triggerFutureReads.

/**
 * Triggers all the Future Reads in the given collection.
 *
 * @param futureReads The Future Reads to trigger.
 */
private void triggerFutureReads(Collection<FutureReadResultEntry> futureReads) {
    for (FutureReadResultEntry r : futureReads) {
        ReadResultEntry entry = getSingleReadResultEntry(r.getStreamSegmentOffset(), r.getRequestedReadLength(), false);
        assert entry != null : "Serving a FutureReadResultEntry with a null result";
        if (entry instanceof FutureReadResultEntry) {
            // The new FutureReadResultEntries will be completed when the rest of the appends are processed.
            assert entry.getStreamSegmentOffset() == r.getStreamSegmentOffset();
            log.warn("{}: triggerFutureReads (Offset = {}). Serving a FutureReadResultEntry ({}) with another FutureReadResultEntry ({}). Segment Info = [{}].", this.traceObjectId, r.getStreamSegmentOffset(), r, entry, this.metadata.getSnapshot());
        }
        log.debug("{}: triggerFutureReads (Offset = {}, Type = {}).", this.traceObjectId, r.getStreamSegmentOffset(), entry.getType());
        if (entry.getType() == ReadResultEntryType.EndOfStreamSegment) {
            // We have attempted to read beyond the end of the stream. Fail the read request with the appropriate message.
            r.fail(new StreamSegmentSealedException(String.format("StreamSegment has been sealed at offset %d. There can be no more reads beyond this offset.", this.metadata.getLength())));
        } else {
            if (!entry.getContent().isDone()) {
                // Normally, all Future Reads are served from Cache, since they reflect data that has just been appended.
                // However, it's possible that after recovery, we get a read for some data that we do not have in the
                // cache (but it's not a tail read) - this data exists in Storage but our StorageLength has not yet been
                // updated. As such, the only solution we have is to return a FutureRead which will be satisfied when
                // the Writer updates the StorageLength (and trigger future reads). In that scenario, entry we get
                // will likely not be auto-fetched, so we need to request the content.
                entry.requestContent(this.config.getStorageReadDefaultTimeout());
            }
            CompletableFuture<BufferView> entryContent = entry.getContent();
            entryContent.thenAccept(r::complete);
            Futures.exceptionListener(entryContent, r::fail);
        }
    }
}

Example 34 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class HashTableSegmentLayout method get.

private CompletableFuture<List<TableEntry>> get(DirectSegmentAccess segment, GetResultBuilder builder, Map<UUID, Long> bucketOffsets, TimeoutTimer timer) {
    val bucketReader = TableBucketReader.entry(segment, this.keyIndex::getBackpointerOffset, this.executor);
    int resultSize = builder.getHashes().size();
    for (int i = 0; i < resultSize; i++) {
        UUID keyHash = builder.getHashes().get(i);
        long offset = bucketOffsets.get(keyHash);
        if (offset == TableKey.NOT_EXISTS) {
            // Bucket does not exist, hence neither does the key.
            builder.includeResult(CompletableFuture.completedFuture(null));
        } else {
            // Find the sought entry in the segment, based on its key.
            BufferView key = builder.getKeys().get(i);
            builder.includeResult(this.keyIndex.findBucketEntry(segment, bucketReader, key, offset, timer).thenApply(this::maybeDeleted));
        }
    }
    return builder.getResultFutures();
}

Example 35 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class HashTableSegmentLayout method newIterator.

private <T> CompletableFuture<AsyncIterator<IteratorItem<T>>> newIterator(@NonNull DirectSegmentAccess segment, @NonNull IteratorArgs args, @NonNull GetBucketReader<T> createBucketReader) {
    Preconditions.checkArgument(args.getFrom() == null && args.getTo() == null, "Range Iterators not supported for HashTableSegments.");
    UUID fromHash;
    BufferView serializedState = args.getContinuationToken();
    try {
        fromHash = KeyHasher.getNextHash(serializedState == null ? null : IteratorStateImpl.deserialize(serializedState).getKeyHash());
    } catch (IOException ex) {
        // Bad IteratorState serialization.
        throw new IllegalDataFormatException("Unable to deserialize `serializedState`.", ex);
    }
    if (fromHash == null) {
        // Nothing to iterate on.
        return CompletableFuture.completedFuture(TableIterator.empty());
    }
    // Create a converter that will use a TableBucketReader to fetch all requested items in the iterated Buckets.
    val bucketReader = createBucketReader.apply(segment, this.keyIndex::getBackpointerOffset, this.executor);
    TableIterator.ConvertResult<IteratorItem<T>> converter = bucket -> bucketReader.findAllExisting(bucket.getSegmentOffset(), new TimeoutTimer(args.getFetchTimeout())).thenApply(result -> new IteratorItemImpl<>(new IteratorStateImpl(bucket.getHash()).serialize(), result));
    // Fetch the Tail (Unindexed) Hashes, then create the TableIterator.
    return this.keyIndex.getUnindexedKeyHashes(segment).thenComposeAsync(cacheHashes -> TableIterator.<IteratorItem<T>>builder().segment(segment).cacheHashes(cacheHashes).firstHash(fromHash).executor(executor).resultConverter(converter).fetchTimeout(args.getFetchTimeout()).build(), this.executor);
}