Examples with ByteArraySegment io.pravega.common.util.ByteArraySegment used on opensource projects

Example 86 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class RevisionDataStreamCommonTests method testByteArrays.

/**
 * Tests the ability to encode and decode {@link BufferView}s.
 */
@Test
public void testByteArrays() throws Exception {
    byte[] numbers = new byte[Byte.MAX_VALUE];
    for (int i = 0; i < numbers.length; i++) {
        numbers[i] = (byte) (i % Byte.MAX_VALUE);
    }
    val toTest = Arrays.<byte[]>asList(null, new byte[0], numbers);
    for (byte[] value : toTest) {
        // Raw byte arrays.
        testEncodeDecode(RevisionDataOutput::writeArray, RevisionDataInput::readArray, (s, v) -> s.getCollectionLength(v == null ? 0 : v.length, 1), value, (s, t) -> Arrays.equals(s == null ? new byte[0] : s, t));
        // Buffer Views.
        testEncodeDecode((RevisionDataOutputStream s, byte[] t) -> s.writeBuffer(t == null ? null : new ByteArraySegment(t)), RevisionDataInput::readArray, (s, v) -> s.getCollectionLength(v == null ? 0 : v.length, 1), value, (s, t) -> Arrays.equals(s == null ? new byte[0] : s, t));
    }
}

Example 87 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class EntryIterator method getNextLeafPage.

private CompletableFuture<PageWrapper> getNextLeafPage(TimeoutTimer timer) {
    // Walk up the parent chain as long as the page's Key is the last key in that parent key list.
    // Once we found a Page which has a next key, look up the first Leaf page that exists down that path.
    PageWrapper lastPage = this.lastPage.get();
    assert lastPage != null;
    int pageKeyPos;
    do {
        PageWrapper parentPage = lastPage.getParent();
        if (parentPage == null) {
            // We have reached the end. No more pages.
            return CompletableFuture.completedFuture(null);
        }
        // Look up the current page's PageKey in the parent and make note of its position.
        ByteArraySegment pageKey = lastPage.getPointer().getKey();
        val pos = parentPage.getPage().search(pageKey, 0);
        assert pos.isExactMatch() : "expecting exact match";
        pageKeyPos = pos.getPosition() + 1;
        // We no longer need this page. Remove it from the PageCollection.
        this.pageCollection.remove(lastPage);
        lastPage = parentPage;
    } while (pageKeyPos == lastPage.getPage().getCount());
    ByteArraySegment referenceKey = lastPage.getPage().getKeyAt(pageKeyPos);
    return this.locatePage.apply(referenceKey, this.pageCollection, timer);
}

Example 88 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class BTreePage method splitIfNecessary.

/**
 * If necessary, splits the contents of this BTreePage instance into multiple BTreePages. This instance will not be
 * modified as a result of this operation (all new BTreePages will be copies).
 *
 * The resulting pages will be about half full each, and when combined in order, they will contain the same elements
 * as this BTreePage, in the same order.
 *
 * Split Conditions:
 * * Length > MaxPageSize
 *
 * @return If a split is made, an ordered List of BTreePage instances. If no split is necessary (condition is not met),
 * returns null.
 */
List<BTreePage> splitIfNecessary() {
    if (this.contents.getLength() <= this.config.getMaxPageSize()) {
        // Nothing to do.
        return null;
    }
    // Calculate how many pages to split into. While doing so, take care to account that we may only have whole entries
    // in each page, and not partial ones.
    int maxDataLength = (this.config.getMaxPageSize() - this.header.getLength() - this.footer.getLength()) / this.config.entryLength * this.config.entryLength;
    int remainingPageCount = (int) Math.ceil((double) this.data.getLength() / maxDataLength);
    ArrayList<BTreePage> result = new ArrayList<>(remainingPageCount);
    int readIndex = 0;
    int remainingItems = getCount();
    while (remainingPageCount > 0) {
        // Calculate how many items to include in this split page. This is the average of the remaining items over
        // the remaining page count (this helps smooth out cases when the original getCount() is not divisible by
        // the calculated number of splits).
        int itemsPerPage = remainingItems / remainingPageCount;
        // Copy data over to the new page.
        ByteArraySegment splitPageData = this.data.slice(readIndex, itemsPerPage * this.config.entryLength);
        result.add(new BTreePage(this.config, itemsPerPage, splitPageData));
        // Update pointers.
        readIndex += splitPageData.getLength();
        remainingPageCount--;
        remainingItems -= itemsPerPage;
    }
    assert readIndex == this.data.getLength() : "did not copy everything";
    return result;
}

Example 89 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class BTreePage method applyInsertsAndRemovals.

/**
 * Inserts the new PageEntry instances at the given offsets.
 *
 * @param ci A {@link ChangeInfo} object containing information to change.
 * @return A new BTreePage instance with the updated contents.
 */
private BTreePage applyInsertsAndRemovals(ChangeInfo ci) {
    int newCount = getCount() + ci.insertCount - ci.deleteCount;
    // Allocate new buffer of the correct size and start copying from the old one.
    val newPage = new BTreePage(this.config, new ByteArraySegment(new byte[DATA_OFFSET + newCount * this.config.entryLength + FOOTER_LENGTH]), false);
    newPage.formatHeaderAndFooter(newCount, getHeaderId());
    int readIndex = 0;
    int writeIndex = 0;
    for (val e : ci.changes) {
        int entryIndex = e.getKey() * this.config.entryLength;
        if (entryIndex > readIndex) {
            // Copy from source.
            int length = entryIndex - readIndex;
            assert length % this.config.entryLength == 0;
            newPage.data.copyFrom(this.data, readIndex, writeIndex, length);
            writeIndex += length;
        }
        // Write new Entry.
        PageEntry entryContents = e.getValue();
        readIndex = entryIndex;
        if (entryContents != null) {
            // Insert new PageEntry.
            newPage.setEntryAtIndex(writeIndex, entryContents);
            writeIndex += this.config.entryLength;
        } else {
            // This PageEntry has been deleted. Skip over it.
            readIndex += this.config.getEntryLength();
        }
    }
    if (readIndex < this.data.getLength()) {
        // Copy the last part that we may have missed.
        int length = this.data.getLength() - readIndex;
        newPage.data.copyFrom(this.data, readIndex, writeIndex, length);
    }
    return newPage;
}

Example 90 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class BTreePage method applyUpdates.

/**
 * Updates (in-place) the contents of this BTreePage with the given entries for those Keys that already exist. For
 * all the new or deleted Keys, collects them into a List and calculates the offset where they would have to be
 * inserted at or removed from.
 *
 * @param entries A List of PageEntries to update, in sorted order by {@link PageEntry#getKey()}.
 * @return A {@link ChangeInfo} object.
 * @throws IllegalDataFormatException If any of the entries do not conform to the Key/Value size constraints.
 * @throws IllegalArgumentException   If the entries are not sorted by {@link PageEntry#getKey()}.
 */
private ChangeInfo applyUpdates(List<PageEntry> entries) {
    // Keep track of new keys to be added along with the offset (in the original page) where they would have belonged.
    val changes = new ArrayList<Map.Entry<Integer, PageEntry>>();
    int removeCount = 0;
    // Process all the Entries, in order (by Key).
    int lastPos = 0;
    ByteArraySegment lastKey = null;
    for (val e : entries) {
        if (e.getKey().getLength() != this.config.keyLength || (e.hasValue() && e.getValue().getLength() != this.config.valueLength)) {
            throw new IllegalDataFormatException("Found an entry with unexpected Key or Value length.");
        }
        if (lastKey != null) {
            Preconditions.checkArgument(KEY_COMPARATOR.compare(lastKey, e.getKey()) < 0, "Entries must be sorted by key and no duplicates are allowed.");
        }
        // Figure out if this entry exists already.
        val searchResult = search(e.getKey(), lastPos);
        if (searchResult.isExactMatch()) {
            if (e.hasValue()) {
                // Key already exists: update in-place.
                setValueAtPosition(searchResult.getPosition(), e.getValue());
            } else {
                // Key exists but this is a removal. Record it for later.
                changes.add(new AbstractMap.SimpleImmutableEntry<>(searchResult.getPosition(), null));
                removeCount++;
            }
        } else if (e.hasValue()) {
            // This entry's key does not exist and we want to insert it (we don't care if we want to delete an inexistent
            // key). We need to remember it for later. Since this was not an exact match, binary search returned the
            // position where it should have been.
            changes.add(new AbstractMap.SimpleImmutableEntry<>(searchResult.getPosition(), e));
        }
        // Remember the last position so we may resume the next search from there.
        lastPos = searchResult.getPosition();
        lastKey = e.getKey();
    }
    return new ChangeInfo(changes, changes.size() - removeCount, removeCount);
}