Examples with ByteArraySegment io.pravega.common.util.ByteArraySegment used on opensource projects

Example 56 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DirectMemoryBufferTests method testDelete.

/**
 * Tests the ability to delete.
 */
@Test
public void testDelete() {
    final byte[] toWrite = new byte[LAYOUT.bufferSize()];
    rnd.nextBytes(toWrite);
    final int multiBlockLength = LAYOUT.blockSize() + 1;
    @Cleanup val b = newBuffer();
    int expectedUsedBlockCount = 1;
    val empty = b.write(new ByteArraySegment(toWrite, 0, 0), CacheLayout.NO_ADDRESS);
    expectedUsedBlockCount += 1;
    val multiBlock = b.write(new ByteArraySegment(toWrite, 0, multiBlockLength), CacheLayout.NO_ADDRESS);
    expectedUsedBlockCount += 2;
    val predecessorAddress = LAYOUT.calculateAddress(BUFFER_ID + 1, 2);
    val multiBuf = b.write(new ByteArraySegment(toWrite, 0, 2 * LAYOUT.blockSize()), predecessorAddress);
    expectedUsedBlockCount += 2;
    val emptyDelete = b.delete(LAYOUT.getBlockId(empty.getLastBlockAddress()));
    Assert.assertEquals(0, emptyDelete.getDeletedLength());
    Assert.assertEquals(CacheLayout.NO_ADDRESS, emptyDelete.getPredecessorAddress());
    expectedUsedBlockCount -= 1;
    Assert.assertEquals(expectedUsedBlockCount, b.getUsedBlockCount());
    val multiBlockDelete = b.delete(LAYOUT.getBlockId(multiBlock.getLastBlockAddress()));
    Assert.assertEquals(multiBlockLength, multiBlockDelete.getDeletedLength());
    Assert.assertEquals(CacheLayout.NO_ADDRESS, multiBlockDelete.getPredecessorAddress());
    expectedUsedBlockCount -= 2;
    Assert.assertEquals(expectedUsedBlockCount, b.getUsedBlockCount());
    val multiBufDelete = b.delete(LAYOUT.getBlockId(multiBuf.getLastBlockAddress()));
    Assert.assertEquals(2 * LAYOUT.blockSize(), multiBufDelete.getDeletedLength());
    Assert.assertEquals(predecessorAddress, multiBufDelete.getPredecessorAddress());
    expectedUsedBlockCount -= 2;
    Assert.assertEquals(expectedUsedBlockCount, b.getUsedBlockCount());
}

Example 57 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DirectMemoryBufferTests method testFreeBlockManagement.

/**
 * Tests the ability to properly allocate and deallocate Buffer-Blocks and re-link empty blocks together. This tests
 * the working of the private method {@link DirectMemoryBuffer#deallocateBlocks}.
 *
 * This test begins by filling up all the buffer with single-block entries, then deleting every other one. Next it
 * fills the leftover capacity with two-block entries, and again deletes every other entry (of all, not just the new
 * ones). This is repeated until we make the largest possible write (all blocks at once).
 *
 * At each step the allocated blocks are checked against a parallel-maintained structure whose purpose is to validate
 * the buffer is used as expected and that we do not lose free blocks once we remove entries.
 */
@Test
public void testFreeBlockManagement() {
    // Excluding metadata block.
    final int maxWriteBlockSize = LAYOUT.blocksPerBuffer() - 1;
    final byte[] data = new byte[maxWriteBlockSize * LAYOUT.blockSize()];
    // Keep a sorted list of free blocks.
    val freeBlocks = new ArrayList<Integer>();
    IntStream.range(1, LAYOUT.blocksPerBuffer()).forEach(freeBlocks::add);
    freeBlocks.sort(Comparator.reverseOrder());
    // val writtenEntries = new HashMap<Integer, WriteEntry>(); // Key=Starting Block Id.
    val writtenEntries = new PriorityQueue<WriteEntry>(Comparator.comparingInt(WriteEntry::getFirstBlockId));
    @Cleanup val b = newBuffer();
    // until we make writes equalling the entire buffer length.
    for (int writeBlocks = 1; writeBlocks <= maxWriteBlockSize; writeBlocks++) {
        int writeLength = writeBlocks * LAYOUT.blockSize();
        // Fill up the buffer with writes of this size.
        while (!freeBlocks.isEmpty()) {
            Assert.assertTrue("Expecting buffer to have capacity left.", b.hasCapacity());
            val w = b.write(new ByteArraySegment(data, 0, writeLength), CacheLayout.NO_ADDRESS);
            int firstBlockId = w.getFirstBlockId();
            int lastBlockId = LAYOUT.getBlockId(w.getLastBlockAddress());
            // Determine how many blocks were actually expected and validate.
            val expectedBlocks = updateFreeBlocksAfterWriting(freeBlocks, writeBlocks);
            Assert.assertEquals("Unexpected First Block Id.", expectedBlocks.getFirstBlockId(), firstBlockId);
            Assert.assertEquals("Unexpected Last Block Id.", expectedBlocks.getLastBlockId(), lastBlockId);
            Assert.assertEquals("Unexpected value from getUsedBlockCount.", LAYOUT.blocksPerBuffer() - freeBlocks.size(), b.getUsedBlockCount());
            writtenEntries.add(expectedBlocks);
        }
        // Delete every 1 other.
        boolean delete = true;
        val entryIterator = writtenEntries.iterator();
        while (entryIterator.hasNext()) {
            WriteEntry e = entryIterator.next();
            if (delete) {
                val d = b.delete(e.getLastBlockId());
                Assert.assertEquals("Expected whole blocks to be deleted.", 0, d.getDeletedLength() % LAYOUT.blockSize());
                Assert.assertEquals("Unexpected number of blocks deleted.", e.blockIds.size(), d.getDeletedLength() / LAYOUT.blockSize());
                updateFreeBlocksAfterDeletion(freeBlocks, e);
                entryIterator.remove();
            }
            delete = !delete;
        }
    }
}

Example 58 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DirectMemoryBufferTests method testWriteError.

/**
 * Tests the ability to handle write errors and rollback if necessary. This indirectly verifies the functionality of
 * the private method {@link DirectMemoryBuffer#rollbackWrite}.
 */
@Test
public void testWriteError() {
    final byte[] toWrite = new byte[3 * LAYOUT.blockSize()];
    rnd.nextBytes(toWrite);
    @Cleanup val b = newBuffer();
    int initialUsedCount = b.getUsedBlockCount();
    AssertExtensions.assertThrows("Expecting ArrayIndexOutOfBoundsException.", () -> b.write(new CorruptedByteArraySegment(toWrite, toWrite.length + 2), CacheLayout.NO_ADDRESS), ex -> ex instanceof ArrayIndexOutOfBoundsException);
    // Verify the getUsedBlockCount is unaffected.
    Assert.assertEquals("Unexpected used block count after rollback.", initialUsedCount, b.getUsedBlockCount());
    // Verify the rollback has properly re-chained the free blocks.
    for (int i = 1; i < LAYOUT.blocksPerBuffer(); i++) {
        val w = b.write(new ByteArraySegment(toWrite, 0, LAYOUT.blockSize()), CacheLayout.NO_ADDRESS);
        Assert.assertEquals("Unexpected block to write (free block re-chaining).", i, w.getFirstBlockId());
        Assert.assertEquals("Expected to have written only 1 block.", w.getFirstBlockId(), LAYOUT.getBlockId(w.getLastBlockAddress()));
    }
}

Example 59 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class BenchmarkTests method testRandomOperations.

private RandomResult testRandomOperations(CacheStorage s) {
    val writeBuffer = new ByteArraySegment(new byte[ENTRY_SIZE]);
    this.random.nextBytes(writeBuffer.array());
    val ids = new ArrayList<Integer>();
    val timer = new Timer();
    val threads = new ArrayList<Thread>();
    val iterations = new AtomicInteger(0);
    val insertCount = new AtomicInteger(0);
    val getCount = new AtomicInteger(0);
    val deleteCount = new AtomicInteger(0);
    for (int threadId = 0; threadId < RANDOM_OPERATIONS_THREAD_COUNT; threadId++) {
        val t = new Thread(() -> {
            val readBuffer = new byte[ENTRY_SIZE * 2];
            int i;
            while ((i = iterations.incrementAndGet()) <= ENTRY_COUNT) {
                boolean insert;
                int length;
                synchronized (ids) {
                    insert = (this.random.nextInt(100) < RANDOM_OPERATIONS_INSERT_PERCENTAGE) || ids.isEmpty();
                    length = insert ? this.random.nextInt(writeBuffer.getLength()) : 0;
                }
                if (insert) {
                    int insertedId = s.insert(writeBuffer.slice(0, length));
                    synchronized (ids) {
                        ids.add(insertedId);
                    }
                    insertCount.incrementAndGet();
                } else {
                    // delete
                    int toRemove;
                    synchronized (ids) {
                        toRemove = ids.remove(this.random.nextInt(ids.size()));
                    }
                    s.delete(toRemove);
                    deleteCount.incrementAndGet();
                }
                int toRead = -1;
                synchronized (ids) {
                    if (!ids.isEmpty()) {
                        toRead = ids.get(this.random.nextInt(ids.size()));
                    }
                }
                if (toRead >= 0) {
                    BufferView result = s.get(toRead);
                    if (result != null) {
                        result.copyTo(ByteBuffer.wrap(readBuffer));
                    }
                    getCount.incrementAndGet();
                }
            }
        });
        t.start();
        threads.add(t);
    }
    for (val t : threads) {
        Exceptions.handleInterrupted(t::join);
    }
    Duration elapsed = timer.getElapsed();
    // do not count this.
    ids.forEach(s::delete);
    return new RandomResult(elapsed, insertCount.get(), getCount.get(), deleteCount.get());
}

Example 60 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DirectMemoryCacheTests method testRandomOperations.

/**
 * Tests the ability to execute operations in random order. At each step we will either be inserting, appending or
 * deleting, and we will always be reading. We will not overfill the cache.
 */
@Test
public void testRandomOperations() {
    final byte[] data = new byte[LAYOUT.bufferSize() * 2];
    final int iterations = 200;
    @Cleanup val c = new TestCache();
    val addresses = new ArrayList<Integer>();
    // Key=Adddress, Value={StartOffset, Length}.
    val contents = new HashMap<Integer, Map.Entry<Integer, Integer>>();
    long storedBytes = 0;
    for (int i = 0; i < iterations; i++) {
        // Add with 60% probability, but only if we have capacity or are empty.
        val s = c.getState();
        val freeBytes = s.getMaxBytes() - s.getUsedBytes();
        boolean add = freeBytes > 0 && rnd.nextInt(100) < 60 || addresses.isEmpty();
        if (add) {
            // Write a new entry of arbitrary length.
            int offset = rnd.nextInt(data.length - 1);
            int length = (int) Math.min(freeBytes, rnd.nextInt(data.length - offset));
            val address = c.insert(new ByteArraySegment(data, offset, length));
            storedBytes += length;
            addresses.add(address);
            contents.put(address, new AbstractMap.SimpleImmutableEntry<>(offset, length));
        } else {
            // Pick an arbitrary entry and remove it.
            int address = addresses.remove(rnd.nextInt(addresses.size()));
            val length = contents.remove(address).getValue();
            c.delete(address);
            storedBytes -= length;
            Assert.assertNull("Entry was still accessible after deletion.", c.get(address));
        }
        checkSnapshot(c, storedBytes, null, null, null, null);
    }
    checkData(c, contents, data);
}