Examples with ByteArraySegment io.pravega.common.util.ByteArraySegment used on opensource projects

Example 21 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class WriteQueueTests method testGetWritesToExecute.

/**
 * Tests the getWritesToExecute() method.
 */
@Test
public void testGetWritesToExecute() {
    final int ledgerChangeIndex = ITEM_COUNT - 5;
    val q = new WriteQueue();
    val writes = new ArrayList<Write>();
    int ledgerId = 0;
    for (int i = 0; i < ITEM_COUNT; i++) {
        if (i == ledgerChangeIndex) {
            ledgerId++;
        }
        val w = new Write(new ByteArraySegment(new byte[i]), new TestWriteLedger(ledgerId), new CompletableFuture<>());
        q.add(w);
        writes.add(w);
    }
    // 1. Max size reached.
    int sizeLimit = 10;
    val maxSizeResult = q.getWritesToExecute(sizeLimit);
    val expectedMaxSizeResult = new ArrayList<Write>();
    for (Write w : writes) {
        if (w.data.getLength() > sizeLimit) {
            break;
        }
        sizeLimit -= w.data.getLength();
        expectedMaxSizeResult.add(w);
    }
    AssertExtensions.assertListEquals("Unexpected writes fetched with size limit.", expectedMaxSizeResult, maxSizeResult, Object::equals);
    // 2. Complete a few writes, then mark a few as in progress.
    writes.get(0).setEntryId(0);
    writes.get(0).complete();
    writes.get(1).beginAttempt();
    val result1 = q.getWritesToExecute(Long.MAX_VALUE);
    // We expect to skip over the first one and second one, but count the second one when doing throttling.
    AssertExtensions.assertListEquals("Unexpected writes fetched when some writes in progress (at beginning).", writes.subList(2, ledgerChangeIndex), result1, Object::equals);
    // 3. Mark a few writes as in progress after a non-progress write.
    writes.get(3).beginAttempt();
    val result2 = q.getWritesToExecute(Long.MAX_VALUE);
    Assert.assertEquals("Unexpected writes fetched when in-progress writes exist after non-in-progress writes.", 0, result2.size());
    // 4. LedgerChange.
    int beginIndex = ledgerChangeIndex - 5;
    for (int i = 0; i < beginIndex; i++) {
        writes.get(i).setEntryId(i);
        writes.get(i).complete();
    }
    q.removeFinishedWrites();
    val result3 = q.getWritesToExecute(Long.MAX_VALUE);
    AssertExtensions.assertListEquals("Unexpected writes fetched when ledger changed.", writes.subList(beginIndex, ledgerChangeIndex), result3, Object::equals);
    result3.forEach(w -> w.setEntryId(0));
    result3.forEach(Write::complete);
    q.removeFinishedWrites();
    val result4 = q.getWritesToExecute(Long.MAX_VALUE);
    AssertExtensions.assertListEquals("Unexpected writes fetched from the end, after ledger changed.", writes.subList(ledgerChangeIndex, writes.size()), result4, Object::equals);
}

Example 22 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class WriteQueueTests method testAdd.

/**
 * Tests the basic functionality of the add() method.
 */
@Test
public void testAdd() {
    // Just over 1ms.
    final int timeIncrement = 1234 * 1000;
    AtomicLong time = new AtomicLong();
    val q = new WriteQueue(time::get);
    val initialStats = q.getStatistics();
    Assert.assertEquals("Unexpected getSize on empty queue.", 0, initialStats.getSize());
    Assert.assertEquals("Unexpected getAverageFillRate on empty queue.", 0, initialStats.getAverageItemFillRatio(), 0);
    Assert.assertEquals("Unexpected getExpectedProcessingTimeMillis on empty queue.", 0, initialStats.getExpectedProcessingTimeMillis());
    int expectedSize = 0;
    long firstItemTime = 0;
    for (int i = 0; i < ITEM_COUNT; i++) {
        time.addAndGet(timeIncrement);
        if (i == 0) {
            firstItemTime = time.get();
        }
        int writeSize = i * 10000;
        q.add(new Write(new ByteArraySegment(new byte[writeSize]), new TestWriteLedger(i), CompletableFuture.completedFuture(null)));
        expectedSize += writeSize;
        val stats = q.getStatistics();
        val expectedFillRatio = (double) expectedSize / stats.getSize() / BookKeeperConfig.MAX_APPEND_LENGTH;
        val expectedProcTime = (time.get() - firstItemTime) / AbstractTimer.NANOS_TO_MILLIS;
        Assert.assertEquals("Unexpected getSize.", i + 1, stats.getSize());
        Assert.assertEquals("Unexpected getAverageFillRate.", expectedFillRatio, stats.getAverageItemFillRatio(), 0.01);
        Assert.assertEquals("Unexpected getExpectedProcessingTimeMillis.", expectedProcTime, stats.getExpectedProcessingTimeMillis());
    }
}

Example 23 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class WriteQueueTests method testClose.

/**
 * Tests the close() method.
 */
@Test
public void testClose() {
    val q = new WriteQueue();
    val expectedWrites = new ArrayList<Write>();
    for (int i = 0; i < ITEM_COUNT; i++) {
        val w = new Write(new ByteArraySegment(new byte[i]), new TestWriteLedger(i), CompletableFuture.completedFuture(null));
        q.add(w);
        expectedWrites.add(w);
    }
    val removedWrites = q.close();
    AssertExtensions.assertListEquals("Unexpected writes removed.", expectedWrites, removedWrites, Object::equals);
    val clearStats = q.getStatistics();
    Assert.assertEquals("Unexpected getSize after clear.", 0, clearStats.getSize());
    Assert.assertEquals("Unexpected getAverageFillRate after clear.", 0, clearStats.getAverageItemFillRatio(), 0);
    Assert.assertEquals("Unexpected getExpectedProcessingTimeMillis after clear.", 0, clearStats.getExpectedProcessingTimeMillis());
    AssertExtensions.assertThrows("add() worked after close().", () -> q.add(new Write(new ByteArraySegment(new byte[1]), new TestWriteLedger(0), CompletableFuture.completedFuture(null))), ex -> ex instanceof ObjectClosedException);
    AssertExtensions.assertThrows("getWritesToExecute() worked after close().", () -> q.getWritesToExecute(1), ex -> ex instanceof ObjectClosedException);
    AssertExtensions.assertThrows("removeFinishedWrites() worked after close().", q::removeFinishedWrites, ex -> ex instanceof ObjectClosedException);
}

Example 24 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DurableDataLogTestBase method testAppendParallel.

/**
 * Tests the ability to append to a DurableDataLog using parallel writes.
 *
 * @throws Exception If one got thrown.
 */
@Test
public void testAppendParallel() throws Exception {
    try (DurableDataLog log = createDurableDataLog()) {
        log.initialize(TIMEOUT);
        // Only verify sequence number monotonicity. We'll verify reads in its own test.
        List<CompletableFuture<LogAddress>> appendFutures = new ArrayList<>();
        int writeCount = getWriteCount();
        for (int i = 0; i < writeCount; i++) {
            appendFutures.add(log.append(new ByteArraySegment(getWriteData()), TIMEOUT));
        }
        val results = Futures.allOfWithResults(appendFutures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        for (int i = 0; i < results.size(); i++) {
            LogAddress address = results.get(i);
            Assert.assertNotNull("No address returned from append() for index " + i, address);
            if (i > 0) {
                AssertExtensions.assertGreaterThan("Sequence Number is not monotonically increasing.", results.get(i - 1).getSequence(), address.getSequence());
            }
        }
    }
}

Example 25 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DurableDataLogTestBase method testOpenCloseClient.

/**
 * Tests the ability to reuse the client after closing and reopening it. Verifies the major operations (append,
 * truncate and read), each operation using a different client (each client is used for exactly one operation).
 *
 * @throws Exception If one got thrown.
 */
@Test
public void testOpenCloseClient() throws Exception {
    // This is a very repetitive test; and we only care about "recovery" from no client; all else is already tested.
    final int writeCount = 10;
    TreeMap<LogAddress, byte[]> writtenData = new TreeMap<>(Comparator.comparingLong(LogAddress::getSequence));
    Object context = createSharedContext();
    LogAddress previousAddress = null;
    for (int i = 0; i < writeCount; i++) {
        // Write one entry at each iteration.
        LogAddress currentAddress;
        try (DurableDataLog log = createDurableDataLog(context)) {
            log.initialize(TIMEOUT);
            byte[] writeData = String.format("Write_%s", i).getBytes();
            currentAddress = log.append(new ByteArraySegment(writeData), TIMEOUT).join();
            writtenData.put(currentAddress, writeData);
        }
        // Truncate up to the previous iteration's entry, if any.
        if (previousAddress != null) {
            try (DurableDataLog log = createDurableDataLog(context)) {
                log.initialize(TIMEOUT);
                log.truncate(previousAddress, TIMEOUT).join();
                writtenData.headMap(previousAddress, true).clear();
            }
        }
        // Verify reads.
        try (DurableDataLog log = createDurableDataLog(context)) {
            log.initialize(TIMEOUT);
            verifyReads(log, writtenData);
        }
        previousAddress = currentAddress;
    }
}