Examples with ByteArraySegment io.pravega.common.util.ByteArraySegment used on opensource projects

Example 16 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DataFrameOutputStreamTests method testWriteSingleBytes.

/**
 * Tests the ability to write records using single-byte writes only.
 */
@Test
public void testWriteSingleBytes() throws Exception {
    // Very small frame, so we can test switching over to new frames.
    int maxFrameSize = 512;
    // This should generate enough records that cross over boundaries.
    ArrayList<byte[]> records = DataFrameTestHelpers.generateRecords(10, 0, 10240);
    // Callback for when a frame is written.
    ArrayList<DataFrame> writtenFrames = new ArrayList<>();
    try (DataFrameOutputStream s = new DataFrameOutputStream(maxFrameSize, writtenFrames::add)) {
        // Write each record, one byte at a time.
        for (byte[] record : records) {
            s.startNewRecord();
            for (byte b : record) {
                s.write(b);
            }
            s.endRecord();
        }
        // Seal whatever is left at the end.
        s.flush();
    }
    AssertExtensions.assertGreaterThan("No frame has been created during the test.", 0, writtenFrames.size());
    val readFrames = writtenFrames.stream().map(this::readFrame).collect(Collectors.toList());
    DataFrameTestHelpers.checkReadRecords(readFrames, records, ByteArraySegment::new);
}

Example 17 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DataFrameOutputStreamTests method testCommitFailure.

/**
 * Tests the behavior of startNewRecord(), write(byte) and write(byte[]) when the commit callback throws an exception.
 */
@Test
public void testCommitFailure() throws Exception {
    int maxFrameSize = 50;
    // Callback for when a frame is written. If we need to throw an exception, do it; otherwise just remember the frame.
    AtomicReference<DataFrame> writtenFrame = new AtomicReference<>();
    AtomicBoolean throwException = new AtomicBoolean();
    Consumer<DataFrame> callback = df -> {
        if (throwException.get()) {
            throw new IntentionalException();
        }
        writtenFrame.set(df);
    };
    // Test #1: write(byte)
    AtomicInteger usableSpace = new AtomicInteger();
    ByteArrayOutputStream writtenData1 = new ByteArrayOutputStream();
    try (DataFrameOutputStream s = new DataFrameOutputStream(maxFrameSize, callback)) {
        // 1. Call write(byte) until it fails. Check that the correct exception is thrown.
        s.startNewRecord();
        throwException.set(true);
        AssertExtensions.assertThrows("write() did not throw when the commit callback threw an exception.", () -> {
            for (int i = 0; i < maxFrameSize; i++) {
                s.write((byte) usableSpace.get());
                writtenData1.write((byte) usableSpace.get());
                usableSpace.incrementAndGet();
            }
        }, ex -> ex instanceof IntentionalException);
        // 2. Call write(byte) again and verify it fails. But this should fail because the DataFrame is sealed
        // (it was sealed prior to the current commit attempt).
        AssertExtensions.assertThrows("write() did not throw when the frame was sealed post-commit failure.", () -> s.write((byte) 1), ex -> ex instanceof IllegalStateException);
        // 3. Allow the commit to succeed. Verify a frame has been committed with the correct content.
        throwException.set(false);
        s.flush();
        Assert.assertNotNull("No frame has been created when a frame was filled.", writtenFrame.get());
        ArrayList<byte[]> records = new ArrayList<>();
        records.add(writtenData1.toByteArray());
        DataFrameTestHelpers.checkReadRecords(readFrame(writtenFrame.get()), records, ByteArraySegment::new);
    }
    // Test #2: startNewRecord()
    ByteArrayOutputStream writtenData2 = new ByteArrayOutputStream();
    writtenFrame.set(null);
    try (DataFrameOutputStream s = new DataFrameOutputStream(maxFrameSize, callback)) {
        // 1. Call write(byte) until we fill up the frame
        throwException.set(false);
        s.startNewRecord();
        for (int i = 0; i < usableSpace.get(); i++) {
            s.write((byte) i);
            writtenData2.write((byte) i);
        }
        // 2. Call startNewRecord(). This should fail because it will try to commit the frame.
        throwException.set(true);
        AssertExtensions.assertThrows("startNewRecord() did not throw when the commit callback threw an exception.", s::startNewRecord, ex -> ex instanceof IntentionalException);
        // 3. Allow the commit to succeed. Verify a frame has been committed with the correct content.
        throwException.set(false);
        s.flush();
        Assert.assertNotNull("No frame has been created when a frame was filled.", writtenFrame.get());
        ArrayList<byte[]> records = new ArrayList<>();
        records.add(writtenData2.toByteArray());
        DataFrameTestHelpers.checkReadRecords(readFrame(writtenFrame.get()), records, ByteArraySegment::new);
    }
    // Test #3: write(byte[])
    writtenFrame.set(null);
    try (DataFrameOutputStream s = new DataFrameOutputStream(maxFrameSize, callback)) {
        // 1. Call write(byte) until we fill up the frame
        throwException.set(false);
        s.startNewRecord();
        for (int i = 0; i < usableSpace.get(); i++) {
            // writtenData2 already contains this.
            s.write((byte) i);
        }
        // 2. Call write(byte[]). This should fail because it will try to commit the frame.
        throwException.set(true);
        AssertExtensions.assertThrows("write(byte[]) did not throw when the commit callback threw an exception.", () -> s.write(new byte[10]), ex -> ex instanceof IntentionalException);
        // 3. Allow the commit to succeed. Verify a frame has been committed with the correct content.
        throwException.set(false);
        s.flush();
        Assert.assertNotNull("No frame has been created when a frame was filled.", writtenFrame.get());
        ArrayList<byte[]> records = new ArrayList<>();
        records.add(writtenData2.toByteArray());
        DataFrameTestHelpers.checkReadRecords(readFrame(writtenFrame.get()), records, ByteArraySegment::new);
    }
}

Example 18 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DataFrameTests method testSerialization.

/**
 * Tests the ability to append a set of records to a DataFrame, serialize it, deserialize it, and then read those
 * records back.
 */
@Test
public void testSerialization() throws Exception {
    int maxFrameSize = 2 * 1024 * 1024;
    int maxRecordCount = 4500;
    int minRecordSize = 0;
    int maxRecordSize = 1024;
    List<ByteArraySegment> allRecords = DataFrameTestHelpers.generateRecords(maxRecordCount, minRecordSize, maxRecordSize, ByteArraySegment::new);
    // Append some records.
    DataFrame writeFrame = DataFrame.ofSize(maxFrameSize);
    int recordsAppended = appendRecords(allRecords, writeFrame);
    AssertExtensions.assertGreaterThan("Did not append enough records. Test may not be valid.", allRecords.size() / 2, recordsAppended);
    writeFrame.seal();
    val frameData = writeFrame.getData();
    Assert.assertEquals("Unexpected length from getData().", writeFrame.getLength(), frameData.getLength());
    // Read them back, by deserializing the frame.
    val contents = DataFrame.read(frameData.getReader(), frameData.getLength(), writeFrame.getAddress());
    DataFrameTestHelpers.checkReadRecords(contents, allRecords, b -> b);
}

Example 19 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class BookKeeperLogTests method testAppendPermanentFailures.

/**
 * Tests the ability to retry writes when Bookies fail.
 */
@Test
public void testAppendPermanentFailures() throws Exception {
    try (DurableDataLog log = createDurableDataLog()) {
        log.initialize(TIMEOUT);
        List<CompletableFuture<LogAddress>> appendFutures = new ArrayList<>();
        try {
            // Suspend a bookie (this will trigger write errors).
            stopFirstBookie();
            // Issue appends in parallel.
            int writeCount = getWriteCount();
            for (int i = 0; i < writeCount; i++) {
                appendFutures.add(log.append(new ByteArraySegment(getWriteData()), TIMEOUT));
            }
            // Verify that all writes failed or got cancelled.
            AtomicBoolean cancellationEncountered = new AtomicBoolean(false);
            for (val f : appendFutures) {
                AssertExtensions.assertThrows("Write did not fail correctly.", () -> f.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS), ex -> {
                    cancellationEncountered.set(cancellationEncountered.get() || ex instanceof CancellationException);
                    if (cancellationEncountered.get()) {
                        return ex instanceof CancellationException;
                    } else {
                        return ex instanceof RetriesExhaustedException || ex instanceof DurableDataLogException;
                    }
                });
            }
        } finally {
            // Don't forget to resume the bookie, but only AFTER we are done testing.
            restartFirstBookie();
        }
    }
}

Example 20 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class BookKeeperLogTests method testAppendTransientBookieFailure.

/**
 * Tests the ability to retry writes when Bookies fail.
 */
@Test
public void testAppendTransientBookieFailure() throws Exception {
    TreeMap<LogAddress, byte[]> writeData = new TreeMap<>(Comparator.comparingLong(LogAddress::getSequence));
    try (DurableDataLog log = createDurableDataLog()) {
        log.initialize(TIMEOUT);
        val dataList = new ArrayList<byte[]>();
        val futures = new ArrayList<CompletableFuture<LogAddress>>();
        try {
            // Suspend a bookie (this will trigger write errors).
            stopFirstBookie();
            // Issue appends in parallel, without waiting for them.
            int writeCount = getWriteCount();
            for (int i = 0; i < writeCount; i++) {
                byte[] data = getWriteData();
                futures.add(log.append(new ByteArraySegment(data), TIMEOUT));
                dataList.add(data);
            }
        } finally {
            // Resume the bookie with the appends still in flight.
            restartFirstBookie();
        }
        // Wait for all writes to complete, then reassemble the data in the order set by LogAddress.
        val addresses = Futures.allOfWithResults(futures).join();
        for (int i = 0; i < dataList.size(); i++) {
            writeData.put(addresses.get(i), dataList.get(i));
        }
    }
    // Verify data.
    try (DurableDataLog log = createDurableDataLog()) {
        log.initialize(TIMEOUT);
        verifyReads(log, writeData);
    }
}