Examples with ByteStreamWriter io.pravega.client.byteStream.ByteStreamWriter used on opensource projects

Example 1 with ByteStreamWriter

use of io.pravega.client.byteStream.ByteStreamWriter in project pravega by pravega.

the class ByteClientTest method byteClientTest.

/**
 * This test verifies the correctness of basic read/write functionality of {@link ByteStreamReader} and {@link ByteStreamWriter}.
 */
@Test
public void byteClientTest() throws IOException {
    log.info("byteClientTest:: with security enabled: {}", Utils.AUTH_ENABLED);
    log.info("Invoking byteClientTest test with Controller URI: {}", controllerURI);
    @Cleanup ByteStreamClientFactory byteStreamClient = createClientFactory(SCOPE);
    @Cleanup("closeAndSeal") ByteStreamWriter writer = byteStreamClient.createByteStreamWriter(STREAM);
    @Cleanup ByteStreamReader reader = byteStreamClient.createByteStreamReader(STREAM);
    for (int i = 1; i <= MAX_PAYLOAD_SIZE; i *= 10) {
        final int payloadSize = i;
        // Create the synthetic payload for the write.
        byte[] payload = new byte[payloadSize];
        byte[] readBuffer = new byte[payloadSize];
        randomFactory.nextBytes(payload);
        final int payloadHashCode = Arrays.hashCode(payload);
        log.info("Created synthetic payload of size {} with hashcode {}.", payload.length, payloadHashCode);
        AtomicInteger writerIterations = new AtomicInteger();
        AtomicInteger readerIterations = new AtomicInteger();
        // Write the same synthetic payload multiple times to the Stream.
        CompletableFuture<Void> writerLoop = Futures.loop(() -> writerIterations.get() < IO_ITERATIONS, () -> CompletableFuture.runAsync(() -> {
            try {
                log.debug("Writing payload of size: {}. Iteration {}.", payload.length, writerIterations.get());
                writer.write(payload);
                if (writerIterations.get() % 2 == 0) {
                    log.debug("Flushing write.");
                    writer.flush();
                }
            } catch (IOException e) {
                throw new CompletionException(e);
            }
            writerIterations.incrementAndGet();
        }, WRITER_EXECUTOR), WRITER_EXECUTOR);
        // Read the written data with a read buffer of the same size than the payload and check that read data is correct.
        CompletableFuture<Void> readerLoop = Futures.loop(() -> readerIterations.get() < IO_ITERATIONS, () -> CompletableFuture.runAsync(() -> {
            try {
                int offset = 0;
                while (offset < payloadSize) {
                    offset += reader.read(readBuffer, offset, readBuffer.length - offset);
                    log.debug("Reading data of size: {}. Iteration {}.", offset, readerIterations.get());
                }
                Assert.assertEquals("Read data differs from data written.", payloadHashCode, Arrays.hashCode(readBuffer));
            } catch (IOException e) {
                throw new CompletionException(e);
            }
            readerIterations.incrementAndGet();
        }, READER_EXECUTOR), READER_EXECUTOR);
        writerLoop.join();
        readerLoop.join();
    }
    log.debug("Data correctly written/read from Stream: byte client test passed.");
}

Example 2 with ByteStreamWriter

use of io.pravega.client.byteStream.ByteStreamWriter in project pravega by pravega.

the class ByteStreamTest method testBlockingRead.

@Test(timeout = 30000)
public void testBlockingRead() throws IOException {
    String scope = "ByteStreamTest";
    String stream = "testBlockingRead";
    StreamConfiguration config = StreamConfiguration.builder().build();
    @Cleanup StreamManager streamManager = new StreamManagerImpl(PRAVEGA.getLocalController(), Mockito.mock(ConnectionPool.class));
    // create a scope
    Boolean createScopeStatus = streamManager.createScope(scope);
    log.info("Create scope status {}", createScopeStatus);
    // create a stream
    Boolean createStreamStatus = streamManager.createStream(scope, stream, config);
    log.info("Create stream status {}", createStreamStatus);
    @Cleanup ByteStreamClientFactory client = createClientFactory(scope);
    byte[] payload = new byte[100];
    Arrays.fill(payload, (byte) 1);
    byte[] readBuffer = new byte[200];
    Arrays.fill(readBuffer, (byte) 0);
    @Cleanup ByteStreamWriter writer = client.createByteStreamWriter(stream);
    @Cleanup ByteStreamReader reader = client.createByteStreamReader(stream);
    AssertExtensions.assertBlocks(() -> {
        assertEquals(100, reader.read(readBuffer));
    }, () -> writer.write(payload));
    assertEquals(1, readBuffer[99]);
    assertEquals(0, readBuffer[100]);
    Arrays.fill(readBuffer, (byte) 0);
    writer.write(payload);
    assertEquals(100, reader.read(readBuffer));
    assertEquals(1, readBuffer[99]);
    assertEquals(0, readBuffer[100]);
    writer.write(payload);
    writer.write(payload);
    assertEquals(200, StreamHelpers.readAll(reader, readBuffer, 0, readBuffer.length));
    AssertExtensions.assertBlocks(() -> {
        assertEquals(100, reader.read(readBuffer));
    }, () -> writer.write(payload));
    writer.closeAndSeal();
    assertEquals(-1, reader.read());
}

Example 3 with ByteStreamWriter

use of io.pravega.client.byteStream.ByteStreamWriter in project pravega by pravega.

the class ByteClientTest method byteClientTruncationTest.

/**
 * This test verifies the correctness of truncation in a ByteStream using {@link ByteStreamReader} and {@link ByteStreamWriter}.
 */
@Test
public void byteClientTruncationTest() throws IOException {
    log.info("byteClientTruncationTest:: with security enabled: {}", Utils.AUTH_ENABLED);
    assertTrue("Creating stream", streamManager.createStream(SCOPE, STREAM_TRUNCATION, config));
    log.info("Invoking byteClientTruncationTest test with Controller URI: {}", controllerURI);
    @Cleanup ByteStreamClientFactory factory = createClientFactory(SCOPE);
    @Cleanup("closeAndSeal") ByteStreamWriter writer = factory.createByteStreamWriter(STREAM);
    @Cleanup ByteStreamReader reader = factory.createByteStreamReader(STREAM);
    // Write events.
    long limit = 10000L;
    LongStream.rangeClosed(1, limit).forEachOrdered(val -> {
        try {
            // write as bytes.
            writer.write(Longs.toByteArray(val));
        } catch (IOException e) {
            log.error("Failed to write to the byte stream ", e);
            fail("IO Error while write to byte stream");
        }
    });
    // read 8 bytes at a time.
    byte[] readBuffer = new byte[Long.BYTES];
    // Number of bytes already fetched by the reader
    assertEquals("The initial offset of reader is zero", 0, reader.getOffset());
    // Number of events that can be read from the byte stream without talking to SegmentStore.
    int eventsToRead = reader.available() / Long.BYTES;
    assertTrue(eventsToRead < limit - 1);
    // Set the truncation offset to a event boundary greater than the prefetched events.
    int truncationOffset = (eventsToRead + 1) * Long.BYTES;
    log.info("Truncation data before offset {}", truncationOffset);
    writer.truncateDataBefore(truncationOffset);
    long lastRead = -1;
    while (reader.getOffset() < limit * Long.BYTES) {
        try {
            // read 8 bytes
            int bytesRead = reader.read(readBuffer);
            assertEquals(Long.BYTES, bytesRead);
            long eventRead = Longs.fromByteArray(readBuffer);
            // validate the read data and ensure data before trunation offset cannot be read.
            if (lastRead >= eventRead || (eventRead > eventsToRead && eventRead < eventsToRead + 1)) {
                log.error("Invalid event read {} while last event that was read is {}", eventRead, lastRead);
                fail("Invalid event read");
            }
            lastRead = eventRead;
        } catch (SegmentTruncatedException e) {
            log.info("Segment truncation observed at offset {}", reader.getOffset());
            reader.seekToOffset(truncationOffset);
        }
    }
    log.info("Data correctly written/read from Stream with truncation");
}

Example 4 with ByteStreamWriter

use of io.pravega.client.byteStream.ByteStreamWriter in project pravega by pravega.

the class ByteStreamTest method readWriteTest.

@Test(timeout = 30000)
public void readWriteTest() throws IOException {
    String scope = "ByteStreamTest";
    String stream = "readWriteTest";
    StreamConfiguration config = StreamConfiguration.builder().build();
    @Cleanup StreamManager streamManager = new StreamManagerImpl(PRAVEGA.getLocalController(), Mockito.mock(ConnectionPool.class));
    // create a scope
    Boolean createScopeStatus = streamManager.createScope(scope);
    log.info("Create scope status {}", createScopeStatus);
    // create a stream
    Boolean createStreamStatus = streamManager.createStream(scope, stream, config);
    log.info("Create stream status {}", createStreamStatus);
    @Cleanup ByteStreamClientFactory client = createClientFactory(scope);
    byte[] payload = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    byte[] readBuffer = new byte[10];
    @Cleanup ByteStreamWriter writer = client.createByteStreamWriter(stream);
    @Cleanup ByteStreamReader reader = client.createByteStreamReader(stream);
    AssertExtensions.assertBlocks(() -> reader.read(readBuffer), () -> writer.write(payload));
    assertArrayEquals(payload, readBuffer);
    Arrays.fill(readBuffer, (byte) 0);
    writer.write(payload);
    writer.write(payload);
    writer.write(payload);
    writer.closeAndSeal();
    assertEquals(10, reader.read(readBuffer));
    assertArrayEquals(payload, readBuffer);
    for (int i = 0; i < 10; i++) {
        assertEquals(i, reader.read());
    }
    Arrays.fill(readBuffer, (byte) -1);
    assertEquals(5, reader.read(readBuffer, 0, 5));
    assertEquals(5, reader.read(readBuffer, 5, 5));
    assertArrayEquals(payload, readBuffer);
    assertEquals(-1, reader.read());
    assertEquals(-1, reader.read(readBuffer));
}

Example 5 with ByteStreamWriter

use of io.pravega.client.byteStream.ByteStreamWriter in project pravega by pravega.

the class ByteStreamTest method readWriteTestTruncate.

@Test(timeout = 30000)
public void readWriteTestTruncate() throws IOException {
    String scope = "ByteStreamTest";
    String stream = "readWriteTestTruncate";
    StreamConfiguration config = StreamConfiguration.builder().build();
    @Cleanup StreamManager streamManager = new StreamManagerImpl(PRAVEGA.getLocalController(), Mockito.mock(ConnectionPool.class));
    // create a scope
    Boolean createScopeStatus = streamManager.createScope(scope);
    log.info("Create scope status {}", createScopeStatus);
    // create a stream
    Boolean createStreamStatus = streamManager.createStream(scope, stream, config);
    log.info("Create stream status {}", createStreamStatus);
    @Cleanup ByteStreamClientFactory client = createClientFactory(scope);
    byte[] payload = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    byte[] readBuffer = new byte[10];
    @Cleanup ByteStreamWriter writer = client.createByteStreamWriter(stream);
    @Cleanup ByteStreamReader reader = client.createByteStreamReader(stream);
    // Verify reads and writes.
    AssertExtensions.assertBlocks(() -> reader.read(readBuffer), () -> writer.write(payload));
    assertArrayEquals(payload, readBuffer);
    // Truncate data before offset 5
    writer.truncateDataBefore(5);
    // seek to an invalid truncated offset and verify if truncation is successful.
    reader.seekToOffset(reader.fetchHeadOffset() - 1);
    assertThrows(SegmentTruncatedException.class, reader::read);
    // seek to the new head and verify if we are able to read the data.
    byte[] data = new byte[] { 5, 6, 7, 8, 9 };
    reader.seekToOffset(reader.fetchHeadOffset());
    byte[] readBuffer1 = new byte[5];
    int bytesRead = reader.read(readBuffer1);
    assertEquals(5, bytesRead);
    assertArrayEquals(readBuffer1, data);
    // create a new byteStream Reader.
    ByteStreamReader reader1 = client.createByteStreamReader(stream);
    // verify it is able to read
    readBuffer1 = new byte[5];
    bytesRead = reader1.read(readBuffer1);
    // verify if all the bytes are read.
    assertEquals(5, bytesRead);
    assertArrayEquals(readBuffer1, data);
}