Examples with SegmentOutputStream io.pravega.client.segment.impl.SegmentOutputStream used on opensource projects

Example 26 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamReaderTest method testAcquireSealedSegment.

@Test(timeout = 10000)
public void testAcquireSealedSegment() throws SegmentSealedException, ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Segment segment1 = Segment.fromScopedName("Foo/Bar/0");
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(new SegmentWithRange(segment1, 0, 0.5), -1L)).thenReturn(Collections.emptyMap());
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    writeInt(stream1, 1);
    writeInt(stream1, 2);
    reader.readNextEvent(0);
    List<EventSegmentReader> readers = reader.getReaders();
    assertEquals(0, readers.size());
    reader.readNextEvent(0);
    readers = reader.getReaders();
    assertEquals(0, readers.size());
    reader.close();
}

Example 27 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamReaderTest method testEndOfSegmentWithoutSuccessors.

@Test(timeout = 10000)
public void testEndOfSegmentWithoutSuccessors() throws ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(new SegmentWithRange(segment, 0, 1), 0L)).thenReturn(Collections.emptyMap());
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    Mockito.when(groupState.handleEndOfSegment(any())).thenReturn(true);
    SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    ByteBuffer buffer = writeInt(stream, 1);
    EventRead<byte[]> read = reader.readNextEvent(0);
    byte[] event = read.getEvent();
    assertEquals(buffer, ByteBuffer.wrap(event));
    read = reader.readNextEvent(0);
    assertNull(read.getEvent());
    read = reader.readNextEvent(0);
    assertNull(read.getEvent());
    assertEquals(0, reader.getReaders().size());
    assertEquals(1, reader.getRanges().size());
    Mockito.when(groupState.getCheckpoint()).thenReturn("CP1");
    read = reader.readNextEvent(0);
    assertTrue(read.isCheckpoint());
    read = reader.readNextEvent(0);
    assertNull(read.getEvent());
    assertEquals(0, reader.getRanges().size());
    reader.close();
}

Example 28 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class ReadTest method readThroughSegmentClient.

@Test(timeout = 10000)
public void readThroughSegmentClient() throws SegmentSealedException, EndOfSegmentException, SegmentTruncatedException {
    String endpoint = "localhost";
    String scope = "scope";
    String stream = "readThroughSegmentClient";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world\n";
    StreamSegmentStore store = SERVICE_BUILDER.createStreamSegmentService();
    TableStore tableStore = SERVICE_BUILDER.createTableStoreService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, tableStore, SERVICE_BUILDER.getLowPriorityExecutor());
    server.startListening();
    @Cleanup SocketConnectionFactoryImpl clientCF = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ConnectionPoolImpl connectionPool = new ConnectionPoolImpl(ClientConfig.builder().build(), clientCF);
    @Cleanup Controller controller = new MockController(endpoint, port, connectionPool, true);
    controller.createScope(scope);
    controller.createStream(scope, stream, StreamConfiguration.builder().build());
    SegmentOutputStreamFactoryImpl segmentproducerClient = new SegmentOutputStreamFactoryImpl(controller, connectionPool);
    SegmentInputStreamFactoryImpl segmentConsumerClient = new SegmentInputStreamFactoryImpl(controller, connectionPool);
    Segment segment = Futures.getAndHandleExceptions(controller.getCurrentSegments(scope, stream), RuntimeException::new).getSegments().iterator().next();
    @Cleanup SegmentOutputStream out = segmentproducerClient.createOutputStreamForSegment(segment, segmentSealedCallback, EventWriterConfig.builder().build(), DelegationTokenProviderFactory.createWithEmptyToken());
    out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(testString.getBytes()), new CompletableFuture<>()));
    out.flush();
    @Cleanup EventSegmentReader in = segmentConsumerClient.createEventReaderForSegment(segment);
    ByteBuffer result = in.read();
    assertEquals(ByteBuffer.wrap(testString.getBytes()), result);
    // Test large write followed by read
    out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(new byte[15]), new CompletableFuture<>()));
    out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(new byte[15]), new CompletableFuture<>()));
    out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(new byte[150000]), new CompletableFuture<>()));
    assertEquals(in.read().capacity(), 15);
    assertEquals(in.read().capacity(), 15);
    assertEquals(in.read().capacity(), 150000);
}

Example 29 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class AppendTest method appendThroughSegmentClient.

@Test(timeout = 10000)
public void appendThroughSegmentClient() throws Exception {
    String endpoint = "localhost";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world\n";
    String scope = "scope";
    String stream = "appendThroughSegmentClient";
    StreamSegmentStore store = SERVICE_BUILDER.createStreamSegmentService();
    TableStore tableStore = SERVICE_BUILDER.createTableStoreService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, tableStore, SERVICE_BUILDER.getLowPriorityExecutor());
    server.startListening();
    @Cleanup SocketConnectionFactoryImpl clientCF = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ConnectionPoolImpl connectionPool = new ConnectionPoolImpl(ClientConfig.builder().build(), clientCF);
    @Cleanup Controller controller = new MockController(endpoint, port, connectionPool, true);
    controller.createScope(scope);
    controller.createStream(scope, stream, StreamConfiguration.builder().build());
    SegmentOutputStreamFactoryImpl segmentClient = new SegmentOutputStreamFactoryImpl(controller, connectionPool);
    Segment segment = Futures.getAndHandleExceptions(controller.getCurrentSegments(scope, stream), RuntimeException::new).getSegments().iterator().next();
    @Cleanup SegmentOutputStream out = segmentClient.createOutputStreamForSegment(segment, segmentSealedCallback, EventWriterConfig.builder().build(), DelegationTokenProviderFactory.createWithEmptyToken());
    CompletableFuture<Void> ack = new CompletableFuture<>();
    out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(testString.getBytes()), ack));
    ack.get(5, TimeUnit.SECONDS);
}

Example 30 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamWriterImpl method flushInternal.

private void flushInternal() {
    writeFlushLock.readLock().lock();
    boolean success = false;
    try {
        while (!success) {
            success = true;
            for (SegmentOutputStream writer : selector.getWriters()) {
                try {
                    writer.flush();
                } catch (SegmentSealedException e) {
                    // Segment sealed exception observed during a flush. Re-run flush on all the available writers.
                    success = false;
                    log.warn("Flush failed due to {}, it will be retried.", e.getMessage());
                }
            }
        }
    } finally {
        writeFlushLock.readLock().unlock();
    }
}