Examples with SegmentOutputStream io.pravega.client.segment.impl.SegmentOutputStream used on opensource projects

Example 46 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamReaderTest method testReaderClose.

@Test
public void testReaderClose() throws SegmentSealedException {
    String scope = "scope";
    String stream = "stream";
    String groupName = "readerGroup";
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", -1);
    @Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    @Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    // Mock for the two SegmentInputStreams.
    Segment segment1 = new Segment(scope, stream, 0);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    writeInt(stream1, 1);
    writeInt(stream1, 1);
    writeInt(stream1, 1);
    Segment segment2 = new Segment(scope, stream, 1);
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    StateSynchronizer<ReaderGroupState> sync = createStateSynchronizerForReaderGroup(connectionFactory, controller, segmentStreamFactory, Stream.of(scope, stream), "reader1", clock, 2, groupName);
    @Cleanup EventStreamReaderImpl<byte[]> reader1 = createReader(controller, segmentStreamFactory, "reader1", sync, clock, scope, groupName);
    @Cleanup EventStreamReaderImpl<byte[]> reader2 = createReader(controller, segmentStreamFactory, "reader2", sync, clock, scope, groupName);
    assertEquals(1, readInt(reader1.readNextEvent(0)));
    assertEquals(2, readInt(reader2.readNextEvent(0)));
    reader2.close();
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertEquals(1, readInt(reader1.readNextEvent(0)));
    assertEquals(2, readInt(reader1.readNextEvent(0)));
    assertEquals(1, readInt(reader1.readNextEvent(0)));
    assertEquals(2, readInt(reader1.readNextEvent(0)));
}

Example 47 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamReaderTest method testTimeWindow.

@Test
public void testTimeWindow() throws SegmentSealedException {
    String scope = "scope";
    String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    String groupName = "readerGroup";
    String readerGroupStream = NameUtils.getStreamForReaderGroup(groupName);
    String markStream = NameUtils.getMarkStreamForStream(streamName);
    // Create factories
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, segmentStreamFactory);
    MockController controller = (MockController) clientFactory.getController();
    @Cleanup InlineExecutor executor = new InlineExecutor();
    // Create streams
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).build());
    controller.createStream(scope, readerGroupStream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
    // Reader group state synchronizer
    ReaderGroupConfig config = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(stream).build();
    StateSynchronizer<ReaderGroupState> sync = clientFactory.createStateSynchronizer(readerGroupStream, new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
    // Watermark reader/writer
    @Cleanup RevisionedStreamClient<Watermark> markWriter = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    @Cleanup WatermarkReaderImpl markReader = new WatermarkReaderImpl(stream, markWriter, executor);
    // Initialize reader group state
    Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
    sync.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
    // Data segment writers
    Segment segment1 = new Segment(scope, streamName, 0);
    Segment segment2 = new Segment(scope, streamName, 1);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    // Write stream data
    writeInt(stream1, 1);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    // Write mark data
    val r1 = new SegmentWithRange(segment1, 0, 0.5).convert();
    val r2 = new SegmentWithRange(segment2, 0.5, 1).convert();
    markWriter.writeUnconditionally(new Watermark(0L, 99L, ImmutableMap.of(r1, 0L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(100L, 199L, ImmutableMap.of(r1, 12L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(200L, 299L, ImmutableMap.of(r1, 12L, r2, 12L)));
    markWriter.writeUnconditionally(new Watermark(300L, 399L, ImmutableMap.of(r1, 12L, r2, 24L)));
    markWriter.writeUnconditionally(new Watermark(400L, 499L, ImmutableMap.of(r1, 12L, r2, 36L)));
    // Create reader
    AtomicLong clock = new AtomicLong();
    ReaderGroupStateManager groupState = new ReaderGroupStateManager(scope, groupName, "reader1", sync, controller, clock::get);
    groupState.initializeReader(0);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, new Orderer(), clock::get, ReaderConfig.builder().build(), ImmutableMap.of(stream, markReader), Mockito.mock(Controller.class));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    EventRead<byte[]> event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    TimeWindow timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(199, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(1, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(299, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(200, timeWindow.getLowerTimeBound().longValue());
    assertEquals(399, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(300, timeWindow.getLowerTimeBound().longValue());
    assertEquals(499, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(null, event.getEvent());
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(400, timeWindow.getLowerTimeBound().longValue());
    assertEquals(null, timeWindow.getUpperTimeBound());
}

Example 48 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamReaderTest method testReadWithEndOfSegmentException.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testReadWithEndOfSegmentException() throws Exception {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    // Prep the mocks.
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    // Mock for the two SegmentInputStreams.
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    EventSegmentReader segmentInputStream1 = Mockito.mock(EventSegmentReader.class);
    Mockito.when(segmentInputStream1.read(anyLong())).thenThrow(new EndOfSegmentException(EndOfSegmentException.ErrorType.END_OFFSET_REACHED));
    Mockito.when(segmentInputStream1.getSegmentId()).thenReturn(segment);
    EventSegmentReader segmentInputStream2 = Mockito.mock(EventSegmentReader.class);
    @Cleanup SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    ByteBuffer buffer = writeInt(stream, 1);
    Mockito.when(segmentInputStream2.read(anyLong())).thenReturn(buffer);
    Mockito.when(segmentInputStream2.getSegmentId()).thenReturn(Segment.fromScopedName("Foo/test/0"));
    Mockito.when(segmentInputStream2.getOffset()).thenReturn(10L);
    SegmentInputStreamFactory inputStreamFactory = Mockito.mock(SegmentInputStreamFactory.class);
    Mockito.when(inputStreamFactory.createEventReaderForSegment(any(Segment.class), anyInt(), any(Semaphore.class), anyLong())).thenReturn(segmentInputStream1);
    // Mock Orderer
    Orderer orderer = Mockito.mock(Orderer.class);
    Mockito.when(orderer.nextSegment(any(List.class))).thenReturn(segmentInputStream1).thenReturn(segmentInputStream2);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(inputStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    InOrder inOrder = Mockito.inOrder(segmentInputStream1, groupState);
    EventRead<byte[]> event = reader.readNextEvent(100L);
    assertNotNull(event.getEvent());
    // Validate that segmentInputStream1.close() is invoked on reaching endOffset.
    Mockito.verify(segmentInputStream1, Mockito.times(1)).close();
    SegmentWithRange segmentWithRange = new SegmentWithRange(segment, 0, 1);
    // Verify groupstate is not updated before the checkpoint, but is after.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(segmentWithRange);
    Mockito.when(groupState.getCheckpoint()).thenReturn("checkpoint").thenReturn(null);
    assertEquals("checkpoint", reader.readNextEvent(0).getCheckpointName());
    inOrder.verify(groupState).getCheckpoint();
    // Verify groupstate is not updated before the checkpoint, but is after.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(segmentWithRange);
    event = reader.readNextEvent(0);
    assertFalse(event.isCheckpoint());
    // Now it is called.
    inOrder.verify(groupState, Mockito.times(1)).handleEndOfSegment(segmentWithRange);
}

Example 49 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamReaderTest method testCheckpoint.

@Test(timeout = 10000)
public void testCheckpoint() throws SegmentSealedException, ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(new SegmentWithRange(segment, 0, 1), 0L)).thenReturn(Collections.emptyMap());
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    @Cleanup SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    ByteBuffer buffer = writeInt(stream, 1);
    Mockito.when(groupState.getCheckpoint()).thenReturn("Foo").thenReturn(null);
    EventRead<byte[]> eventRead = reader.readNextEvent(0);
    assertTrue(eventRead.isCheckpoint());
    assertNull(eventRead.getEvent());
    assertEquals("Foo", eventRead.getCheckpointName());
    InOrder order = Mockito.inOrder(groupState);
    order.verify(groupState).getCheckpoint();
    order.verify(groupState, Mockito.never()).checkpoint(Mockito.anyString(), Mockito.any());
    assertEquals(buffer, ByteBuffer.wrap(reader.readNextEvent(0).getEvent()));
    assertNull(reader.readNextEvent(0).getEvent());
    order.verify(groupState).checkpoint(Mockito.eq("Foo"), Mockito.any());
    order.verify(groupState).getCheckpoint();
    reader.close();
}

Example 50 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamReaderTest method testDataTruncated.

@Test(timeout = 10000)
public void testDataTruncated() throws SegmentSealedException, ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(new SegmentWithRange(segment, 0, 1), 0L)).thenReturn(Collections.emptyMap());
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    @Cleanup SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    @Cleanup SegmentMetadataClient metadataClient = segmentStreamFactory.createSegmentMetadataClient(segment, DelegationTokenProviderFactory.createWithEmptyToken());
    ByteBuffer buffer1 = writeInt(stream, 1);
    ByteBuffer buffer2 = writeInt(stream, 2);
    writeInt(stream, 3);
    long length = metadataClient.fetchCurrentSegmentLength().join();
    assertEquals(0, length % 3);
    EventRead<byte[]> event1 = reader.readNextEvent(0);
    assertEquals(buffer1, ByteBuffer.wrap(event1.getEvent()));
    metadataClient.truncateSegment(length / 3).join();
    assertEquals(buffer2, ByteBuffer.wrap(reader.readNextEvent(0).getEvent()));
    metadataClient.truncateSegment(length).join();
    ByteBuffer buffer4 = writeInt(stream, 4);
    assertThrows(TruncatedDataException.class, () -> reader.readNextEvent(0));
    assertEquals(buffer4, ByteBuffer.wrap(reader.readNextEvent(0).getEvent()));
    assertNull(reader.readNextEvent(0).getEvent());
    assertThrows(NoSuchEventException.class, () -> reader.fetchEvent(event1.getEventPointer()));
    reader.close();
}