Examples with MockSegmentStreamFactory io.pravega.client.stream.mock.MockSegmentStreamFactory used on opensource projects

Example 36 with MockSegmentStreamFactory

use of io.pravega.client.stream.mock.MockSegmentStreamFactory in project pravega by pravega.

the class EventStreamReaderTest method testRead.

@Test(timeout = 10000)
public void testRead() throws SegmentSealedException, ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    SegmentWithRange segment = new SegmentWithRange(Segment.fromScopedName("Foo/Bar/0"), 0, 1);
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(segment, 0L)).thenReturn(Collections.emptyMap());
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    @Cleanup SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment.getSegment(), segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    ByteBuffer buffer1 = writeInt(stream, 1);
    ByteBuffer buffer2 = writeInt(stream, 2);
    ByteBuffer buffer3 = writeInt(stream, 3);
    EventRead<byte[]> e = reader.readNextEvent(0);
    StringBuilder name = new StringBuilder();
    name.append("Foo");
    name.append("/");
    name.append("Bar");
    assertEquals(buffer1, ByteBuffer.wrap(e.getEvent()));
    assertEquals(name.toString(), e.getEventPointer().getStream().getScopedName());
    assertEquals(Long.valueOf(WireCommands.TYPE_PLUS_LENGTH_SIZE + Integer.BYTES), e.getPosition().asImpl().getOffsetForOwnedSegment(Segment.fromScopedName("Foo/Bar/0")));
    e = reader.readNextEvent(0);
    assertEquals(buffer2, ByteBuffer.wrap(e.getEvent()));
    assertEquals(name.toString(), e.getEventPointer().getStream().getScopedName());
    assertEquals(Long.valueOf(2 * (WireCommands.TYPE_PLUS_LENGTH_SIZE + Integer.BYTES)), e.getPosition().asImpl().getOffsetForOwnedSegment(Segment.fromScopedName("Foo/Bar/0")));
    e = reader.readNextEvent(0);
    assertEquals(buffer3, ByteBuffer.wrap(e.getEvent()));
    assertEquals(name.toString(), e.getEventPointer().getStream().getScopedName());
    assertEquals(Long.valueOf(3 * (WireCommands.TYPE_PLUS_LENGTH_SIZE + Integer.BYTES)), e.getPosition().asImpl().getOffsetForOwnedSegment(Segment.fromScopedName("Foo/Bar/0")));
    assertEquals(name.toString(), e.getEventPointer().getStream().getScopedName());
    e = reader.readNextEvent(0);
    assertNull(e.getEvent());
    assertNull(e.getEventPointer());
    assertEquals(Long.valueOf(-1), e.getPosition().asImpl().getOffsetForOwnedSegment(Segment.fromScopedName("Foo/Bar/0")));
    reader.close();
}

Example 37 with MockSegmentStreamFactory

use of io.pravega.client.stream.mock.MockSegmentStreamFactory in project pravega by pravega.

the class EventStreamReaderTest method testReaderClose.

@Test
public void testReaderClose() throws SegmentSealedException {
    String scope = "scope";
    String stream = "stream";
    String groupName = "readerGroup";
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", -1);
    @Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    @Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    // Mock for the two SegmentInputStreams.
    Segment segment1 = new Segment(scope, stream, 0);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    writeInt(stream1, 1);
    writeInt(stream1, 1);
    writeInt(stream1, 1);
    Segment segment2 = new Segment(scope, stream, 1);
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    StateSynchronizer<ReaderGroupState> sync = createStateSynchronizerForReaderGroup(connectionFactory, controller, segmentStreamFactory, Stream.of(scope, stream), "reader1", clock, 2, groupName);
    @Cleanup EventStreamReaderImpl<byte[]> reader1 = createReader(controller, segmentStreamFactory, "reader1", sync, clock, scope, groupName);
    @Cleanup EventStreamReaderImpl<byte[]> reader2 = createReader(controller, segmentStreamFactory, "reader2", sync, clock, scope, groupName);
    assertEquals(1, readInt(reader1.readNextEvent(0)));
    assertEquals(2, readInt(reader2.readNextEvent(0)));
    reader2.close();
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertEquals(1, readInt(reader1.readNextEvent(0)));
    assertEquals(2, readInt(reader1.readNextEvent(0)));
    assertEquals(1, readInt(reader1.readNextEvent(0)));
    assertEquals(2, readInt(reader1.readNextEvent(0)));
}

Example 38 with MockSegmentStreamFactory

use of io.pravega.client.stream.mock.MockSegmentStreamFactory in project pravega by pravega.

the class EventStreamReaderTest method testTimeWindow.

@Test
public void testTimeWindow() throws SegmentSealedException {
    String scope = "scope";
    String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    String groupName = "readerGroup";
    String readerGroupStream = NameUtils.getStreamForReaderGroup(groupName);
    String markStream = NameUtils.getMarkStreamForStream(streamName);
    // Create factories
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, segmentStreamFactory);
    MockController controller = (MockController) clientFactory.getController();
    @Cleanup InlineExecutor executor = new InlineExecutor();
    // Create streams
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).build());
    controller.createStream(scope, readerGroupStream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
    // Reader group state synchronizer
    ReaderGroupConfig config = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(stream).build();
    StateSynchronizer<ReaderGroupState> sync = clientFactory.createStateSynchronizer(readerGroupStream, new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
    // Watermark reader/writer
    @Cleanup RevisionedStreamClient<Watermark> markWriter = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    @Cleanup WatermarkReaderImpl markReader = new WatermarkReaderImpl(stream, markWriter, executor);
    // Initialize reader group state
    Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
    sync.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
    // Data segment writers
    Segment segment1 = new Segment(scope, streamName, 0);
    Segment segment2 = new Segment(scope, streamName, 1);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    // Write stream data
    writeInt(stream1, 1);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    // Write mark data
    val r1 = new SegmentWithRange(segment1, 0, 0.5).convert();
    val r2 = new SegmentWithRange(segment2, 0.5, 1).convert();
    markWriter.writeUnconditionally(new Watermark(0L, 99L, ImmutableMap.of(r1, 0L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(100L, 199L, ImmutableMap.of(r1, 12L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(200L, 299L, ImmutableMap.of(r1, 12L, r2, 12L)));
    markWriter.writeUnconditionally(new Watermark(300L, 399L, ImmutableMap.of(r1, 12L, r2, 24L)));
    markWriter.writeUnconditionally(new Watermark(400L, 499L, ImmutableMap.of(r1, 12L, r2, 36L)));
    // Create reader
    AtomicLong clock = new AtomicLong();
    ReaderGroupStateManager groupState = new ReaderGroupStateManager(scope, groupName, "reader1", sync, controller, clock::get);
    groupState.initializeReader(0);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, new Orderer(), clock::get, ReaderConfig.builder().build(), ImmutableMap.of(stream, markReader), Mockito.mock(Controller.class));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    EventRead<byte[]> event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    TimeWindow timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(199, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(1, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(299, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(200, timeWindow.getLowerTimeBound().longValue());
    assertEquals(399, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(300, timeWindow.getLowerTimeBound().longValue());
    assertEquals(499, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(null, event.getEvent());
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(400, timeWindow.getLowerTimeBound().longValue());
    assertEquals(null, timeWindow.getUpperTimeBound());
}

Example 39 with MockSegmentStreamFactory

use of io.pravega.client.stream.mock.MockSegmentStreamFactory in project pravega by pravega.

the class EventStreamReaderTest method testReadWithEndOfSegmentException.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testReadWithEndOfSegmentException() throws Exception {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    // Prep the mocks.
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    // Mock for the two SegmentInputStreams.
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    EventSegmentReader segmentInputStream1 = Mockito.mock(EventSegmentReader.class);
    Mockito.when(segmentInputStream1.read(anyLong())).thenThrow(new EndOfSegmentException(EndOfSegmentException.ErrorType.END_OFFSET_REACHED));
    Mockito.when(segmentInputStream1.getSegmentId()).thenReturn(segment);
    EventSegmentReader segmentInputStream2 = Mockito.mock(EventSegmentReader.class);
    @Cleanup SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    ByteBuffer buffer = writeInt(stream, 1);
    Mockito.when(segmentInputStream2.read(anyLong())).thenReturn(buffer);
    Mockito.when(segmentInputStream2.getSegmentId()).thenReturn(Segment.fromScopedName("Foo/test/0"));
    Mockito.when(segmentInputStream2.getOffset()).thenReturn(10L);
    SegmentInputStreamFactory inputStreamFactory = Mockito.mock(SegmentInputStreamFactory.class);
    Mockito.when(inputStreamFactory.createEventReaderForSegment(any(Segment.class), anyInt(), any(Semaphore.class), anyLong())).thenReturn(segmentInputStream1);
    // Mock Orderer
    Orderer orderer = Mockito.mock(Orderer.class);
    Mockito.when(orderer.nextSegment(any(List.class))).thenReturn(segmentInputStream1).thenReturn(segmentInputStream2);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(inputStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    InOrder inOrder = Mockito.inOrder(segmentInputStream1, groupState);
    EventRead<byte[]> event = reader.readNextEvent(100L);
    assertNotNull(event.getEvent());
    // Validate that segmentInputStream1.close() is invoked on reaching endOffset.
    Mockito.verify(segmentInputStream1, Mockito.times(1)).close();
    SegmentWithRange segmentWithRange = new SegmentWithRange(segment, 0, 1);
    // Verify groupstate is not updated before the checkpoint, but is after.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(segmentWithRange);
    Mockito.when(groupState.getCheckpoint()).thenReturn("checkpoint").thenReturn(null);
    assertEquals("checkpoint", reader.readNextEvent(0).getCheckpointName());
    inOrder.verify(groupState).getCheckpoint();
    // Verify groupstate is not updated before the checkpoint, but is after.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(segmentWithRange);
    event = reader.readNextEvent(0);
    assertFalse(event.isCheckpoint());
    // Now it is called.
    inOrder.verify(groupState, Mockito.times(1)).handleEndOfSegment(segmentWithRange);
}

Example 40 with MockSegmentStreamFactory

use of io.pravega.client.stream.mock.MockSegmentStreamFactory in project pravega by pravega.

the class SynchronizerTest method testCompaction.

@Test(timeout = 20000)
public void testCompaction() {
    String streamName = "streamName";
    String scope = "scope";
    MockSegmentStreamFactory ioFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, ioFactory);
    createScopeAndStream(streamName, scope, clientFactory.getController());
    StateSynchronizer<RevisionedImpl> sync = clientFactory.createStateSynchronizer(streamName, new JavaSerializer<>(), new JavaSerializer<>(), SynchronizerConfig.builder().build());
    assertEquals(0, sync.bytesWrittenSinceCompaction());
    AtomicInteger callCount = new AtomicInteger(0);
    sync.initialize(new RegularUpdate("a"));
    sync.updateState((state, updates) -> {
        callCount.incrementAndGet();
        updates.add(new RegularUpdate("b"));
    });
    assertEquals(sync.getState().getValue(), "b");
    assertEquals(1, callCount.get());
    long size = sync.bytesWrittenSinceCompaction();
    assertTrue(size > 0);
    sync.updateState((state, updates) -> {
        callCount.incrementAndGet();
        updates.add(new RegularUpdate("c"));
    });
    assertEquals(sync.getState().getValue(), "c");
    assertEquals(2, callCount.get());
    assertTrue(sync.bytesWrittenSinceCompaction() > size);
    sync.compact(state -> {
        callCount.incrementAndGet();
        return new RegularUpdate("c");
    });
    assertEquals(sync.getState().getValue(), "c");
    assertEquals(3, callCount.get());
    assertEquals(0, sync.bytesWrittenSinceCompaction());
    sync.updateState((state, updates) -> {
        callCount.incrementAndGet();
        updates.add(new RegularUpdate("e"));
    });
    assertEquals(sync.getState().getValue(), "e");
    assertEquals(5, callCount.get());
    assertEquals(size, sync.bytesWrittenSinceCompaction());
    sync.compact(state -> {
        callCount.incrementAndGet();
        return new RegularUpdate("e");
    });
    assertEquals(sync.getState().getValue(), "e");
    assertEquals(6, callCount.get());
    assertEquals(0, sync.bytesWrittenSinceCompaction());
}