Search in sources :

Example 1 with ReaderGroupStateInitSerializer

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer in project pravega by pravega.

the class MockStreamManager method deleteReaderGroup.

@Override
public void deleteReaderGroup(String groupName) {
    @Cleanup StateSynchronizer<ReaderGroupState> synchronizer = clientFactory.createStateSynchronizer(NameUtils.getStreamForReaderGroup(groupName), new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
    synchronizer.fetchUpdates();
    UUID groupId = synchronizer.getState().getConfig().getReaderGroupId();
    long generation = synchronizer.getState().getConfig().getGeneration();
    getAndHandleExceptions(controller.deleteReaderGroup(scope, groupName, groupId), RuntimeException::new);
}
Also used : ReaderGroupStateUpdatesSerializer(io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateUpdatesSerializer) ReaderGroupStateInitSerializer(io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer) ReaderGroupState(io.pravega.client.stream.impl.ReaderGroupState) UUID(java.util.UUID) Cleanup(lombok.Cleanup)

Example 2 with ReaderGroupStateInitSerializer

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer in project pravega by pravega.

the class EventStreamReaderTest method createStateSynchronizerForReaderGroup.

private StateSynchronizer<ReaderGroupState> createStateSynchronizerForReaderGroup(ConnectionPool connectionPool, Controller controller, MockSegmentStreamFactory streamFactory, Stream stream, String readerId, AtomicLong clock, int numSegments, String groupName) {
    String readerGroupStream = NameUtils.getStreamForReaderGroup(groupName);
    StreamConfiguration streamConfig = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(numSegments)).build();
    controller.createScope(stream.getScope());
    controller.createStream(stream.getScope(), stream.getStreamName(), streamConfig);
    controller.createStream(stream.getScope(), readerGroupStream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
    ClientFactoryImpl clientFactory = new ClientFactoryImpl(stream.getScope(), controller, connectionPool, streamFactory, streamFactory, streamFactory, streamFactory);
    createdClientFactories.add(clientFactory);
    ReaderGroupConfig config = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(stream).build();
    StateSynchronizer<ReaderGroupState> sync = clientFactory.createStateSynchronizer(readerGroupStream, new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
    Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
    sync.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
    return sync;
}
Also used : ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) ReaderGroupStateUpdatesSerializer(io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateUpdatesSerializer) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) AtomicLong(java.util.concurrent.atomic.AtomicLong) ReaderGroupStateInitSerializer(io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer)

Example 3 with ReaderGroupStateInitSerializer

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer in project pravega by pravega.

the class EventStreamReaderTest method testTimeWindow.

@Test
public void testTimeWindow() throws SegmentSealedException {
    String scope = "scope";
    String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    String groupName = "readerGroup";
    String readerGroupStream = NameUtils.getStreamForReaderGroup(groupName);
    String markStream = NameUtils.getMarkStreamForStream(streamName);
    // Create factories
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, segmentStreamFactory);
    MockController controller = (MockController) clientFactory.getController();
    @Cleanup InlineExecutor executor = new InlineExecutor();
    // Create streams
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).build());
    controller.createStream(scope, readerGroupStream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
    // Reader group state synchronizer
    ReaderGroupConfig config = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(stream).build();
    StateSynchronizer<ReaderGroupState> sync = clientFactory.createStateSynchronizer(readerGroupStream, new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
    // Watermark reader/writer
    @Cleanup RevisionedStreamClient<Watermark> markWriter = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    @Cleanup WatermarkReaderImpl markReader = new WatermarkReaderImpl(stream, markWriter, executor);
    // Initialize reader group state
    Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
    sync.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
    // Data segment writers
    Segment segment1 = new Segment(scope, streamName, 0);
    Segment segment2 = new Segment(scope, streamName, 1);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    // Write stream data
    writeInt(stream1, 1);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    // Write mark data
    val r1 = new SegmentWithRange(segment1, 0, 0.5).convert();
    val r2 = new SegmentWithRange(segment2, 0.5, 1).convert();
    markWriter.writeUnconditionally(new Watermark(0L, 99L, ImmutableMap.of(r1, 0L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(100L, 199L, ImmutableMap.of(r1, 12L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(200L, 299L, ImmutableMap.of(r1, 12L, r2, 12L)));
    markWriter.writeUnconditionally(new Watermark(300L, 399L, ImmutableMap.of(r1, 12L, r2, 24L)));
    markWriter.writeUnconditionally(new Watermark(400L, 499L, ImmutableMap.of(r1, 12L, r2, 36L)));
    // Create reader
    AtomicLong clock = new AtomicLong();
    ReaderGroupStateManager groupState = new ReaderGroupStateManager(scope, groupName, "reader1", sync, controller, clock::get);
    groupState.initializeReader(0);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, new Orderer(), clock::get, ReaderConfig.builder().build(), ImmutableMap.of(stream, markReader), Mockito.mock(Controller.class));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    EventRead<byte[]> event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    TimeWindow timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(199, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(1, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(299, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(200, timeWindow.getLowerTimeBound().longValue());
    assertEquals(399, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(300, timeWindow.getLowerTimeBound().longValue());
    assertEquals(499, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(null, event.getEvent());
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(400, timeWindow.getLowerTimeBound().longValue());
    assertEquals(null, timeWindow.getUpperTimeBound());
}
Also used : ReaderGroupStateInitSerializer(io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer) Cleanup(lombok.Cleanup) MockClientFactory(io.pravega.client.stream.mock.MockClientFactory) Segment(io.pravega.client.segment.impl.Segment) InlineExecutor(io.pravega.test.common.InlineExecutor) SegmentOutputStream(io.pravega.client.segment.impl.SegmentOutputStream) Stream(io.pravega.client.stream.Stream) SegmentOutputStream(io.pravega.client.segment.impl.SegmentOutputStream) ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) lombok.val(lombok.val) MockSegmentStreamFactory(io.pravega.client.stream.mock.MockSegmentStreamFactory) WatermarkSerializer(io.pravega.client.watermark.WatermarkSerializer) MockController(io.pravega.client.stream.mock.MockController) Controller(io.pravega.client.control.impl.Controller) TimeWindow(io.pravega.client.stream.TimeWindow) ReaderGroupStateUpdatesSerializer(io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateUpdatesSerializer) AtomicLong(java.util.concurrent.atomic.AtomicLong) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) AtomicLong(java.util.concurrent.atomic.AtomicLong) MockController(io.pravega.client.stream.mock.MockController) Watermark(io.pravega.shared.watermarks.Watermark) Test(org.junit.Test)

Example 4 with ReaderGroupStateInitSerializer

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer in project pravega by pravega.

the class MockStreamManager method createReaderGroup.

@Override
public boolean createReaderGroup(String groupName, ReaderGroupConfig config) {
    NameUtils.validateReaderGroupName(groupName);
    createStreamHelper(NameUtils.getStreamForReaderGroup(groupName), StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
    if (ReaderGroupConfig.DEFAULT_UUID.equals(config.getReaderGroupId())) {
        config = ReaderGroupConfig.cloneConfig(config, UUID.randomUUID(), 0L);
    }
    @Cleanup StateSynchronizer<ReaderGroupState> synchronizer = clientFactory.createStateSynchronizer(NameUtils.getStreamForReaderGroup(groupName), new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
    Futures.getThrowingException(controller.createReaderGroup(scope, groupName, config));
    Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
    synchronizer.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
    return true;
}
Also used : ReaderGroupStateUpdatesSerializer(io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateUpdatesSerializer) ReaderGroupStateInitSerializer(io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer) SegmentWithRange(io.pravega.client.stream.impl.SegmentWithRange) ReaderGroupState(io.pravega.client.stream.impl.ReaderGroupState) Cleanup(lombok.Cleanup)

Aggregations

ReaderGroupStateInitSerializer (io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer)4 ReaderGroupStateUpdatesSerializer (io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateUpdatesSerializer)4 Cleanup (lombok.Cleanup)3 ReaderGroupConfig (io.pravega.client.stream.ReaderGroupConfig)2 ReaderGroupState (io.pravega.client.stream.impl.ReaderGroupState)2 AtomicLong (java.util.concurrent.atomic.AtomicLong)2 ArgumentMatchers.anyLong (org.mockito.ArgumentMatchers.anyLong)2 Controller (io.pravega.client.control.impl.Controller)1 Segment (io.pravega.client.segment.impl.Segment)1 SegmentOutputStream (io.pravega.client.segment.impl.SegmentOutputStream)1 Stream (io.pravega.client.stream.Stream)1 StreamConfiguration (io.pravega.client.stream.StreamConfiguration)1 TimeWindow (io.pravega.client.stream.TimeWindow)1 SegmentWithRange (io.pravega.client.stream.impl.SegmentWithRange)1 MockClientFactory (io.pravega.client.stream.mock.MockClientFactory)1 MockController (io.pravega.client.stream.mock.MockController)1 MockSegmentStreamFactory (io.pravega.client.stream.mock.MockSegmentStreamFactory)1 WatermarkSerializer (io.pravega.client.watermark.WatermarkSerializer)1 Watermark (io.pravega.shared.watermarks.Watermark)1 InlineExecutor (io.pravega.test.common.InlineExecutor)1