use of io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer in project pravega by pravega.
the class MockStreamManager method deleteReaderGroup.
@Override
public void deleteReaderGroup(String groupName) {
@Cleanup StateSynchronizer<ReaderGroupState> synchronizer = clientFactory.createStateSynchronizer(NameUtils.getStreamForReaderGroup(groupName), new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
synchronizer.fetchUpdates();
UUID groupId = synchronizer.getState().getConfig().getReaderGroupId();
long generation = synchronizer.getState().getConfig().getGeneration();
getAndHandleExceptions(controller.deleteReaderGroup(scope, groupName, groupId), RuntimeException::new);
}
use of io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer in project pravega by pravega.
the class EventStreamReaderTest method createStateSynchronizerForReaderGroup.
private StateSynchronizer<ReaderGroupState> createStateSynchronizerForReaderGroup(ConnectionPool connectionPool, Controller controller, MockSegmentStreamFactory streamFactory, Stream stream, String readerId, AtomicLong clock, int numSegments, String groupName) {
String readerGroupStream = NameUtils.getStreamForReaderGroup(groupName);
StreamConfiguration streamConfig = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(numSegments)).build();
controller.createScope(stream.getScope());
controller.createStream(stream.getScope(), stream.getStreamName(), streamConfig);
controller.createStream(stream.getScope(), readerGroupStream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
ClientFactoryImpl clientFactory = new ClientFactoryImpl(stream.getScope(), controller, connectionPool, streamFactory, streamFactory, streamFactory, streamFactory);
createdClientFactories.add(clientFactory);
ReaderGroupConfig config = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(stream).build();
StateSynchronizer<ReaderGroupState> sync = clientFactory.createStateSynchronizer(readerGroupStream, new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
sync.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
return sync;
}
use of io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer in project pravega by pravega.
the class EventStreamReaderTest method testTimeWindow.
@Test
public void testTimeWindow() throws SegmentSealedException {
String scope = "scope";
String streamName = "stream";
Stream stream = Stream.of(scope, streamName);
String groupName = "readerGroup";
String readerGroupStream = NameUtils.getStreamForReaderGroup(groupName);
String markStream = NameUtils.getMarkStreamForStream(streamName);
// Create factories
MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
@Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, segmentStreamFactory);
MockController controller = (MockController) clientFactory.getController();
@Cleanup InlineExecutor executor = new InlineExecutor();
// Create streams
controller.createScope(scope).join();
controller.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).build());
controller.createStream(scope, readerGroupStream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
// Reader group state synchronizer
ReaderGroupConfig config = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(stream).build();
StateSynchronizer<ReaderGroupState> sync = clientFactory.createStateSynchronizer(readerGroupStream, new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
// Watermark reader/writer
@Cleanup RevisionedStreamClient<Watermark> markWriter = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
@Cleanup WatermarkReaderImpl markReader = new WatermarkReaderImpl(stream, markWriter, executor);
// Initialize reader group state
Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
sync.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
// Data segment writers
Segment segment1 = new Segment(scope, streamName, 0);
Segment segment2 = new Segment(scope, streamName, 1);
@Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
@Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
// Write stream data
writeInt(stream1, 1);
writeInt(stream2, 2);
writeInt(stream2, 2);
writeInt(stream2, 2);
// Write mark data
val r1 = new SegmentWithRange(segment1, 0, 0.5).convert();
val r2 = new SegmentWithRange(segment2, 0.5, 1).convert();
markWriter.writeUnconditionally(new Watermark(0L, 99L, ImmutableMap.of(r1, 0L, r2, 0L)));
markWriter.writeUnconditionally(new Watermark(100L, 199L, ImmutableMap.of(r1, 12L, r2, 0L)));
markWriter.writeUnconditionally(new Watermark(200L, 299L, ImmutableMap.of(r1, 12L, r2, 12L)));
markWriter.writeUnconditionally(new Watermark(300L, 399L, ImmutableMap.of(r1, 12L, r2, 24L)));
markWriter.writeUnconditionally(new Watermark(400L, 499L, ImmutableMap.of(r1, 12L, r2, 36L)));
// Create reader
AtomicLong clock = new AtomicLong();
ReaderGroupStateManager groupState = new ReaderGroupStateManager(scope, groupName, "reader1", sync, controller, clock::get);
groupState.initializeReader(0);
@Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, new Orderer(), clock::get, ReaderConfig.builder().build(), ImmutableMap.of(stream, markReader), Mockito.mock(Controller.class));
clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
EventRead<byte[]> event = reader.readNextEvent(100);
assertEquals(2, readInt(event));
TimeWindow timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
assertEquals(0, timeWindow.getLowerTimeBound().longValue());
assertEquals(199, timeWindow.getUpperTimeBound().longValue());
clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
event = reader.readNextEvent(100);
assertEquals(1, readInt(event));
timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
assertEquals(0, timeWindow.getLowerTimeBound().longValue());
assertEquals(299, timeWindow.getUpperTimeBound().longValue());
clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
event = reader.readNextEvent(100);
assertEquals(2, readInt(event));
timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
assertEquals(200, timeWindow.getLowerTimeBound().longValue());
assertEquals(399, timeWindow.getUpperTimeBound().longValue());
clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
event = reader.readNextEvent(100);
assertEquals(2, readInt(event));
timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
assertEquals(300, timeWindow.getLowerTimeBound().longValue());
assertEquals(499, timeWindow.getUpperTimeBound().longValue());
clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
event = reader.readNextEvent(100);
assertEquals(null, event.getEvent());
timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
assertEquals(400, timeWindow.getLowerTimeBound().longValue());
assertEquals(null, timeWindow.getUpperTimeBound());
}
use of io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer in project pravega by pravega.
the class MockStreamManager method createReaderGroup.
@Override
public boolean createReaderGroup(String groupName, ReaderGroupConfig config) {
NameUtils.validateReaderGroupName(groupName);
createStreamHelper(NameUtils.getStreamForReaderGroup(groupName), StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
if (ReaderGroupConfig.DEFAULT_UUID.equals(config.getReaderGroupId())) {
config = ReaderGroupConfig.cloneConfig(config, UUID.randomUUID(), 0L);
}
@Cleanup StateSynchronizer<ReaderGroupState> synchronizer = clientFactory.createStateSynchronizer(NameUtils.getStreamForReaderGroup(groupName), new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
Futures.getThrowingException(controller.createReaderGroup(scope, groupName, config));
Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
synchronizer.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
return true;
}
Aggregations