use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.
the class SecureControllerRestApiTest method secureReaderGroupRestApiTest.
@Test
public void secureReaderGroupRestApiTest() throws Exception {
Invocation.Builder builder;
Response response;
restServerURI = CLUSTER.controllerRestUri();
log.info("REST Server URI: {}", restServerURI);
// TEST REST server status, ping test
resourceURl = new StringBuilder(restServerURI).append("/ping").toString();
webTarget = client.target(resourceURl);
builder = webTarget.request();
response = builder.get();
assertEquals("Ping test", OK.getStatusCode(), response.getStatus());
log.info("REST Server is running. Ping successful.");
// Test reader groups APIs.
// Prepare the streams and readers using the admin client.
final String testScope = RandomStringUtils.randomAlphanumeric(10);
final String testStream1 = RandomStringUtils.randomAlphanumeric(10);
final String testStream2 = RandomStringUtils.randomAlphanumeric(10);
URI controllerUri = new URI(CLUSTER.controllerUri());
@Cleanup("shutdown") InlineExecutor inlineExecutor = new InlineExecutor();
ClientConfig clientConfig = ClientConfig.builder().controllerURI(controllerUri).credentials(new DefaultCredentials(SecurityConfigDefaults.AUTH_ADMIN_PASSWORD, SecurityConfigDefaults.AUTH_ADMIN_USERNAME)).trustStore(TRUSTSTORE_PATH).validateHostName(false).build();
try (ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
StreamManager streamManager = new StreamManagerImpl(createController(controllerUri, inlineExecutor), cp)) {
log.info("Creating scope: {}", testScope);
streamManager.createScope(testScope);
log.info("Creating stream: {}", testStream1);
StreamConfiguration streamConf1 = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
streamManager.createStream(testScope, testStream1, streamConf1);
log.info("Creating stream: {}", testStream2);
StreamConfiguration streamConf2 = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
streamManager.createStream(testScope, testStream2, streamConf2);
}
final String readerGroupName1 = RandomStringUtils.randomAlphanumeric(10);
final String readerGroupName2 = RandomStringUtils.randomAlphanumeric(10);
final String reader1 = RandomStringUtils.randomAlphanumeric(10);
final String reader2 = RandomStringUtils.randomAlphanumeric(10);
try (ClientFactoryImpl clientFactory = new ClientFactoryImpl(testScope, createController(controllerUri, inlineExecutor), clientConfig);
ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(testScope, ClientConfig.builder().controllerURI(controllerUri).credentials(new DefaultCredentials(SecurityConfigDefaults.AUTH_ADMIN_PASSWORD, SecurityConfigDefaults.AUTH_ADMIN_USERNAME)).trustStore(TRUSTSTORE_PATH).validateHostName(false).build())) {
readerGroupManager.createReaderGroup(readerGroupName1, ReaderGroupConfig.builder().stream(Stream.of(testScope, testStream1)).stream(Stream.of(testScope, testStream2)).build());
readerGroupManager.createReaderGroup(readerGroupName2, ReaderGroupConfig.builder().stream(Stream.of(testScope, testStream1)).stream(Stream.of(testScope, testStream2)).build());
clientFactory.createReader(reader1, readerGroupName1, new JavaSerializer<Long>(), ReaderConfig.builder().build());
clientFactory.createReader(reader2, readerGroupName1, new JavaSerializer<Long>(), ReaderConfig.builder().build());
}
// Test fetching readergroups.
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups").toString();
response = client.target(resourceURl).request().get();
assertEquals("Get readergroups status", OK.getStatusCode(), response.getStatus());
ReaderGroupsList readerGroupsList = response.readEntity(ReaderGroupsList.class);
assertEquals("Get readergroups size", 2, readerGroupsList.getReaderGroups().size());
assertTrue(readerGroupsList.getReaderGroups().contains(new ReaderGroupsListReaderGroups().readerGroupName(readerGroupName1)));
assertTrue(readerGroupsList.getReaderGroups().contains(new ReaderGroupsListReaderGroups().readerGroupName(readerGroupName2)));
log.info("Get readergroups successful");
// Test fetching readergroup info.
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups/" + readerGroupName1).toString();
response = client.target(resourceURl).request().get();
assertEquals("Get readergroup properties status", OK.getStatusCode(), response.getStatus());
ReaderGroupProperty readerGroupProperty = response.readEntity(ReaderGroupProperty.class);
assertEquals("Get readergroup name", readerGroupName1, readerGroupProperty.getReaderGroupName());
assertEquals("Get readergroup scope name", testScope, readerGroupProperty.getScopeName());
assertEquals("Get readergroup streams size", 2, readerGroupProperty.getStreamList().size());
assertTrue(readerGroupProperty.getStreamList().contains(Stream.of(testScope, testStream1).getScopedName()));
assertTrue(readerGroupProperty.getStreamList().contains(Stream.of(testScope, testStream2).getScopedName()));
assertEquals("Get readergroup onlinereaders size", 2, readerGroupProperty.getOnlineReaderIds().size());
assertTrue(readerGroupProperty.getOnlineReaderIds().contains(reader1));
assertTrue(readerGroupProperty.getOnlineReaderIds().contains(reader2));
// Test readergroup or scope not found.
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups/" + "unknownreadergroup").toString();
response = client.target(resourceURl).request().get();
assertEquals("Get readergroup properties status", NOT_FOUND.getStatusCode(), response.getStatus());
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + "unknownscope" + "/readergroups/" + readerGroupName1).toString();
response = client.target(resourceURl).request().get();
assertEquals("Get readergroup properties status", NOT_FOUND.getStatusCode(), response.getStatus());
log.info("Get readergroup properties successful");
log.info("Test restApiTests passed successfully!");
}
use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.
the class StreamSeekTest method testStreamSeek.
@Test(timeout = 50000)
public void testStreamSeek() throws Exception {
createScope(SCOPE);
createStream(STREAM1);
createStream(STREAM2);
@Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(controllerUri).build());
@Cleanup EventStreamWriter<String> writer1 = clientFactory.createEventWriter(STREAM1, serializer, EventWriterConfig.builder().build());
@Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerUri);
groupManager.createReaderGroup("group", ReaderGroupConfig.builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(0).stream(Stream.of(SCOPE, STREAM1)).stream(Stream.of(SCOPE, STREAM2)).build());
@Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup("group");
// Prep the stream with data.
// 1.Write two events with event size of 30
writer1.writeEvent(keyGenerator.get(), getEventData.apply(1)).get();
writer1.writeEvent(keyGenerator.get(), getEventData.apply(2)).get();
// 2.Scale stream
Map<Double, Double> newKeyRanges = new HashMap<>();
newKeyRanges.put(0.0, 0.33);
newKeyRanges.put(0.33, 0.66);
newKeyRanges.put(0.66, 1.0);
scaleStream(STREAM1, newKeyRanges);
// 3.Write three events with event size of 30
writer1.writeEvent(keyGenerator.get(), getEventData.apply(3)).get();
writer1.writeEvent(keyGenerator.get(), getEventData.apply(4)).get();
writer1.writeEvent(keyGenerator.get(), getEventData.apply(5)).get();
// Create a reader
@Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "group", serializer, ReaderConfig.builder().build());
// Offset of a streamCut is always set to zero.
// Stream cut 1
Map<Stream, StreamCut> streamCut1 = readerGroup.getStreamCuts();
readAndVerify(reader, 1, 2);
// Sees the segments are empty prior to scaling
assertNull(reader.readNextEvent(100).getEvent());
// Checkpoint to move past the scale
readerGroup.initiateCheckpoint("cp1", executorService());
// Old segments are released and new ones can be read
readAndVerify(reader, 3, 4, 5);
// Stream cut 2
Map<Stream, StreamCut> streamCut2 = readerGroup.getStreamCuts();
// reset the readers to offset 0.
readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromStreamCuts(streamCut1).build());
verifyReinitializationRequiredException(reader);
@Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "group", serializer, ReaderConfig.builder().build());
// verify that we are at streamCut1
readAndVerify(reader1, 1, 2);
// reset readers to post scale offset 0
readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromStreamCuts(streamCut2).build());
verifyReinitializationRequiredException(reader1);
@Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("readerId", "group", serializer, ReaderConfig.builder().build());
// verify that we are at streamCut2
readAndVerify(reader2, 3, 4, 5);
}
use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.
the class UnreadBytesTest method testUnreadBytesWithEndStreamCuts.
@Test(timeout = 50000)
public void testUnreadBytesWithEndStreamCuts() throws Exception {
StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
String streamName = "testUnreadBytesWithEndStreamCuts";
Controller controller = PRAVEGA.getLocalController();
controller.createScope("unreadbytes").get();
controller.createStream("unreadbytes", streamName, config).get();
@Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope("unreadbytes", ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
@Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
// Write just 2 events to simplify simulating a checkpoint.
writer.writeEvent("0", "data of size 30").get();
writer.writeEvent("0", "data of size 30").get();
String group = "testUnreadBytesWithEndStreamCuts-group";
@Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope("unreadbytes", ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
// create a bounded reader group.
groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("unreadbytes/" + streamName, StreamCut.UNBOUNDED, getStreamCut(streamName, 90L, 0)).build());
ReaderGroup readerGroup = groupManager.getReaderGroup(group);
@Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new JavaSerializer<>(), ReaderConfig.builder().build());
EventRead<String> firstEvent = reader.readNextEvent(15000);
EventRead<String> secondEvent = reader.readNextEvent(15000);
assertNotNull(firstEvent);
assertEquals("data of size 30", firstEvent.getEvent());
assertNotNull(secondEvent);
assertEquals("data of size 30", secondEvent.getEvent());
// trigger a checkpoint.
CompletableFuture<Checkpoint> chkPointResult = readerGroup.initiateCheckpoint("test", executorService());
EventRead<String> chkpointEvent = reader.readNextEvent(15000);
assertEquals("test", chkpointEvent.getCheckpointName());
EventRead<String> emptyEvent = reader.readNextEvent(100);
assertEquals(false, emptyEvent.isCheckpoint());
assertEquals(null, emptyEvent.getEvent());
chkPointResult.join();
// Writer events, to ensure 120Bytes are written.
writer.writeEvent("0", "data of size 30").get();
writer.writeEvent("0", "data of size 30").get();
long unreadBytes = readerGroup.getMetrics().unreadBytes();
// Ensure the endoffset of 90 Bytes is taken into consideration when computing unread
assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 30);
}
use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.
the class StreamCutsTest method testReaderGroupCuts.
@Test(timeout = 40000)
public void testReaderGroupCuts() throws Exception {
StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
Controller controller = controllerWrapper.getController();
controllerWrapper.getControllerService().createScope("test", 0L).get();
controller.createStream("test", "test", config).get();
@Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
@Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
@Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().build());
writer.writeEvent("0", "fpj was here").get();
writer.writeEvent("0", "fpj was here again").get();
@Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
groupManager.createReaderGroup("cuts", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/test").groupRefreshTimeMillis(0).build());
@Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup("cuts");
@Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "cuts", new JavaSerializer<>(), ReaderConfig.builder().initialAllocationDelay(0).build());
EventRead<String> firstEvent = reader.readNextEvent(5000);
assertNotNull(firstEvent.getEvent());
assertEquals("fpj was here", firstEvent.getEvent());
readerGroup.initiateCheckpoint("cp1", executor);
EventRead<String> cpEvent = reader.readNextEvent(5000);
assertEquals("cp1", cpEvent.getCheckpointName());
EventRead<String> secondEvent = reader.readNextEvent(5000);
assertNotNull(secondEvent.getEvent());
assertEquals("fpj was here again", secondEvent.getEvent());
Map<Stream, StreamCut> cuts = readerGroup.getStreamCuts();
validateCuts(readerGroup, cuts, Collections.singleton(getQualifiedStreamSegmentName("test", "test", 0L)));
// Scale the stream to verify that we get more segments in the cut.
Stream stream = Stream.of("test", "test");
Map<Double, Double> map = new HashMap<>();
map.put(0.0, 0.5);
map.put(0.5, 1.0);
Boolean result = controller.scaleStream(stream, Collections.singletonList(0L), map, executor).getFuture().get();
assertTrue(result);
log.info("Finished 1st scaling");
writer.writeEvent("0", "fpj was here again0").get();
writer.writeEvent("1", "fpj was here again1").get();
EventRead<String> eosEvent = reader.readNextEvent(100);
// Reader does not yet see the data becasue there has been no CP
assertNull(eosEvent.getEvent());
CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("cp2", executor);
cpEvent = reader.readNextEvent(100);
EventRead<String> event0 = reader.readNextEvent(100);
EventRead<String> event1 = reader.readNextEvent(100);
cuts = checkpoint.get(5, TimeUnit.SECONDS).asImpl().getPositions();
// Validate the reader did not release the segments before the checkpoint.
// This is important because it means that once the checkpoint is initiated no segments change readers.
Set<String> segmentNames = ImmutableSet.of(getQualifiedStreamSegmentName("test", "test", computeSegmentId(0, 0)));
validateCuts(readerGroup, cuts, segmentNames);
CompletableFuture<Map<Stream, StreamCut>> futureCuts = readerGroup.generateStreamCuts(executor);
EventRead<String> emptyEvent = reader.readNextEvent(100);
cuts = futureCuts.get();
segmentNames = ImmutableSet.of(getQualifiedStreamSegmentName("test", "test", computeSegmentId(1, 1)), getQualifiedStreamSegmentName("test", "test", computeSegmentId(2, 1)));
validateCuts(readerGroup, cuts, segmentNames);
// Scale down to verify that the number drops back.
map = new HashMap<>();
map.put(0.0, 1.0);
ArrayList<Long> toSeal = new ArrayList<>();
toSeal.add(computeSegmentId(1, 1));
toSeal.add(computeSegmentId(2, 1));
result = controller.scaleStream(stream, Collections.unmodifiableList(toSeal), map, executor).getFuture().get();
assertTrue(result);
log.info("Finished 2nd scaling");
writer.writeEvent("0", "fpj was here again2").get();
// Reader sees the segment is empty
emptyEvent = reader.readNextEvent(100);
assertNull(emptyEvent.getEvent());
checkpoint = readerGroup.initiateCheckpoint("cp3", executor);
cpEvent = reader.readNextEvent(100);
assertEquals("cp3", cpEvent.getCheckpointName());
// Reader releases segments here
event0 = reader.readNextEvent(5000);
assertTrue(event0.getEvent().endsWith("2"));
cuts = readerGroup.getStreamCuts();
long three = computeSegmentId(3, 2);
validateCuts(readerGroup, cuts, Collections.singleton(getQualifiedStreamSegmentName("test", "test", three)));
// Scale up to 4 segments again.
map = new HashMap<>();
map.put(0.0, 0.25);
map.put(0.25, 0.5);
map.put(0.5, 0.75);
map.put(0.75, 1.0);
result = controller.scaleStream(stream, Collections.singletonList(three), map, executor).getFuture().get();
assertTrue(result);
log.info("Finished 3rd scaling");
writer.writeEvent("0", "fpj was here again3").get();
// Reader sees the segment is empty
emptyEvent = reader.readNextEvent(100);
assertNull(emptyEvent.getEvent());
readerGroup.initiateCheckpoint("cp4", executor);
cpEvent = reader.readNextEvent(1000);
assertEquals("cp4", cpEvent.getCheckpointName());
// Reader releases segments here
event0 = reader.readNextEvent(5000);
assertNotNull(event0.getEvent());
cuts = readerGroup.getStreamCuts();
segmentNames = new HashSet<>();
long four = computeSegmentId(4, 3);
long five = computeSegmentId(5, 3);
long six = computeSegmentId(6, 3);
long seven = computeSegmentId(7, 3);
segmentNames.add(getQualifiedStreamSegmentName("test", "test", four));
segmentNames.add(getQualifiedStreamSegmentName("test", "test", five));
segmentNames.add(getQualifiedStreamSegmentName("test", "test", six));
segmentNames.add(getQualifiedStreamSegmentName("test", "test", seven));
validateCuts(readerGroup, cuts, Collections.unmodifiableSet(segmentNames));
}
use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.
the class StreamRecreationTest method testStreamRecreation.
@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testStreamRecreation() throws Exception {
final String myScope = "myScope";
final String myStream = "myStream";
final String myReaderGroup = "myReaderGroup";
final int numIterations = 6;
// Create the scope and the stream.
@Cleanup StreamManager streamManager = StreamManager.create(controllerURI);
streamManager.createScope(myScope);
@Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(myScope, controllerURI);
final ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder().stream(Stream.of(myScope, myStream)).build();
for (int i = 0; i < numIterations; i++) {
log.info("Stream re-creation iteration {}.", i);
final String eventContent = "myEvent" + String.valueOf(i);
StreamConfiguration streamConfiguration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(i + 1)).build();
EventWriterConfig eventWriterConfig = EventWriterConfig.builder().build();
streamManager.createStream(myScope, myStream, streamConfiguration);
// Write a single event.
@Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(myScope, ClientConfig.builder().controllerURI(controllerURI).build());
@Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(myStream, new JavaSerializer<>(), eventWriterConfig);
TransactionalEventStreamWriter<String> txnWriter = clientFactory.createTransactionalEventWriter(myStream, new JavaSerializer<>(), eventWriterConfig);
// Write events regularly and with transactions.
if (i % 2 == 0) {
writer.writeEvent(eventContent).join();
} else {
Transaction<String> myTransaction = txnWriter.beginTxn();
myTransaction.writeEvent(eventContent);
myTransaction.commit();
while (myTransaction.checkStatus() != Transaction.Status.COMMITTED) {
Exceptions.handleInterrupted(() -> Thread.sleep(100));
}
}
writer.close();
// Read the event.
readerGroupManager.createReaderGroup(myReaderGroup, readerGroupConfig);
readerGroupManager.getReaderGroup(myReaderGroup).resetReaderGroup(readerGroupConfig);
@Cleanup EventStreamReader<String> reader = clientFactory.createReader("myReader", myReaderGroup, new JavaSerializer<>(), ReaderConfig.builder().build());
String readResult;
do {
readResult = reader.readNextEvent(1000).getEvent();
} while (readResult == null);
assertEquals("Wrong event read in re-created stream", eventContent, readResult);
// Delete the stream.
StreamInfo streamInfo = streamManager.getStreamInfo(myScope, myStream);
assertFalse(streamInfo.isSealed());
assertTrue("Unable to seal re-created stream.", streamManager.sealStream(myScope, myStream));
streamInfo = streamManager.getStreamInfo(myScope, myStream);
assertTrue(streamInfo.isSealed());
assertTrue("Unable to delete re-created stream.", streamManager.deleteStream(myScope, myStream));
}
}
Aggregations