use of io.pravega.client.admin.impl.StreamManagerImpl in project pravega by pravega.
the class ReadWriteTest method readWriteTest.
@Test(timeout = 60000)
public void readWriteTest() throws InterruptedException, ExecutionException {
String scope = "testMultiReaderWriterScope";
String readerGroupName = "testMultiReaderWriterReaderGroup";
// 20 readers -> 20 stream segments ( to have max read parallelism)
ScalingPolicy scalingPolicy = ScalingPolicy.fixed(20);
StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(STREAM_NAME).scalingPolicy(scalingPolicy).build();
eventsReadFromPravega = new ConcurrentLinkedQueue<>();
// data used by each of the writers.
eventData = new AtomicLong();
// used by readers to maintain a count of events.
eventReadCount = new AtomicLong();
stopReadFlag = new AtomicBoolean(false);
try (StreamManager streamManager = new StreamManagerImpl(controller)) {
// create a scope
Boolean createScopeStatus = streamManager.createScope(scope);
log.info("Create scope status {}", createScopeStatus);
// create a stream
Boolean createStreamStatus = streamManager.createStream(scope, STREAM_NAME, config);
log.info("Create stream status {}", createStreamStatus);
}
try (ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory, connectionFactory)) {
// start writing events to the stream
log.info("Creating {} writers", NUM_WRITERS);
List<CompletableFuture<Void>> writerList = new ArrayList<>();
for (int i = 0; i < NUM_WRITERS; i++) {
log.info("Starting writer{}", i);
writerList.add(startNewWriter(eventData, clientFactory));
}
// create a reader group
log.info("Creating Reader group : {}", readerGroupName);
readerGroupManager.createReaderGroup(readerGroupName, ReaderGroupConfig.builder().stream(Stream.of(scope, STREAM_NAME)).build());
log.info("Reader group name {} ", readerGroupManager.getReaderGroup(readerGroupName).getGroupName());
log.info("Reader group scope {}", readerGroupManager.getReaderGroup(readerGroupName).getScope());
// create readers
log.info("Creating {} readers", NUM_READERS);
List<CompletableFuture<Void>> readerList = new ArrayList<>();
String readerName = "reader" + new Random().nextInt(Integer.MAX_VALUE);
// start reading events
for (int i = 0; i < NUM_READERS; i++) {
log.info("Starting reader{}", i);
readerList.add(startNewReader(readerName + i, clientFactory, readerGroupName, eventsReadFromPravega, eventData, eventReadCount, stopReadFlag));
}
// wait for writers completion
Futures.allOf(writerList).get();
// set stop read flag to true
stopReadFlag.set(true);
// wait for readers completion
Futures.allOf(readerList).get();
// delete readergroup
log.info("Deleting readergroup {}", readerGroupName);
readerGroupManager.deleteReaderGroup(readerGroupName);
}
log.info("All writers have stopped. Setting Stop_Read_Flag. Event Written Count:{}, Event Read " + "Count: {}", eventData.get(), eventsReadFromPravega.size());
assertEquals(TOTAL_NUM_EVENTS, eventsReadFromPravega.size());
// check unique events.
assertEquals(TOTAL_NUM_EVENTS, new TreeSet<>(eventsReadFromPravega).size());
// seal the stream
CompletableFuture<Boolean> sealStreamStatus = controller.sealStream(scope, STREAM_NAME);
log.info("Sealing stream {}", STREAM_NAME);
assertTrue(sealStreamStatus.get());
// delete the stream
CompletableFuture<Boolean> deleteStreamStatus = controller.deleteStream(scope, STREAM_NAME);
log.info("Deleting stream {}", STREAM_NAME);
assertTrue(deleteStreamStatus.get());
// delete the scope
CompletableFuture<Boolean> deleteScopeStatus = controller.deleteScope(scope);
log.info("Deleting scope {}", scope);
assertTrue(deleteScopeStatus.get());
log.info("Read write test succeeds");
}
use of io.pravega.client.admin.impl.StreamManagerImpl in project pravega by pravega.
the class MultiReaderTxnWriterWithFailoverTest method setup.
@Before
public void setup() {
// Get zk details to verify if controller, SSS are running
Service zkService = Utils.createZookeeperService();
List<URI> zkUris = zkService.getServiceDetails();
log.debug("Zookeeper service details: {}", zkUris);
// get the zk ip details and pass it to host, controller
URI zkUri = zkUris.get(0);
// Verify controller is running.
controllerInstance = Utils.createPravegaControllerService(zkUri);
assertTrue(controllerInstance.isRunning());
List<URI> conURIs = controllerInstance.getServiceDetails();
log.info("Pravega Controller service instance details: {}", conURIs);
// Fetch all the RPC endpoints and construct the client URIs.
final List<String> uris = conURIs.stream().filter(uri -> Utils.DOCKER_BASED ? uri.getPort() == Utils.DOCKER_CONTROLLER_PORT : uri.getPort() == Utils.MARATHON_CONTROLLER_PORT).map(URI::getAuthority).collect(Collectors.toList());
controllerURIDirect = URI.create("tcp://" + String.join(",", uris));
log.info("Controller Service direct URI: {}", controllerURIDirect);
// Verify segment store is running.
segmentStoreInstance = Utils.createPravegaSegmentStoreService(zkUri, controllerURIDirect);
assertTrue(segmentStoreInstance.isRunning());
log.info("Pravega Segmentstore service instance details: {}", segmentStoreInstance.getServiceDetails());
// executor service
executorService = ExecutorServiceHelpers.newScheduledThreadPool(NUM_READERS + NUM_WRITERS + 2, "MultiReaderTxnWriterWithFailoverTest-main");
controllerExecutorService = ExecutorServiceHelpers.newScheduledThreadPool(2, "MultiReaderTxnWriterWithFailoverTest-controller");
// get Controller Uri
controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(ClientConfig.builder().controllerURI(controllerURIDirect).build()).maxBackoffMillis(5000).build(), controllerExecutorService);
testState = new TestState(true);
// read and write count variables
testState.writersListComplete.add(0, testState.writersComplete);
streamManager = new StreamManagerImpl(ClientConfig.builder().controllerURI(controllerURIDirect).build());
createScopeAndStream(scope, STREAM_NAME, config, streamManager);
log.info("Scope passed to client factory {}", scope);
clientFactory = new ClientFactoryImpl(scope, controller);
readerGroupManager = ReaderGroupManager.withScope(scope, ClientConfig.builder().controllerURI(controllerURIDirect).build());
}
use of io.pravega.client.admin.impl.StreamManagerImpl in project pravega by pravega.
the class ReadTxnWriteScaleWithFailoverTest method setup.
@Before
public void setup() {
// Get zk details to verify if controller, segmentstore are running
Service zkService = Utils.createZookeeperService();
List<URI> zkUris = zkService.getServiceDetails();
log.debug("Zookeeper service details: {}", zkUris);
// get the zk ip details and pass it to host, controller
URI zkUri = zkUris.get(0);
// Verify controller is running.
controllerInstance = Utils.createPravegaControllerService(zkUri);
assertTrue(controllerInstance.isRunning());
List<URI> conURIs = controllerInstance.getServiceDetails();
log.info("Pravega Controller service instance details: {}", conURIs);
// Fetch all the RPC endpoints and construct the client URIs.
final List<String> uris = conURIs.stream().filter(uri -> Utils.DOCKER_BASED ? uri.getPort() == Utils.DOCKER_CONTROLLER_PORT : uri.getPort() == Utils.MARATHON_CONTROLLER_PORT).map(URI::getAuthority).collect(Collectors.toList());
controllerURIDirect = URI.create("tcp://" + String.join(",", uris));
log.info("Controller Service direct URI: {}", controllerURIDirect);
// Verify segment store is running.
segmentStoreInstance = Utils.createPravegaSegmentStoreService(zkUri, controllerURIDirect);
assertTrue(segmentStoreInstance.isRunning());
log.info("Pravega Segmentstore service instance details: {}", segmentStoreInstance.getServiceDetails());
// num. of readers + num. of writers + 1 to run checkScale operation
executorService = ExecutorServiceHelpers.newScheduledThreadPool(NUM_READERS + NUM_WRITERS + 1, "ReadTxnWriteScaleWithFailoverTest-main");
controllerExecutorService = ExecutorServiceHelpers.newScheduledThreadPool(2, "ReadTxnWriteScaleWithFailoverTest-controller");
// get Controller Uri
controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(ClientConfig.builder().controllerURI(controllerURIDirect).build()).maxBackoffMillis(5000).build(), controllerExecutorService);
testState = new TestState(true);
testState.writersListComplete.add(0, testState.writersComplete);
streamManager = new StreamManagerImpl(ClientConfig.builder().controllerURI(controllerURIDirect).build());
createScopeAndStream(scope, stream, config, streamManager);
log.info("Scope passed to client factory {}", scope);
clientFactory = new ClientFactoryImpl(scope, controller);
readerGroupManager = ReaderGroupManager.withScope(scope, ClientConfig.builder().controllerURI(controllerURIDirect).build());
}
use of io.pravega.client.admin.impl.StreamManagerImpl in project pravega by pravega.
the class ReadWriteAndAutoScaleWithFailoverTest method setup.
@Before
public void setup() {
// Get zk details to verify if controller, SSS are running
Service zkService = Utils.createZookeeperService();
List<URI> zkUris = zkService.getServiceDetails();
log.debug("Zookeeper service details: {}", zkUris);
// get the zk ip details and pass it to host, controller
URI zkUri = zkUris.get(0);
// Verify controller is running.
controllerInstance = Utils.createPravegaControllerService(zkUri);
assertTrue(controllerInstance.isRunning());
List<URI> conURIs = controllerInstance.getServiceDetails();
log.info("Pravega Controller service instance details: {}", conURIs);
// Fetch all the RPC endpoints and construct the client URIs.
final List<String> uris = conURIs.stream().filter(uri -> Utils.DOCKER_BASED ? uri.getPort() == Utils.DOCKER_CONTROLLER_PORT : uri.getPort() == Utils.MARATHON_CONTROLLER_PORT).map(URI::getAuthority).collect(Collectors.toList());
controllerURIDirect = URI.create("tcp://" + String.join(",", uris));
log.info("Controller Service direct URI: {}", controllerURIDirect);
// Verify segment store is running.
segmentStoreInstance = Utils.createPravegaSegmentStoreService(zkUri, controllerURIDirect);
assertTrue(segmentStoreInstance.isRunning());
log.info("Pravega Segmentstore service instance details: {}", segmentStoreInstance.getServiceDetails());
// executor service
executorService = ExecutorServiceHelpers.newScheduledThreadPool(NUM_READERS + TOTAL_NUM_WRITERS + 1, "ReadWriteAndAutoScaleWithFailoverTest-main");
controllerExecutorService = ExecutorServiceHelpers.newScheduledThreadPool(2, "ReadWriteAndAutoScaleWithFailoverTest-controller");
// get Controller Uri
controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(ClientConfig.builder().controllerURI(controllerURIDirect).build()).maxBackoffMillis(5000).build(), controllerExecutorService);
testState = new TestState(false);
testState.writersListComplete.add(0, testState.writersComplete);
testState.writersListComplete.add(1, testState.newWritersComplete);
streamManager = new StreamManagerImpl(ClientConfig.builder().controllerURI(controllerURIDirect).build());
createScopeAndStream(scope, AUTO_SCALE_STREAM, config, streamManager);
log.info("Scope passed to client factory {}", scope);
clientFactory = new ClientFactoryImpl(scope, controller);
readerGroupManager = ReaderGroupManager.withScope(scope, ClientConfig.builder().controllerURI(controllerURIDirect).build());
}
use of io.pravega.client.admin.impl.StreamManagerImpl in project pravega by pravega.
the class ControllerRestApiTest method restApiTests.
@Test
public void restApiTests() {
Invocation.Builder builder;
Response response;
restServerURI = SETUP_UTILS.getControllerRestUri().toString();
log.info("REST Server URI: {}", restServerURI);
// TEST REST server status, ping test
resourceURl = new StringBuilder(restServerURI).append("/ping").toString();
webTarget = client.target(resourceURl);
builder = webTarget.request();
response = builder.get();
assertEquals("Ping test", OK.getStatusCode(), response.getStatus());
log.info("REST Server is running. Ping successful.");
final String scope1 = RandomStringUtils.randomAlphanumeric(10);
final String stream1 = RandomStringUtils.randomAlphanumeric(10);
// TEST CreateScope POST http://controllerURI:Port/v1/scopes
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes").toString();
webTarget = client.target(resourceURl);
final CreateScopeRequest createScopeRequest = new CreateScopeRequest();
createScopeRequest.setScopeName(scope1);
builder = webTarget.request(MediaType.APPLICATION_JSON_TYPE);
response = builder.post(Entity.json(createScopeRequest));
assertEquals("Create scope status", CREATED.getStatusCode(), response.getStatus());
Assert.assertEquals("Create scope response", scope1, response.readEntity(ScopeProperty.class).getScopeName());
log.info("Create scope: {} successful ", scope1);
// TEST CreateStream POST http://controllerURI:Port/v1/scopes/{scopeName}/streams
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams").toString();
webTarget = client.target(resourceURl);
CreateStreamRequest createStreamRequest = new CreateStreamRequest();
ScalingConfig scalingConfig = new ScalingConfig();
scalingConfig.setType(ScalingConfig.TypeEnum.FIXED_NUM_SEGMENTS);
scalingConfig.setTargetRate(2);
scalingConfig.scaleFactor(2);
scalingConfig.minSegments(2);
RetentionConfig retentionConfig = new RetentionConfig();
retentionConfig.setType(RetentionConfig.TypeEnum.LIMITED_DAYS);
retentionConfig.setValue(123L);
createStreamRequest.setStreamName(stream1);
createStreamRequest.setScalingPolicy(scalingConfig);
createStreamRequest.setRetentionPolicy(retentionConfig);
builder = webTarget.request(MediaType.APPLICATION_JSON_TYPE);
response = builder.post(Entity.json(createStreamRequest));
assertEquals("Create stream status", CREATED.getStatusCode(), response.getStatus());
final StreamProperty streamPropertyResponse = response.readEntity(StreamProperty.class);
assertEquals("Scope name in response", scope1, streamPropertyResponse.getScopeName());
assertEquals("Stream name in response", stream1, streamPropertyResponse.getStreamName());
log.info("Create stream: {} successful", stream1);
// Test listScopes GET http://controllerURI:Port/v1/scopes/{scopeName}/streams
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes").toString();
webTarget = client.target(resourceURl);
builder = webTarget.request();
response = builder.get();
assertEquals("List scopes", OK.getStatusCode(), response.getStatus());
log.info("List scopes successful");
// Test listStream GET /v1/scopes/scope1/streams
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams").toString();
webTarget = client.target(resourceURl);
builder = webTarget.request();
response = builder.get();
assertEquals("List streams", OK.getStatusCode(), response.getStatus());
Assert.assertEquals("List streams size", 1, response.readEntity(StreamsList.class).getStreams().size());
log.info("List streams successful");
// Test getScope
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1).toString();
response = client.target(resourceURl).request().get();
assertEquals("Get scope status", OK.getStatusCode(), response.getStatus());
assertEquals("Get scope scope1 response", scope1, response.readEntity(ScopeProperty.class).getScopeName());
log.info("Get scope successful");
// Test updateStream
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1).toString();
UpdateStreamRequest updateStreamRequest = new UpdateStreamRequest();
ScalingConfig scalingConfig1 = new ScalingConfig();
scalingConfig1.setType(ScalingConfig.TypeEnum.FIXED_NUM_SEGMENTS);
scalingConfig1.setTargetRate(2);
// update existing scaleFactor from 2 to 3
scalingConfig1.scaleFactor(3);
// update existing minSegments from 2 to 4
scalingConfig1.minSegments(4);
updateStreamRequest.setScalingPolicy(scalingConfig1);
updateStreamRequest.setRetentionPolicy(retentionConfig);
response = client.target(resourceURl).request(MediaType.APPLICATION_JSON_TYPE).put(Entity.json(updateStreamRequest));
assertEquals("Update stream status", OK.getStatusCode(), response.getStatus());
assertEquals("Verify updated property", 4, response.readEntity(StreamProperty.class).getScalingPolicy().getMinSegments().intValue());
log.info("Update stream successful");
// Test getStream
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1).toString();
response = client.target(resourceURl).request().get();
assertEquals("Get stream status", OK.getStatusCode(), response.getStatus());
assertEquals("Get stream stream1 response", stream1, response.readEntity(StreamProperty.class).getStreamName());
log.info("Get stream successful");
// Test updateStreamState
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1 + "/state").toString();
StreamState streamState = new StreamState();
streamState.setStreamState(StreamState.StreamStateEnum.SEALED);
response = client.target(resourceURl).request(MediaType.APPLICATION_JSON_TYPE).put(Entity.json(streamState));
assertEquals("UpdateStreamState status", OK.getStatusCode(), response.getStatus());
assertEquals("UpdateStreamState status in response", streamState.getStreamState(), response.readEntity(StreamState.class).getStreamState());
log.info("Update stream state successful");
// Test deleteStream
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1 + "/streams/" + stream1).toString();
response = client.target(resourceURl).request().delete();
assertEquals("DeleteStream status", NO_CONTENT.getStatusCode(), response.getStatus());
log.info("Delete stream successful");
// Test deleteScope
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + scope1).toString();
response = client.target(resourceURl).request().delete();
assertEquals("Get scope status", NO_CONTENT.getStatusCode(), response.getStatus());
log.info("Delete Scope successful");
// Test reader groups APIs.
// Prepare the streams and readers using the admin client.
final String testScope = RandomStringUtils.randomAlphanumeric(10);
final String testStream1 = RandomStringUtils.randomAlphanumeric(10);
final String testStream2 = RandomStringUtils.randomAlphanumeric(10);
URI controllerUri = SETUP_UTILS.getControllerUri();
@Cleanup("shutdown") InlineExecutor inlineExecutor = new InlineExecutor();
try (StreamManager streamManager = new StreamManagerImpl(createController(controllerUri, inlineExecutor))) {
log.info("Creating scope: {}", testScope);
streamManager.createScope(testScope);
log.info("Creating stream: {}", testStream1);
StreamConfiguration streamConf1 = StreamConfiguration.builder().scope(testScope).streamName(testStream1).scalingPolicy(ScalingPolicy.fixed(1)).build();
streamManager.createStream(testScope, testStream1, streamConf1);
log.info("Creating stream: {}", testStream2);
StreamConfiguration streamConf2 = StreamConfiguration.builder().scope(testScope).streamName(testStream2).scalingPolicy(ScalingPolicy.fixed(1)).build();
streamManager.createStream(testScope, testStream2, streamConf2);
}
final String readerGroupName1 = RandomStringUtils.randomAlphanumeric(10);
final String readerGroupName2 = RandomStringUtils.randomAlphanumeric(10);
final String reader1 = RandomStringUtils.randomAlphanumeric(10);
final String reader2 = RandomStringUtils.randomAlphanumeric(10);
try (ClientFactory clientFactory = new ClientFactoryImpl(testScope, createController(controllerUri, inlineExecutor));
ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(testScope, ClientConfig.builder().controllerURI(controllerUri).build())) {
readerGroupManager.createReaderGroup(readerGroupName1, ReaderGroupConfig.builder().stream(Stream.of(testScope, testStream1)).stream(Stream.of(testScope, testStream2)).build());
readerGroupManager.createReaderGroup(readerGroupName2, ReaderGroupConfig.builder().stream(Stream.of(testScope, testStream1)).stream(Stream.of(testScope, testStream2)).build());
clientFactory.createReader(reader1, readerGroupName1, new JavaSerializer<Long>(), ReaderConfig.builder().build());
clientFactory.createReader(reader2, readerGroupName1, new JavaSerializer<Long>(), ReaderConfig.builder().build());
}
// Test fetching readergroups.
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups").toString();
response = client.target(resourceURl).request().get();
assertEquals("Get readergroups status", OK.getStatusCode(), response.getStatus());
ReaderGroupsList readerGroupsList = response.readEntity(ReaderGroupsList.class);
assertEquals("Get readergroups size", 2, readerGroupsList.getReaderGroups().size());
assertTrue(readerGroupsList.getReaderGroups().contains(new ReaderGroupsListReaderGroups().readerGroupName(readerGroupName1)));
assertTrue(readerGroupsList.getReaderGroups().contains(new ReaderGroupsListReaderGroups().readerGroupName(readerGroupName2)));
log.info("Get readergroups successful");
// Test fetching readergroup info.
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups/" + readerGroupName1).toString();
response = client.target(resourceURl).request().get();
assertEquals("Get readergroup properties status", OK.getStatusCode(), response.getStatus());
ReaderGroupProperty readerGroupProperty = response.readEntity(ReaderGroupProperty.class);
assertEquals("Get readergroup name", readerGroupName1, readerGroupProperty.getReaderGroupName());
assertEquals("Get readergroup scope name", testScope, readerGroupProperty.getScopeName());
assertEquals("Get readergroup streams size", 2, readerGroupProperty.getStreamList().size());
assertTrue(readerGroupProperty.getStreamList().contains(testStream1));
assertTrue(readerGroupProperty.getStreamList().contains(testStream2));
assertEquals("Get readergroup onlinereaders size", 2, readerGroupProperty.getOnlineReaderIds().size());
assertTrue(readerGroupProperty.getOnlineReaderIds().contains(reader1));
assertTrue(readerGroupProperty.getOnlineReaderIds().contains(reader2));
// Test readergroup or scope not found.
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + testScope + "/readergroups/" + "unknownreadergroup").toString();
response = client.target(resourceURl).request().get();
assertEquals("Get readergroup properties status", NOT_FOUND.getStatusCode(), response.getStatus());
resourceURl = new StringBuilder(restServerURI).append("/v1/scopes/" + "unknownscope" + "/readergroups/" + readerGroupName1).toString();
response = client.target(resourceURl).request().get();
assertEquals("Get readergroup properties status", NOT_FOUND.getStatusCode(), response.getStatus());
log.info("Get readergroup properties successful");
log.info("Test restApiTests passed successfully!");
}
Aggregations