use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.
the class EndToEndTruncationTest method testWriteDuringScaleAndTruncation.
@Test(timeout = 50000)
public void testWriteDuringScaleAndTruncation() throws Exception {
String streamName = "testWriteDuringScaleAndTruncation";
Stream stream = new StreamImpl("test", streamName);
StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 2)).build();
LocalController controller = (LocalController) PRAVEGA.getLocalController();
controller.createScope("test").get();
controller.createStream("test", streamName, config).get();
config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
controller.updateStream("test", streamName, config).get();
@Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
@Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
@Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
// routing key "0" translates to key 0.8. This write happens to segment 1.
writer.writeEvent("0", "truncationTest1").get();
// Peform scaling operations on the stream.
ImmutableMap<Double, Double> singleSegmentKeyRange = ImmutableMap.of(0.0, 1.0);
ImmutableMap<Double, Double> twoSegmentKeyRange = ImmutableMap.of(0.0, 0.5, 0.5, 1.0);
// scale down to 1 segment.
assertTrue("Stream Scale down", controller.scaleStream(stream, Lists.newArrayList(0L, 1L), singleSegmentKeyRange, executorService()).getFuture().get());
// scale up to 2 segments.
assertTrue("Stream Scale up", controller.scaleStream(stream, Lists.newArrayList(computeSegmentId(2, 1)), twoSegmentKeyRange, executorService()).getFuture().get());
// scale down to 1 segment.
assertTrue("Stream Scale down", controller.scaleStream(stream, Lists.newArrayList(computeSegmentId(3, 2), computeSegmentId(4, 2)), singleSegmentKeyRange, executorService()).getFuture().get());
// scale up to 2 segments.
assertTrue("Stream Scale up", controller.scaleStream(stream, Lists.newArrayList(computeSegmentId(5, 3)), twoSegmentKeyRange, executorService()).getFuture().get());
// truncateStream.
Map<Long, Long> streamCutPositions = new HashMap<>();
streamCutPositions.put(computeSegmentId(3, 2), 0L);
streamCutPositions.put(computeSegmentId(4, 2), 0L);
assertTrue("Truncate stream", controller.truncateStream("test", streamName, streamCutPositions).get());
// write an event.
writer.writeEvent("0", "truncationTest3");
writer.flush();
// Read the event back.
String group = "testWriteDuringScaleAndTruncation-group";
@Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(0).stream("test/" + streamName).build());
@Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new JavaSerializer<>(), ReaderConfig.builder().build());
EventRead<String> event = reader.readNextEvent(200);
assertNull(event.getEvent());
groupManager.getReaderGroup(group).initiateCheckpoint("cp1", executorService());
event = reader.readNextEvent(2000);
assertEquals("cp1", event.getCheckpointName());
event = reader.readNextEvent(200);
assertNull(event.getEvent());
groupManager.getReaderGroup(group).initiateCheckpoint("cp2", executorService());
event = reader.readNextEvent(2000);
assertEquals("cp2", event.getCheckpointName());
event = reader.readNextEvent(10000);
assertEquals("truncationTest3", event.getEvent());
}
use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.
the class EndToEndWithScaleTest method testScale.
@Test(timeout = 90000)
public void testScale() throws Exception {
final String scope = "test";
final String streamName = "test";
StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
// Test scale both in a new stream and in a re-created one.
for (int i = 0; i < 2; i++) {
@Cleanup Controller controller = controllerWrapper.getController();
controllerWrapper.getControllerService().createScope(scope, 0L).get();
controller.createStream(scope, streamName, config).get();
@Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
@Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
@Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
writer.writeEvent("0", "txntest1" + i).get();
// scale
Stream stream = new StreamImpl(scope, streamName);
Map<Double, Double> map = new HashMap<>();
map.put(0.0, 0.33);
map.put(0.33, 0.66);
map.put(0.66, 1.0);
Boolean result = controller.scaleStream(stream, Collections.singletonList(i * 4L), map, executorService()).getFuture().get();
assertTrue(result);
writer.writeEvent("0", "txntest2" + i).get();
@Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory);
groupManager.createReaderGroup("reader" + i, ReaderGroupConfig.builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(0).stream(Stream.of(scope, streamName).getScopedName()).build());
@Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId" + i, "reader" + i, new JavaSerializer<>(), ReaderConfig.builder().build());
EventRead<String> event = reader.readNextEvent(10000);
assertNotNull(event.getEvent());
assertEquals("txntest1" + i, event.getEvent());
event = reader.readNextEvent(100);
assertNull(event.getEvent());
groupManager.getReaderGroup("reader" + i).initiateCheckpoint("cp" + i, executorService());
event = reader.readNextEvent(10000);
assertEquals("cp" + i, event.getCheckpointName());
event = reader.readNextEvent(10000);
assertEquals("txntest2" + i, event.getEvent());
assertTrue(controller.sealStream(scope, streamName).join());
assertTrue(controller.deleteStream(scope, streamName).join());
}
}
use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.
the class ScopeTest method testForceDeleteScope.
@Test
public void testForceDeleteScope() throws Exception {
final String scope = "test";
final String streamName1 = "test1";
final String streamName2 = "test2";
final String streamName3 = "test3";
final String kvtName1 = "kvt1";
final String kvtName2 = "kvt2";
final String groupName1 = "rg1";
final String groupName2 = "rg2";
StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
@Cleanup Controller controller = controllerWrapper.getController();
ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create("tcp://localhost:" + controllerPort)).build();
@Cleanup ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
@Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
controllerWrapper.getControllerService().createScope(scope, 0L).get();
controller.createStream(scope, streamName1, config).get();
controller.createStream(scope, streamName2, config).get();
controller.createStream(scope, streamName3, config).get();
@Cleanup StreamManager streamManager = new StreamManagerImpl(controller, cp);
@Cleanup KeyValueTableManager keyValueTableManager = new KeyValueTableManagerImpl(clientConfig);
@Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, clientConfig, connectionFactory);
KeyValueTableConfiguration kvtConfig = KeyValueTableConfiguration.builder().partitionCount(2).primaryKeyLength(4).secondaryKeyLength(4).build();
keyValueTableManager.createKeyValueTable(scope, kvtName1, kvtConfig);
keyValueTableManager.createKeyValueTable(scope, kvtName2, kvtConfig);
readerGroupManager.createReaderGroup(groupName1, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName1)).build());
readerGroupManager.createReaderGroup(groupName2, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName2)).build());
assertTrue(streamManager.deleteScope(scope, true));
}
use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.
the class ScopeTest method testDeleteScopeRecursive.
@Test
public void testDeleteScopeRecursive() throws Exception {
final String scope = "testDeleteScope";
final String testFalseScope = "falseScope";
final String streamName1 = "test1";
final String streamName2 = "test2";
final String streamName3 = "test3";
final String streamName4 = "test4";
final String kvtName1 = "kvt1";
final String kvtName2 = "kvt2";
final String kvtName3 = "kvt3";
final String groupName1 = "rg1";
final String groupName2 = "rg2";
final String groupName3 = "rg3";
StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
@Cleanup Controller controller = controllerWrapper.getController();
ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create("tcp://localhost:" + controllerPort)).build();
@Cleanup ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
@Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
// Create scope
controllerWrapper.getControllerService().createScope(scope, 0L).get();
assertTrue(controller.checkScopeExists(scope).get());
// Create streams
assertTrue(controller.createStream(scope, streamName1, config).get());
assertTrue(controller.createStream(scope, streamName2, config).get());
assertTrue(controller.createStream(scope, streamName3, config).get());
@Cleanup StreamManager streamManager = new StreamManagerImpl(controller, cp);
@Cleanup KeyValueTableManager keyValueTableManager = new KeyValueTableManagerImpl(clientConfig);
@Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, clientConfig, connectionFactory);
// 1. Call deleteScopeRecursive() without creating a scope
assertTrue(streamManager.deleteScopeRecursive(testFalseScope));
// Create KVT under the scope
KeyValueTableConfiguration kvtConfig = KeyValueTableConfiguration.builder().partitionCount(2).primaryKeyLength(4).secondaryKeyLength(4).build();
assertTrue(keyValueTableManager.createKeyValueTable(scope, kvtName1, kvtConfig));
assertTrue(keyValueTableManager.createKeyValueTable(scope, kvtName2, kvtConfig));
// Create RG under the same scope
assertTrue(readerGroupManager.createReaderGroup(groupName1, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName1)).build()));
assertTrue(readerGroupManager.createReaderGroup(groupName2, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName2)).build()));
// Call deleteScopeRecursive to delete the scope recursively
assertTrue(streamManager.deleteScopeRecursive(scope));
// Validate that the scope is deleted
assertFalse(controller.checkScopeExists(scope).get());
// Validate create operation of Stream/RG/KVT should throw error
AssertExtensions.assertThrows("Failed to create Reader Group as Scope does not exits", () -> readerGroupManager.createReaderGroup(groupName3, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName2)).build()), e -> e instanceof IllegalArgumentException);
AssertExtensions.assertThrows("Scope does not exist", () -> controller.createStream(scope, streamName4, config).get(), e -> e instanceof IllegalArgumentException);
AssertExtensions.assertThrows("Scope does not exist", () -> keyValueTableManager.createKeyValueTable(scope, kvtName3, kvtConfig), e -> e instanceof IllegalArgumentException);
}
use of io.pravega.client.admin.ReaderGroupManager in project pravega by pravega.
the class StreamTransactionMetadataTasksTest method createEventProcessor.
private <T extends ControllerEvent> void createEventProcessor(final String readerGroupName, final String streamName, final EventStreamReader<T> reader, final EventStreamWriter<T> writer, Supplier<EventProcessor<T>> factory) throws CheckpointStoreException {
EventStreamClientFactory clientFactory = Mockito.mock(EventStreamClientFactory.class);
Mockito.when(clientFactory.<T>createReader(anyString(), anyString(), any(), any())).thenReturn(reader);
Mockito.when(clientFactory.<T>createEventWriter(anyString(), any(), any())).thenReturn(writer);
ReaderGroup readerGroup = Mockito.mock(ReaderGroup.class);
Mockito.when(readerGroup.getGroupName()).thenReturn(readerGroupName);
ReaderGroupManager readerGroupManager = Mockito.mock(ReaderGroupManager.class);
EventProcessorSystemImpl system = new EventProcessorSystemImpl("system", "host", SCOPE, clientFactory, readerGroupManager);
EventProcessorGroupConfig eventProcessorConfig = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(readerGroupName).streamName(streamName).checkpointConfig(CheckpointConfig.periodic(1, 1)).build();
EventProcessorConfig<T> config = EventProcessorConfig.<T>builder().config(eventProcessorConfig).decider(ExceptionHandler.DEFAULT_EXCEPTION_HANDLER).serializer(new EventSerializer<>()).supplier(factory).build();
system.createEventProcessorGroup(config, CheckpointStoreFactory.createInMemoryStore(), executor);
}
Aggregations