Examples with ReaderGroupManagerImpl io.pravega.client.admin.impl.ReaderGroupManagerImpl used on opensource projects

Example 31 with ReaderGroupManagerImpl

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl in project pravega by pravega.

the class EndToEndTransactionOrderTest method setUp.

@Before
public void setUp() throws Exception {
    zkTestServer = new TestingServerStarter().start();
    int port = Config.SERVICE_PORT;
    controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), false, controllerPort, serviceHost, servicePort, Config.HOST_STORE_CONTAINER_COUNT);
    controller = controllerWrapper.getController();
    connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, connectionFactory);
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    controllerWrapper.getControllerService().createScope(NameUtils.INTERNAL_SCOPE_NAME, 0L).get();
    autoScaleMonitor = new AutoScaleMonitor(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).build());
    server = new PravegaConnectionListener(false, false, "localhost", servicePort, store, tableStore, autoScaleMonitor.getStatsRecorder(), autoScaleMonitor.getTableSegmentStatsRecorder(), null, null, null, true, serviceBuilder.getLowPriorityExecutor(), SecurityConfigDefaults.TLS_PROTOCOL_VERSION);
    server.startListening();
    controllerWrapper.awaitRunning();
    controllerWrapper.getControllerService().createScope("test", 0L).get();
    controller.createStream("test", "test", config).get();
    clientFactory = new MockClientFactory("test", controller, internalCF.getConnectionPool());
    readerGroupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
    readerGroupManager.createReaderGroup("readergrp", ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(2000).groupRefreshTimeMillis(1000).stream("test/test").build());
    reader = clientFactory.createReader("1", "readergrp", new IntegerSerializer(), ReaderConfig.builder().build());
}

Example 32 with ReaderGroupManagerImpl

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl in project pravega by pravega.

the class EndToEndWithScaleTest method testScale.

@Test(timeout = 90000)
public void testScale() throws Exception {
    final String scope = "test";
    final String streamName = "test";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    // Test scale both in a new stream and in a re-created one.
    for (int i = 0; i < 2; i++) {
        @Cleanup Controller controller = controllerWrapper.getController();
        controllerWrapper.getControllerService().createScope(scope, 0L).get();
        controller.createStream(scope, streamName, config).get();
        @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
        @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
        @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
        writer.writeEvent("0", "txntest1" + i).get();
        // scale
        Stream stream = new StreamImpl(scope, streamName);
        Map<Double, Double> map = new HashMap<>();
        map.put(0.0, 0.33);
        map.put(0.33, 0.66);
        map.put(0.66, 1.0);
        Boolean result = controller.scaleStream(stream, Collections.singletonList(i * 4L), map, executorService()).getFuture().get();
        assertTrue(result);
        writer.writeEvent("0", "txntest2" + i).get();
        @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory);
        groupManager.createReaderGroup("reader" + i, ReaderGroupConfig.builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(0).stream(Stream.of(scope, streamName).getScopedName()).build());
        @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId" + i, "reader" + i, new JavaSerializer<>(), ReaderConfig.builder().build());
        EventRead<String> event = reader.readNextEvent(10000);
        assertNotNull(event.getEvent());
        assertEquals("txntest1" + i, event.getEvent());
        event = reader.readNextEvent(100);
        assertNull(event.getEvent());
        groupManager.getReaderGroup("reader" + i).initiateCheckpoint("cp" + i, executorService());
        event = reader.readNextEvent(10000);
        assertEquals("cp" + i, event.getCheckpointName());
        event = reader.readNextEvent(10000);
        assertEquals("txntest2" + i, event.getEvent());
        assertTrue(controller.sealStream(scope, streamName).join());
        assertTrue(controller.deleteStream(scope, streamName).join());
    }
}

Example 33 with ReaderGroupManagerImpl

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl in project pravega by pravega.

the class ScopeTest method testForceDeleteScope.

@Test
public void testForceDeleteScope() throws Exception {
    final String scope = "test";
    final String streamName1 = "test1";
    final String streamName2 = "test2";
    final String streamName3 = "test3";
    final String kvtName1 = "kvt1";
    final String kvtName2 = "kvt2";
    final String groupName1 = "rg1";
    final String groupName2 = "rg2";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    @Cleanup Controller controller = controllerWrapper.getController();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create("tcp://localhost:" + controllerPort)).build();
    @Cleanup ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    controllerWrapper.getControllerService().createScope(scope, 0L).get();
    controller.createStream(scope, streamName1, config).get();
    controller.createStream(scope, streamName2, config).get();
    controller.createStream(scope, streamName3, config).get();
    @Cleanup StreamManager streamManager = new StreamManagerImpl(controller, cp);
    @Cleanup KeyValueTableManager keyValueTableManager = new KeyValueTableManagerImpl(clientConfig);
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, clientConfig, connectionFactory);
    KeyValueTableConfiguration kvtConfig = KeyValueTableConfiguration.builder().partitionCount(2).primaryKeyLength(4).secondaryKeyLength(4).build();
    keyValueTableManager.createKeyValueTable(scope, kvtName1, kvtConfig);
    keyValueTableManager.createKeyValueTable(scope, kvtName2, kvtConfig);
    readerGroupManager.createReaderGroup(groupName1, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName1)).build());
    readerGroupManager.createReaderGroup(groupName2, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName2)).build());
    assertTrue(streamManager.deleteScope(scope, true));
}

Example 34 with ReaderGroupManagerImpl

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl in project pravega by pravega.

the class ScopeTest method testDeleteScopeRecursive.

@Test
public void testDeleteScopeRecursive() throws Exception {
    final String scope = "testDeleteScope";
    final String testFalseScope = "falseScope";
    final String streamName1 = "test1";
    final String streamName2 = "test2";
    final String streamName3 = "test3";
    final String streamName4 = "test4";
    final String kvtName1 = "kvt1";
    final String kvtName2 = "kvt2";
    final String kvtName3 = "kvt3";
    final String groupName1 = "rg1";
    final String groupName2 = "rg2";
    final String groupName3 = "rg3";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    @Cleanup Controller controller = controllerWrapper.getController();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create("tcp://localhost:" + controllerPort)).build();
    @Cleanup ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    // Create scope
    controllerWrapper.getControllerService().createScope(scope, 0L).get();
    assertTrue(controller.checkScopeExists(scope).get());
    // Create streams
    assertTrue(controller.createStream(scope, streamName1, config).get());
    assertTrue(controller.createStream(scope, streamName2, config).get());
    assertTrue(controller.createStream(scope, streamName3, config).get());
    @Cleanup StreamManager streamManager = new StreamManagerImpl(controller, cp);
    @Cleanup KeyValueTableManager keyValueTableManager = new KeyValueTableManagerImpl(clientConfig);
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, clientConfig, connectionFactory);
    // 1. Call deleteScopeRecursive() without creating a scope
    assertTrue(streamManager.deleteScopeRecursive(testFalseScope));
    // Create KVT under the scope
    KeyValueTableConfiguration kvtConfig = KeyValueTableConfiguration.builder().partitionCount(2).primaryKeyLength(4).secondaryKeyLength(4).build();
    assertTrue(keyValueTableManager.createKeyValueTable(scope, kvtName1, kvtConfig));
    assertTrue(keyValueTableManager.createKeyValueTable(scope, kvtName2, kvtConfig));
    // Create RG under the same scope
    assertTrue(readerGroupManager.createReaderGroup(groupName1, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName1)).build()));
    assertTrue(readerGroupManager.createReaderGroup(groupName2, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName2)).build()));
    // Call deleteScopeRecursive to delete the scope recursively
    assertTrue(streamManager.deleteScopeRecursive(scope));
    // Validate that the scope is deleted
    assertFalse(controller.checkScopeExists(scope).get());
    // Validate create operation of Stream/RG/KVT should throw error
    AssertExtensions.assertThrows("Failed to create Reader Group as Scope does not exits", () -> readerGroupManager.createReaderGroup(groupName3, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName2)).build()), e -> e instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("Scope does not exist", () -> controller.createStream(scope, streamName4, config).get(), e -> e instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("Scope does not exist", () -> keyValueTableManager.createKeyValueTable(scope, kvtName3, kvtConfig), e -> e instanceof IllegalArgumentException);
}

Example 35 with ReaderGroupManagerImpl

use of io.pravega.client.admin.impl.ReaderGroupManagerImpl in project pravega by pravega.

the class EndToEndCBRTest method testReaderGroupAutoRetention.

@Test(timeout = 60000)
public void testReaderGroupAutoRetention() throws Exception {
    String scope = "test";
    String streamName = "testReaderGroupAutoRetention";
    String groupName = "testReaderGroupAutoRetention-group";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).retentionPolicy(RetentionPolicy.bySizeBytes(10, Long.MAX_VALUE)).build();
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    controller.createScope(scope).get();
    controller.createStream(scope, streamName, config).get();
    Stream stream = Stream.of(scope, streamName);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    // write events
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    writer.writeEvent("1", "e1").join();
    writer.writeEvent("2", "e2").join();
    // Create a ReaderGroup
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory);
    groupManager.createReaderGroup(groupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().retentionType(ReaderGroupConfig.StreamDataRetention.AUTOMATIC_RELEASE_AT_LAST_CHECKPOINT).stream(stream).build());
    // Create a Reader
    AtomicLong clock = new AtomicLong();
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("reader1", groupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    EventRead<String> read = reader.readNextEvent(60000);
    assertEquals("e1", read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    @Cleanup("shutdown") final InlineExecutor backgroundExecutor = new InlineExecutor();
    ReaderGroup readerGroup = groupManager.getReaderGroup(groupName);
    CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("Checkpoint", backgroundExecutor);
    assertFalse(checkpoint.isDone());
    read = reader.readNextEvent(60000);
    assertTrue(read.isCheckpoint());
    assertEquals("Checkpoint", read.getCheckpointName());
    assertNull(read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(60000);
    assertEquals("e2", read.getEvent());
    Checkpoint cpResult = checkpoint.get(5, TimeUnit.SECONDS);
    assertTrue(checkpoint.isDone());
    assertEquals("Checkpoint", cpResult.getName());
    read = reader.readNextEvent(100);
    assertNull(read.getEvent());
    assertFalse(read.isCheckpoint());
    AssertExtensions.assertEventuallyEquals(true, () -> controller.getSegmentsAtTime(new StreamImpl(scope, streamName), 0L).join().values().stream().anyMatch(off -> off > 0), 30 * 1000L);
    String group2 = groupName + "2";
    groupManager.createReaderGroup(group2, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(NameUtils.getScopedStreamName(scope, streamName)).build());
    EventStreamReader<String> reader2 = clientFactory.createReader("reader2", group2, serializer, ReaderConfig.builder().build());
    EventRead<String> eventRead2 = reader2.readNextEvent(10000);
    assertEquals("e2", eventRead2.getEvent());
}