Examples with StreamManager io.pravega.client.admin.StreamManager used on opensource projects

Example 36 with StreamManager

use of io.pravega.client.admin.StreamManager in project pravega by pravega.

the class EndToEndTruncationTest method testDeleteStreamWhileReading.

/**
 * This test checks the behavior of a reader (or group of readers) that gets a delete event while reading. While the
 * client is reading events (Segment Store) the test deletes the Stream (Controller and metadata). Once the client
 * reads all the events and reaches the end of segment, it contacts the Controller to retrieve subsequent segments
 * (if any). However, the Stream-related metadata to answer this request has been previously deleted.
 */
// @Ignore //TODO: The controller does not currently handle the stream being deleted properly.
// Once it does so the client will need to throw an appropriate exception, and this test should reflect it.
@Test(timeout = 20000)
public void testDeleteStreamWhileReading() {
    final String scope = "truncationTests";
    final String streamName = "testDeleteStreamWhileReading";
    final String readerGroup = "RGTestDeleteStreamWhileReading";
    final int totalEvents = 100;
    final int parallelism = 1;
    StreamConfiguration streamConfiguration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(parallelism)).build();
    @Cleanup StreamManager streamManager = StreamManager.create(PRAVEGA.getControllerURI());
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, streamConfiguration);
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    // Write totalEvents to the Stream.
    writeEvents(clientFactory, streamName, totalEvents);
    // Instantiate readers to consume from Stream.
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(scope, PRAVEGA.getControllerURI());
    groupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(500).stream(Stream.of(scope, streamName)).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(String.valueOf(0), readerGroup, new UTF8StringSerializer(), ReaderConfig.builder().build());
    assertEquals(totalEvents / 2, ReadWriteUtils.readEventsUntil(reader, eventRead -> true, totalEvents / 2, 0));
    reader.close();
    val readerRecreated = clientFactory.createReader(String.valueOf(0), readerGroup, new JavaSerializer<>(), ReaderConfig.builder().build());
    assertTrue(streamManager.sealStream(scope, streamName));
    assertTrue(streamManager.deleteStream(scope, streamName));
    assertThrows(InvalidStreamException.class, () -> clientFactory.createReader(String.valueOf(1), readerGroup, new JavaSerializer<>(), ReaderConfig.builder().build()));
    // At the control plane, we expect a RetriesExhaustedException as readers try to get successor segments from a deleted stream.
    assertThrows(TruncatedDataException.class, () -> ReadWriteUtils.readEvents(readerRecreated, totalEvents / 2, 0));
    assertFalse(streamManager.deleteStream(scope, streamName));
}

Example 37 with StreamManager

use of io.pravega.client.admin.StreamManager in project pravega by pravega.

the class EndToEndTruncationTest method testTruncation.

@Test(timeout = 60000)
public void testTruncation() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 2)).build();
    String streamName = "testTruncation";
    @Cleanup StreamManager streamManager = StreamManager.create(PRAVEGA.getControllerURI());
    String scope = "test";
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, config);
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    writer.writeEvent("0", "truncationTest1").get();
    // scale
    Stream stream = new StreamImpl("test", streamName);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    Boolean result = controller.scaleStream(stream, Lists.newArrayList(0L, 1L), map, executorService()).getFuture().get();
    assertTrue(result);
    writer.writeEvent("0", "truncationTest2").get();
    Map<Long, Long> streamCutPositions = new HashMap<>();
    streamCutPositions.put(computeSegmentId(2, 1), 0L);
    streamCutPositions.put(computeSegmentId(3, 1), 0L);
    streamCutPositions.put(computeSegmentId(4, 1), 0L);
    controller.truncateStream(stream.getScope(), stream.getStreamName(), streamCutPositions).join();
    String group = "testTruncation-group";
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", clientConfig, connectionFactory);
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/" + streamName).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, new JavaSerializer<>(), ReaderConfig.builder().build());
    EventRead<String> event = reader.readNextEvent(10000);
    assertNotNull(event);
    assertEquals("truncationTest2", event.getEvent());
    event = reader.readNextEvent(1000);
    assertNull(event.getEvent());
}

Example 38 with StreamManager

use of io.pravega.client.admin.StreamManager in project pravega by pravega.

the class ScopeTest method testForceDeleteScope.

@Test
public void testForceDeleteScope() throws Exception {
    final String scope = "test";
    final String streamName1 = "test1";
    final String streamName2 = "test2";
    final String streamName3 = "test3";
    final String kvtName1 = "kvt1";
    final String kvtName2 = "kvt2";
    final String groupName1 = "rg1";
    final String groupName2 = "rg2";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    @Cleanup Controller controller = controllerWrapper.getController();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create("tcp://localhost:" + controllerPort)).build();
    @Cleanup ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    controllerWrapper.getControllerService().createScope(scope, 0L).get();
    controller.createStream(scope, streamName1, config).get();
    controller.createStream(scope, streamName2, config).get();
    controller.createStream(scope, streamName3, config).get();
    @Cleanup StreamManager streamManager = new StreamManagerImpl(controller, cp);
    @Cleanup KeyValueTableManager keyValueTableManager = new KeyValueTableManagerImpl(clientConfig);
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, clientConfig, connectionFactory);
    KeyValueTableConfiguration kvtConfig = KeyValueTableConfiguration.builder().partitionCount(2).primaryKeyLength(4).secondaryKeyLength(4).build();
    keyValueTableManager.createKeyValueTable(scope, kvtName1, kvtConfig);
    keyValueTableManager.createKeyValueTable(scope, kvtName2, kvtConfig);
    readerGroupManager.createReaderGroup(groupName1, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName1)).build());
    readerGroupManager.createReaderGroup(groupName2, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName2)).build());
    assertTrue(streamManager.deleteScope(scope, true));
}

Example 39 with StreamManager

use of io.pravega.client.admin.StreamManager in project pravega by pravega.

the class ScopeTest method testListStreamForTag.

@Test(timeout = 30000)
public void testListStreamForTag() {
    final String scope = "sc";
    final String stream1 = "s1";
    final String stream2 = "s2";
    final String stream3 = "s3";
    final Set<String> tagSet1 = Set.of("t1", "t2", "t3");
    final Set<String> tagSet2 = Set.of("t2", "t3", "t4");
    final Set<String> tagSet3 = Set.of("t3", "t4", "t5");
    StreamConfiguration cfg = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byDataRate(10, 2, 4)).build();
    @Cleanup StreamManager manager = StreamManager.create(URI.create("tcp://localhost:" + this.controllerPort));
    manager.createScope(scope);
    // fetch tags of a non-existent stream.
    AssertExtensions.assertThrows("Non-existent Stream", () -> manager.getStreamTags(scope, stream1), t -> t instanceof StatusRuntimeException && (((StatusRuntimeException) t).getStatus().getCode().equals(Status.Code.NOT_FOUND)));
    manager.createStream(scope, stream1, cfg.toBuilder().tags(tagSet1).build());
    manager.createStream(scope, stream2, cfg.toBuilder().tags(tagSet2).build());
    manager.createStream(scope, stream3, cfg.toBuilder().tags(tagSet3).build());
    assertEquals(tagSet1, manager.getStreamTags(scope, stream1));
    assertEquals(tagSet2, manager.getStreamTags(scope, stream2));
    assertEquals(tagSet3, manager.getStreamTags(scope, stream3));
    assertEquals(singletonList(Stream.of(scope, stream1)), newArrayList(manager.listStreams(scope, "t1")));
    List<Stream> listedStreams = newArrayList(manager.listStreams(scope, "t3"));
    List<Stream> expectedStreams = Arrays.asList(Stream.of(scope, stream3), Stream.of(scope, stream2), Stream.of(scope, stream1));
    assertTrue((listedStreams.size() == expectedStreams.size()) && listedStreams.containsAll(expectedStreams) && expectedStreams.containsAll(listedStreams));
    // update a stream tag and verify if it is reflected.
    manager.updateStream(scope, stream3, cfg.toBuilder().clearTags().tag("t4").tag("t5").build());
    listedStreams = newArrayList(manager.listStreams(scope, "t3"));
    expectedStreams = Arrays.asList(Stream.of(scope, stream2), Stream.of(scope, stream1));
    assertTrue((listedStreams.size() == expectedStreams.size()) && listedStreams.containsAll(expectedStreams) && expectedStreams.containsAll(listedStreams));
    // seal and delete stream
    manager.sealStream(scope, stream2);
    manager.deleteStream(scope, stream2);
    // check if list streams is updated.
    assertEquals(singletonList(Stream.of(scope, stream1)), newArrayList(manager.listStreams(scope, "t3")));
    manager.sealStream(scope, stream1);
    manager.deleteStream(scope, stream1);
    assertEquals(emptyList(), newArrayList(manager.listStreams(scope, "t3")));
    manager.sealStream(scope, stream3);
    manager.deleteStream(scope, stream3);
    assertEquals(emptyList(), newArrayList(manager.listStreams(scope, "t4")));
}

Example 40 with StreamManager

use of io.pravega.client.admin.StreamManager in project pravega by pravega.

the class ScopeTest method testDeleteScopeRecursive.

@Test
public void testDeleteScopeRecursive() throws Exception {
    final String scope = "testDeleteScope";
    final String testFalseScope = "falseScope";
    final String streamName1 = "test1";
    final String streamName2 = "test2";
    final String streamName3 = "test3";
    final String streamName4 = "test4";
    final String kvtName1 = "kvt1";
    final String kvtName2 = "kvt2";
    final String kvtName3 = "kvt3";
    final String groupName1 = "rg1";
    final String groupName2 = "rg2";
    final String groupName3 = "rg3";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    @Cleanup Controller controller = controllerWrapper.getController();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create("tcp://localhost:" + controllerPort)).build();
    @Cleanup ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    // Create scope
    controllerWrapper.getControllerService().createScope(scope, 0L).get();
    assertTrue(controller.checkScopeExists(scope).get());
    // Create streams
    assertTrue(controller.createStream(scope, streamName1, config).get());
    assertTrue(controller.createStream(scope, streamName2, config).get());
    assertTrue(controller.createStream(scope, streamName3, config).get());
    @Cleanup StreamManager streamManager = new StreamManagerImpl(controller, cp);
    @Cleanup KeyValueTableManager keyValueTableManager = new KeyValueTableManagerImpl(clientConfig);
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, clientConfig, connectionFactory);
    // 1. Call deleteScopeRecursive() without creating a scope
    assertTrue(streamManager.deleteScopeRecursive(testFalseScope));
    // Create KVT under the scope
    KeyValueTableConfiguration kvtConfig = KeyValueTableConfiguration.builder().partitionCount(2).primaryKeyLength(4).secondaryKeyLength(4).build();
    assertTrue(keyValueTableManager.createKeyValueTable(scope, kvtName1, kvtConfig));
    assertTrue(keyValueTableManager.createKeyValueTable(scope, kvtName2, kvtConfig));
    // Create RG under the same scope
    assertTrue(readerGroupManager.createReaderGroup(groupName1, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName1)).build()));
    assertTrue(readerGroupManager.createReaderGroup(groupName2, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName2)).build()));
    // Call deleteScopeRecursive to delete the scope recursively
    assertTrue(streamManager.deleteScopeRecursive(scope));
    // Validate that the scope is deleted
    assertFalse(controller.checkScopeExists(scope).get());
    // Validate create operation of Stream/RG/KVT should throw error
    AssertExtensions.assertThrows("Failed to create Reader Group as Scope does not exits", () -> readerGroupManager.createReaderGroup(groupName3, ReaderGroupConfig.builder().stream(getScopedStreamName(scope, streamName2)).build()), e -> e instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("Scope does not exist", () -> controller.createStream(scope, streamName4, config).get(), e -> e instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("Scope does not exist", () -> keyValueTableManager.createKeyValueTable(scope, kvtName3, kvtConfig), e -> e instanceof IllegalArgumentException);
}