Examples with JavaSerializer io.pravega.client.stream.impl.JavaSerializer used on opensource projects

Example 21 with JavaSerializer

use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.

the class StateSynchronizerTest method testStateTracker.

@Test(timeout = 20000)
public void testStateTracker() {
    String endpoint = "localhost";
    String stateName = "testStateTracker";
    int port = TestUtils.getAvailableListenPort();
    StreamSegmentStore store = SERVICE_BUILDER.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, mock(TableStore.class), SERVICE_BUILDER.getLowPriorityExecutor());
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager("scope", endpoint, port);
    streamManager.createScope("scope");
    streamManager.createStream("scope", stateName, null);
    JavaSerializer<TestUpdate> serializer = new JavaSerializer<TestUpdate>();
    @Cleanup val a = streamManager.getClientFactory().createStateSynchronizer(stateName, serializer, serializer, SynchronizerConfig.builder().build());
    @Cleanup val b = streamManager.getClientFactory().createStateSynchronizer(stateName, serializer, serializer, SynchronizerConfig.builder().build());
    a.initialize(new TestUpdate("init"));
    b.fetchUpdates();
    assertEquals("init", b.getState().value);
    assertEquals(1, update(a, "already up to date 1"));
    assertEquals(2, update(b, "fail Initially 2"));
    assertEquals("already up to date 1", a.getState().value);
    assertEquals("fail Initially 2", b.getState().value);
    assertEquals(1, update(b, "already up to date 3"));
    assertEquals("already up to date 1", a.getState().value);
    a.fetchUpdates();
    assertEquals("already up to date 3", a.getState().value);
    assertEquals(1, update(a, "already up to date 4"));
    assertEquals("already up to date 4", a.getState().value);
    assertEquals("already up to date 3", b.getState().value);
    assertEquals(2, update(b, "fail Initially 5"));
    assertEquals("already up to date 4", a.getState().value);
    a.fetchUpdates();
    assertEquals("fail Initially 5", a.getState().value);
    a.fetchUpdates();
    b.fetchUpdates();
    assertEquals("fail Initially 5", a.getState().value);
    assertEquals("fail Initially 5", b.getState().value);
}

Example 22 with JavaSerializer

use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.

the class RevisionedStreamClientTest method testRetryOnTimeout.

@Test
public void testRetryOnTimeout() throws ConnectionFailedException {
    String scope = "scope";
    String stream = "stream";
    Segment segment = new Segment(scope, stream, 0L);
    // Setup Environment
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    // Setup Mocks
    JavaSerializer<String> serializer = new JavaSerializer<>();
    @Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    @Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    // Setup client connection.
    ClientConnection c = mock(ClientConnection.class);
    connectionFactory.provideConnection(endpoint, c);
    // Create Scope and Stream.
    createScopeAndStream(scope, stream, controller);
    // Create mock ClientFactory.
    SegmentInputStreamFactory segInputFactory = new SegmentInputStreamFactoryImpl(controller, connectionFactory);
    SegmentOutputStreamFactory segOutputFactory = mock(SegmentOutputStreamFactory.class);
    ConditionalOutputStreamFactory condOutputFactory = new ConditionalOutputStreamFactoryImpl(controller, connectionFactory);
    SegmentMetadataClientFactory segMetaFactory = mock(SegmentMetadataClientFactory.class);
    SegmentMetadataClient segMetaClient = mock(SegmentMetadataClient.class);
    when(segMetaFactory.createSegmentMetadataClient(eq(segment), any(DelegationTokenProvider.class))).thenReturn(segMetaClient);
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, segInputFactory, segOutputFactory, condOutputFactory, segMetaFactory);
    RevisionedStreamClientImpl<String> client = spy((RevisionedStreamClientImpl<String>) clientFactory.createRevisionedStreamClient(stream, serializer, SynchronizerConfig.builder().build()));
    // Override the readTimeout value for RevisionedClient to 1 second.
    doReturn(1000L).when(client).getReadTimeout();
    // Setup the SegmentMetadataClient mock.
    doReturn(CompletableFuture.completedFuture(new SegmentInfo(segment, 0L, 30L, false, 1L))).when(segMetaClient).getSegmentInfo();
    // Get the iterator from Revisioned Stream Client.
    Iterator<Entry<Revision, String>> iterator = client.readFrom(new RevisionImpl(segment, 15, 1));
    // since we are trying to read @ offset 15 and the writeOffset is 30L a true is returned for hasNext().
    assertTrue(iterator.hasNext());
    // Setup mock to validate a retry.
    doNothing().doAnswer(i -> {
        WireCommands.ReadSegment request = i.getArgument(0);
        ReplyProcessor rp = connectionFactory.getProcessor(endpoint);
        WireCommands.Event event = new WireCommands.Event(Unpooled.wrappedBuffer(serializer.serialize("A")));
        ByteArrayOutputStream bout = new ByteArrayOutputStream();
        event.writeFields(new DataOutputStream(bout));
        ByteBuf eventData = Unpooled.wrappedBuffer(bout.toByteArray());
        // Invoke Reply processor to simulate a successful read.
        rp.process(new WireCommands.SegmentRead(request.getSegment(), 15L, true, true, eventData, request.getRequestId()));
        return null;
    }).when(c).send(any(WireCommands.ReadSegment.class));
    Entry<Revision, String> r = iterator.next();
    assertEquals("A", r.getValue());
    // Verify retries have been performed.
    verify(c, times(3)).send(any(WireCommands.ReadSegment.class));
}

Example 23 with JavaSerializer

use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.

the class EndToEndReaderGroupTest method testDeleteReaderGroup.

@Test(timeout = 30000)
public void testDeleteReaderGroup() throws Exception {
    StreamConfiguration config = getStreamConfig();
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    String streamName = "testDeleteReaderGroup";
    controller.createScope("test").get();
    controller.createStream("test", streamName, config).get();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl("test", controller, connectionFactory);
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
    // Create a ReaderGroup
    String groupName = "testDeleteReaderGroup-group";
    groupManager.createReaderGroup(groupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/" + streamName).build());
    // Create a Reader
    EventStreamReader<String> reader = clientFactory.createReader("reader1", groupName, serializer, ReaderConfig.builder().build());
    // Write events into the stream.
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    writer.writeEvent("0", "data1").get();
    EventRead<String> eventRead = reader.readNextEvent(10000);
    assertEquals("data1", eventRead.getEvent());
    // Close the reader, this internally invokes ReaderGroup#readerOffline
    reader.close();
    // delete the readerGroup.
    groupManager.deleteReaderGroup(groupName);
    // create a new readerGroup with the same name.
    groupManager.createReaderGroup(groupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("test/" + streamName).build());
    reader = clientFactory.createReader("reader1", groupName, new JavaSerializer<>(), ReaderConfig.builder().build());
    eventRead = reader.readNextEvent(10000);
    assertEquals("data1", eventRead.getEvent());
}

Example 24 with JavaSerializer

use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.

the class EndToEndTruncationTest method testTruncationOffsets.

@Test(timeout = 7000)
public void testTruncationOffsets() throws InterruptedException, ExecutionException, TimeoutException, TruncatedDataException, ReinitializationRequiredException {
    String scope = "scope";
    String streamName = "testTruncationOffsets";
    String testString = "Hello world\n";
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, StreamConfiguration.builder().build()).join();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    Serializer<String> serializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    Future<Void> ack = producer.writeEvent(testString);
    ack.get(5, TimeUnit.SECONDS);
    SegmentMetadataClientFactory metadataClientFactory = new SegmentMetadataClientFactoryImpl(controller, clientFactory.getConnectionPool());
    @Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
    Segment segment = new Segment(scope, streamName, 0);
    @Cleanup SegmentMetadataClient metadataClient = metadataClientFactory.createSegmentMetadataClient(segment, DelegationTokenProviderFactory.createWithEmptyToken());
    assertEquals(0, metadataClient.getSegmentInfo().join().getStartingOffset());
    long writeOffset = metadataClient.getSegmentInfo().join().getWriteOffset();
    assertEquals(writeOffset, metadataClient.fetchCurrentSegmentLength().join().longValue());
    assertTrue(metadataClient.getSegmentInfo().join().getWriteOffset() > testString.length());
    metadataClient.truncateSegment(writeOffset).join();
    assertEquals(writeOffset, metadataClient.getSegmentInfo().join().getStartingOffset());
    assertEquals(writeOffset, metadataClient.getSegmentInfo().join().getWriteOffset());
    assertEquals(writeOffset, metadataClient.fetchCurrentSegmentLength().join().longValue());
    ack = producer.writeEvent(testString);
    ack.get(5, TimeUnit.SECONDS);
    String group = "testTruncationOffsets-group";
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(new StreamImpl(scope, streamName)).build();
    readerGroupManager.createReaderGroup(group, groupConfig);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("reader", group, serializer, ReaderConfig.builder().build());
    AssertExtensions.assertThrows(TruncatedDataException.class, () -> reader.readNextEvent(2000));
    EventRead<String> event = reader.readNextEvent(2000);
    assertEquals(testString, event.getEvent());
    event = reader.readNextEvent(100);
    assertEquals(null, event.getEvent());
}

Example 25 with JavaSerializer

use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.

the class EndToEndTruncationTest method testDeleteStreamWhileReading.

/**
 * This test checks the behavior of a reader (or group of readers) that gets a delete event while reading. While the
 * client is reading events (Segment Store) the test deletes the Stream (Controller and metadata). Once the client
 * reads all the events and reaches the end of segment, it contacts the Controller to retrieve subsequent segments
 * (if any). However, the Stream-related metadata to answer this request has been previously deleted.
 */
// @Ignore //TODO: The controller does not currently handle the stream being deleted properly.
// Once it does so the client will need to throw an appropriate exception, and this test should reflect it.
@Test(timeout = 20000)
public void testDeleteStreamWhileReading() {
    final String scope = "truncationTests";
    final String streamName = "testDeleteStreamWhileReading";
    final String readerGroup = "RGTestDeleteStreamWhileReading";
    final int totalEvents = 100;
    final int parallelism = 1;
    StreamConfiguration streamConfiguration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(parallelism)).build();
    @Cleanup StreamManager streamManager = StreamManager.create(PRAVEGA.getControllerURI());
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, streamConfiguration);
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    // Write totalEvents to the Stream.
    writeEvents(clientFactory, streamName, totalEvents);
    // Instantiate readers to consume from Stream.
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(scope, PRAVEGA.getControllerURI());
    groupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(500).stream(Stream.of(scope, streamName)).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(String.valueOf(0), readerGroup, new UTF8StringSerializer(), ReaderConfig.builder().build());
    assertEquals(totalEvents / 2, ReadWriteUtils.readEventsUntil(reader, eventRead -> true, totalEvents / 2, 0));
    reader.close();
    val readerRecreated = clientFactory.createReader(String.valueOf(0), readerGroup, new JavaSerializer<>(), ReaderConfig.builder().build());
    assertTrue(streamManager.sealStream(scope, streamName));
    assertTrue(streamManager.deleteStream(scope, streamName));
    assertThrows(InvalidStreamException.class, () -> clientFactory.createReader(String.valueOf(1), readerGroup, new JavaSerializer<>(), ReaderConfig.builder().build()));
    // At the control plane, we expect a RetriesExhaustedException as readers try to get successor segments from a deleted stream.
    assertThrows(TruncatedDataException.class, () -> ReadWriteUtils.readEvents(readerRecreated, totalEvents / 2, 0));
    assertFalse(streamManager.deleteStream(scope, streamName));
}