Examples with EventStreamClientFactory io.pravega.client.EventStreamClientFactory used on opensource projects

Example 31 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method createEventProcessor.

private <T extends ControllerEvent> void createEventProcessor(final String readerGroupName, final String streamName, final EventStreamReader<T> reader, final EventStreamWriter<T> writer, Supplier<EventProcessor<T>> factory) throws CheckpointStoreException {
    EventStreamClientFactory clientFactory = Mockito.mock(EventStreamClientFactory.class);
    Mockito.when(clientFactory.<T>createReader(anyString(), anyString(), any(), any())).thenReturn(reader);
    Mockito.when(clientFactory.<T>createEventWriter(anyString(), any(), any())).thenReturn(writer);
    ReaderGroup readerGroup = Mockito.mock(ReaderGroup.class);
    Mockito.when(readerGroup.getGroupName()).thenReturn(readerGroupName);
    ReaderGroupManager readerGroupManager = Mockito.mock(ReaderGroupManager.class);
    EventProcessorSystemImpl system = new EventProcessorSystemImpl("system", "host", SCOPE, clientFactory, readerGroupManager);
    EventProcessorGroupConfig eventProcessorConfig = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(readerGroupName).streamName(streamName).checkpointConfig(CheckpointConfig.periodic(1, 1)).build();
    EventProcessorConfig<T> config = EventProcessorConfig.<T>builder().config(eventProcessorConfig).decider(ExceptionHandler.DEFAULT_EXCEPTION_HANDLER).serializer(new EventSerializer<>()).supplier(factory).build();
    system.createEventProcessorGroup(config, CheckpointStoreFactory.createInMemoryStore(), executor);
}

Example 32 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class StreamRecreationTest method testStreamRecreation.

@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testStreamRecreation() throws Exception {
    final String myScope = "myScope";
    final String myStream = "myStream";
    final String myReaderGroup = "myReaderGroup";
    final int numIterations = 6;
    // Create the scope and the stream.
    @Cleanup StreamManager streamManager = StreamManager.create(controllerURI);
    streamManager.createScope(myScope);
    @Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(myScope, controllerURI);
    final ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder().stream(Stream.of(myScope, myStream)).build();
    for (int i = 0; i < numIterations; i++) {
        log.info("Stream re-creation iteration {}.", i);
        final String eventContent = "myEvent" + String.valueOf(i);
        StreamConfiguration streamConfiguration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(i + 1)).build();
        EventWriterConfig eventWriterConfig = EventWriterConfig.builder().build();
        streamManager.createStream(myScope, myStream, streamConfiguration);
        // Write a single event.
        @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(myScope, ClientConfig.builder().controllerURI(controllerURI).build());
        @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(myStream, new JavaSerializer<>(), eventWriterConfig);
        TransactionalEventStreamWriter<String> txnWriter = clientFactory.createTransactionalEventWriter(myStream, new JavaSerializer<>(), eventWriterConfig);
        // Write events regularly and with transactions.
        if (i % 2 == 0) {
            writer.writeEvent(eventContent).join();
        } else {
            Transaction<String> myTransaction = txnWriter.beginTxn();
            myTransaction.writeEvent(eventContent);
            myTransaction.commit();
            while (myTransaction.checkStatus() != Transaction.Status.COMMITTED) {
                Exceptions.handleInterrupted(() -> Thread.sleep(100));
            }
        }
        writer.close();
        // Read the event.
        readerGroupManager.createReaderGroup(myReaderGroup, readerGroupConfig);
        readerGroupManager.getReaderGroup(myReaderGroup).resetReaderGroup(readerGroupConfig);
        @Cleanup EventStreamReader<String> reader = clientFactory.createReader("myReader", myReaderGroup, new JavaSerializer<>(), ReaderConfig.builder().build());
        String readResult;
        do {
            readResult = reader.readNextEvent(1000).getEvent();
        } while (readResult == null);
        assertEquals("Wrong event read in re-created stream", eventContent, readResult);
        // Delete the stream.
        StreamInfo streamInfo = streamManager.getStreamInfo(myScope, myStream);
        assertFalse(streamInfo.isSealed());
        assertTrue("Unable to seal re-created stream.", streamManager.sealStream(myScope, myStream));
        streamInfo = streamManager.getStreamInfo(myScope, myStream);
        assertTrue(streamInfo.isSealed());
        assertTrue("Unable to delete re-created stream.", streamManager.deleteStream(myScope, myStream));
    }
}

Example 33 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class WatermarkingTest method recreateStreamWatermarkTest.

@Test(timeout = 120000)
public void recreateStreamWatermarkTest() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    // then delete stream and move to next iteration and verify that watermarks are generated.
    for (int i = 0; i < 2; i++) {
        String scope = "scope";
        String stream = "recreateStreamWatermarkTest";
        streamManager.createScope(scope);
        streamManager.createStream(scope, stream, config);
        // create writer
        @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
        JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
        @Cleanup EventStreamWriter<Long> writer = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
        AtomicBoolean stopFlag = new AtomicBoolean(false);
        // write events
        CompletableFuture<Void> writerFuture = writeEvents(writer, stopFlag);
        @Cleanup SynchronizerClientFactory syncClientFactory = SynchronizerClientFactory.withScope(scope, clientConfig);
        String markStream = NameUtils.getMarkStreamForStream(stream);
        @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
        LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
        fetchWatermarks(watermarkReader, watermarks, stopFlag);
        AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
        // stop run and seal and delete stream
        stopFlag.set(true);
        writerFuture.join();
        streamManager.sealStream(scope, stream);
        streamManager.deleteStream(scope, stream);
    }
}

Example 34 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class BoundedStreamReaderTest method testBoundedStreamTest.

@Test(timeout = 60000)
public void testBoundedStreamTest() throws Exception {
    createScope(SCOPE);
    createStream(STREAM1);
    createStream(STREAM2);
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(controllerUri).build());
    @Cleanup EventStreamWriter<String> writer1 = clientFactory.createEventWriter(STREAM1, serializer, EventWriterConfig.builder().build());
    // Prep the stream with data.
    // 1.Write events with event size of 30
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(1)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(2)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(3)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(4)).get();
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerUri);
    groupManager.createReaderGroup("group", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, STREAM1), // startStreamCut points to the current HEAD of stream
    StreamCut.UNBOUNDED, // endStreamCut points to the offset after two events.(i.e 2 * 30(event size) = 60)
    getStreamCut(STREAM1, 60L, 0)).stream(Stream.of(SCOPE, STREAM2)).build());
    // Create a reader
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "group", serializer, ReaderConfig.builder().build());
    // 2. Verify if endStreamCut configuration is enforced.
    readAndVerify(reader, 1, 2);
    // The following read should not return events 3, 4 due to the endStreamCut configuration.
    Assert.assertNull("Null is expected", reader.readNextEvent(2000).getEvent());
    // 3. Write events to the STREAM2.
    @Cleanup EventStreamWriter<String> writer2 = clientFactory.createEventWriter(STREAM2, serializer, EventWriterConfig.builder().build());
    writer2.writeEvent(keyGenerator.get(), getEventData.apply(5)).get();
    writer2.writeEvent(keyGenerator.get(), getEventData.apply(6)).get();
    // 4. Verify that events can be read from STREAM2. (Events from STREAM1 are not read since endStreamCut is reached).
    readAndVerify(reader, 5, 6);
    Assert.assertNull("Null is expected", reader.readNextEvent(2000).getEvent());
}

Example 35 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class StreamMetricsTest method testTransactionMetrics.

@Test(timeout = 30000)
public void testTransactionMetrics() throws Exception {
    String txScopeName = "scopeTx";
    String txStreamName = "streamTx";
    controllerWrapper.getControllerService().createScope(txScopeName, 0L).get();
    if (!controller.createStream(txScopeName, txStreamName, config).get()) {
        log.error("Stream {} for tx testing already existed, exiting", txScopeName + "/" + txStreamName);
        return;
    }
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(txScopeName, ClientConfig.builder().controllerURI(URI.create("tcp://localhost:" + controllerPort)).build());
    @Cleanup TransactionalEventStreamWriter<String> writer = clientFactory.createTransactionalEventWriter(Stream.of(txScopeName, txStreamName).getStreamName(), new JavaSerializer<>(), EventWriterConfig.builder().build());
    Transaction<String> transaction = writer.beginTxn();
    assertEquals(1, (long) MetricRegistryUtils.getCounter(MetricsNames.CREATE_TRANSACTION, streamTags(txScopeName, txStreamName)).count());
    transaction.writeEvent("Test");
    transaction.flush();
    transaction.commit();
    AssertExtensions.assertEventuallyEquals(true, () -> transaction.checkStatus().equals(Transaction.Status.COMMITTED), 10000);
    AssertExtensions.assertEventuallyEquals(true, () -> MetricRegistryUtils.getCounter(MetricsNames.COMMIT_TRANSACTION, streamTags(txScopeName, txStreamName)) != null, 10000);
    assertEquals(1, (long) MetricRegistryUtils.getCounter(MetricsNames.COMMIT_TRANSACTION, streamTags(txScopeName, txStreamName)).count());
    Transaction<String> transaction2 = writer.beginTxn();
    transaction2.writeEvent("Test");
    transaction2.abort();
    AssertExtensions.assertEventuallyEquals(true, () -> transaction2.checkStatus().equals(Transaction.Status.ABORTED), 10000);
    AssertExtensions.assertEventuallyEquals(true, () -> MetricRegistryUtils.getCounter(MetricsNames.ABORT_TRANSACTION, streamTags(txScopeName, txStreamName)) != null, 10000);
    assertEquals(1, (long) MetricRegistryUtils.getCounter(MetricsNames.ABORT_TRANSACTION, streamTags(txScopeName, txStreamName)).count());
}