Search in sources :

Example 1 with IntegerSerializer

use of io.pravega.test.integration.utils.IntegerSerializer in project pravega by pravega.

the class MultiReadersEndToEndTest method readAllEvents.

private Collection<Integer> readAllEvents(final int numParallelReaders, ClientFactory clientFactory, final String readerGroupName, final int numSegments) {
    ConcurrentLinkedQueue<Integer> read = new ConcurrentLinkedQueue<>();
    final ExecutorService executorService = Executors.newFixedThreadPool(numParallelReaders, new ThreadFactoryBuilder().setNameFormat("testreader-pool-%d").build());
    List<Future<?>> futures = new ArrayList<>();
    for (int i = 0; i < numParallelReaders; i++) {
        futures.add(executorService.submit(() -> {
            final String readerId = UUID.randomUUID().toString();
            @Cleanup final EventStreamReader<Integer> reader = clientFactory.createReader(readerId, readerGroupName, new IntegerSerializer(), ReaderConfig.builder().build());
            int emptyCount = 0;
            while (emptyCount <= numSegments) {
                try {
                    final Integer integerEventRead = reader.readNextEvent(100).getEvent();
                    if (integerEventRead != null) {
                        read.add(integerEventRead);
                        emptyCount = 0;
                    } else {
                        emptyCount++;
                    }
                } catch (ReinitializationRequiredException e) {
                    throw new RuntimeException(e);
                }
            }
        }));
    }
    // Wait until all readers are done.
    futures.forEach(f -> Futures.getAndHandleExceptions(f, RuntimeException::new));
    executorService.shutdownNow();
    return read;
}
Also used : ArrayList(java.util.ArrayList) IntegerSerializer(io.pravega.test.integration.utils.IntegerSerializer) EventStreamReader(io.pravega.client.stream.EventStreamReader) ReinitializationRequiredException(io.pravega.client.stream.ReinitializationRequiredException) ExecutorService(java.util.concurrent.ExecutorService) ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) Future(java.util.concurrent.Future) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue)

Example 2 with IntegerSerializer

use of io.pravega.test.integration.utils.IntegerSerializer in project pravega by pravega.

the class MultiReadersEndToEndTest method runTest.

private void runTest(final Set<String> streamNames, final int numParallelReaders, final int numSegments) throws Exception {
    @Cleanup StreamManager streamManager = StreamManager.create(ClientConfig.builder().controllerURI(SETUP_UTILS.getControllerUri()).build());
    streamManager.createScope(SETUP_UTILS.getScope());
    streamNames.stream().forEach(stream -> {
        streamManager.createStream(SETUP_UTILS.getScope(), stream, StreamConfiguration.builder().scope(SETUP_UTILS.getScope()).streamName(stream).scalingPolicy(ScalingPolicy.fixed(numSegments)).build());
        log.info("Created stream: {}", stream);
    });
    @Cleanup ClientFactory clientFactory = ClientFactory.withScope(SETUP_UTILS.getScope(), ClientConfig.builder().controllerURI(SETUP_UTILS.getControllerUri()).build());
    streamNames.stream().forEach(stream -> {
        EventStreamWriter<Integer> eventWriter = clientFactory.createEventWriter(stream, new IntegerSerializer(), EventWriterConfig.builder().build());
        for (Integer i = 0; i < NUM_TEST_EVENTS; i++) {
            eventWriter.writeEvent(String.valueOf(i), i);
        }
        eventWriter.flush();
        log.info("Wrote {} events", NUM_TEST_EVENTS);
    });
    final String readerGroupName = "testreadergroup" + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
    @Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(SETUP_UTILS.getScope(), ClientConfig.builder().controllerURI(SETUP_UTILS.getControllerUri()).build());
    ReaderGroupConfig.ReaderGroupConfigBuilder builder = ReaderGroupConfig.builder();
    streamNames.forEach(s -> builder.stream(Stream.of(SETUP_UTILS.getScope(), s)));
    readerGroupManager.createReaderGroup(readerGroupName, builder.build());
    Collection<Integer> read = readAllEvents(numParallelReaders, clientFactory, readerGroupName, numSegments);
    Assert.assertEquals(NUM_TEST_EVENTS * streamNames.size(), read.size());
    // Check unique events.
    Assert.assertEquals(NUM_TEST_EVENTS, new TreeSet<>(read).size());
    readerGroupManager.deleteReaderGroup(readerGroupName);
}
Also used : ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) StreamManager(io.pravega.client.admin.StreamManager) MockStreamManager(io.pravega.client.stream.mock.MockStreamManager) TreeSet(java.util.TreeSet) ClientFactory(io.pravega.client.ClientFactory) MockClientFactory(io.pravega.client.stream.mock.MockClientFactory) Cleanup(lombok.Cleanup) IntegerSerializer(io.pravega.test.integration.utils.IntegerSerializer)

Example 3 with IntegerSerializer

use of io.pravega.test.integration.utils.IntegerSerializer in project pravega by pravega.

the class MultiReadersEndToEndTest method runTestUsingMock.

private void runTestUsingMock(final Set<String> streamNames, final int numParallelReaders, final int numSegments) throws Exception {
    int servicePort = TestUtils.getAvailableListenPort();
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, servicePort, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager("scope", "localhost", servicePort);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    streamManager.createScope("scope");
    streamNames.stream().forEach(stream -> {
        streamManager.createStream("scope", stream, StreamConfiguration.builder().scope("scope").streamName(stream).scalingPolicy(ScalingPolicy.fixed(numSegments)).build());
        EventStreamWriter<Integer> eventWriter = clientFactory.createEventWriter(stream, new IntegerSerializer(), EventWriterConfig.builder().build());
        for (Integer i = 0; i < NUM_TEST_EVENTS; i++) {
            eventWriter.writeEvent(String.valueOf(i), i);
        }
        eventWriter.flush();
        log.info("Wrote {} events", NUM_TEST_EVENTS);
    });
    final String readerGroupName = "testReaderGroup";
    ReaderGroupConfig.ReaderGroupConfigBuilder builder = ReaderGroupConfig.builder();
    streamNames.forEach(s -> builder.stream(Stream.of("scope", s)));
    streamManager.createReaderGroup(readerGroupName, builder.build());
    Collection<Integer> read = readAllEvents(numParallelReaders, clientFactory, readerGroupName, numSegments);
    Assert.assertEquals(NUM_TEST_EVENTS * streamNames.size(), read.size());
    // Check unique events.
    Assert.assertEquals(NUM_TEST_EVENTS, new TreeSet<>(read).size());
    streamManager.deleteReaderGroup(readerGroupName);
}
Also used : ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) Cleanup(lombok.Cleanup) PravegaConnectionListener(io.pravega.segmentstore.server.host.handler.PravegaConnectionListener) IntegerSerializer(io.pravega.test.integration.utils.IntegerSerializer) MockClientFactory(io.pravega.client.stream.mock.MockClientFactory) ServiceBuilder(io.pravega.segmentstore.server.store.ServiceBuilder) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) TreeSet(java.util.TreeSet) MockStreamManager(io.pravega.client.stream.mock.MockStreamManager)

Aggregations

IntegerSerializer (io.pravega.test.integration.utils.IntegerSerializer)3 ReaderGroupConfig (io.pravega.client.stream.ReaderGroupConfig)2 MockClientFactory (io.pravega.client.stream.mock.MockClientFactory)2 MockStreamManager (io.pravega.client.stream.mock.MockStreamManager)2 TreeSet (java.util.TreeSet)2 Cleanup (lombok.Cleanup)2 ThreadFactoryBuilder (com.google.common.util.concurrent.ThreadFactoryBuilder)1 ClientFactory (io.pravega.client.ClientFactory)1 ReaderGroupManager (io.pravega.client.admin.ReaderGroupManager)1 StreamManager (io.pravega.client.admin.StreamManager)1 EventStreamReader (io.pravega.client.stream.EventStreamReader)1 ReinitializationRequiredException (io.pravega.client.stream.ReinitializationRequiredException)1 StreamSegmentStore (io.pravega.segmentstore.contracts.StreamSegmentStore)1 PravegaConnectionListener (io.pravega.segmentstore.server.host.handler.PravegaConnectionListener)1 ServiceBuilder (io.pravega.segmentstore.server.store.ServiceBuilder)1 ArrayList (java.util.ArrayList)1 ConcurrentLinkedQueue (java.util.concurrent.ConcurrentLinkedQueue)1 ExecutorService (java.util.concurrent.ExecutorService)1 Future (java.util.concurrent.Future)1