Examples with JavaSerializer io.pravega.client.stream.impl.JavaSerializer used on opensource projects

Example 31 with JavaSerializer

use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.

the class AutoCheckpointTest method testCheckpointsOccur.

@Test(timeout = 30000)
public void testCheckpointsOccur() throws ReinitializationRequiredException, DurableDataLogException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerName = "reader";
    String readerGroup = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world: ";
    String scope = "Scope1";
    @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, mock(TableStore.class), serviceBuilder.getLowPriorityExecutor());
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    @Cleanup MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(10000).stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, null);
    streamManager.createReaderGroup(readerGroup, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    populateEvents(streamName, testString, clientFactory, serializer);
    AtomicLong fakeClock = new AtomicLong(0);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
    int numRead = 0;
    int checkpointCount = 0;
    while (numRead < 100) {
        fakeClock.addAndGet(NANOS_PER_SECOND);
        EventRead<String> event = reader.readNextEvent(1000);
        if (event.isCheckpoint()) {
            checkpointCount++;
        } else {
            String message = event.getEvent();
            assertEquals(testString + numRead, message);
            numRead++;
        }
    }
    assertTrue("Count was " + checkpointCount, checkpointCount > 5);
    assertTrue("Count was " + checkpointCount, checkpointCount < 20);
}

Example 32 with JavaSerializer

use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.

the class EndToEndAutoScaleUpTest method main.

public static void main(String[] args) throws Exception {
    try {
        @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
        int port = Config.SERVICE_PORT;
        @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port, false);
        Controller controller = controllerWrapper.getController();
        ClientFactoryImpl internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, new SocketConnectionFactoryImpl(ClientConfig.builder().build()));
        @Cleanup("shutdownNow") val executor = ExecutorServiceHelpers.newScheduledThreadPool(1, "test");
        @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
        serviceBuilder.initialize();
        StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
        TableStore tableStore = serviceBuilder.createTableStoreService();
        @Cleanup AutoScaleMonitor autoScaleMonitor = new AutoScaleMonitor(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).build());
        @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, false, "localhost", 12345, store, tableStore, autoScaleMonitor.getStatsRecorder(), autoScaleMonitor.getTableSegmentStatsRecorder(), null, null, null, true, serviceBuilder.getLowPriorityExecutor(), Config.TLS_PROTOCOL_VERSION.toArray(new String[Config.TLS_PROTOCOL_VERSION.size()]));
        server.startListening();
        controllerWrapper.awaitRunning();
        controllerWrapper.getControllerService().createScope("test", 0L).get();
        controller.createStream("test", "test", CONFIG).get();
        @Cleanup MockClientFactory clientFactory = new MockClientFactory("test", controller, internalCF.getConnectionPool());
        // Mocking pravega service by putting scale up and scale down requests for the stream
        @Cleanup EventStreamWriter<String> test = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().build());
        // keep writing. Scale should happen
        long start = System.currentTimeMillis();
        char[] chars = new char[1];
        Arrays.fill(chars, 'a');
        String str = new String(chars);
        CompletableFuture.runAsync(() -> {
            while (System.currentTimeMillis() - start < Duration.ofMinutes(3).toMillis()) {
                try {
                    test.writeEvent("0", str).get();
                } catch (Throwable e) {
                    System.err.println("test exception writing events " + e.getMessage());
                    break;
                }
            }
        });
        Retry.withExpBackoff(10, 10, 100, 10000).retryingOn(NotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments("test", "test").thenAccept(streamSegments -> {
            if (streamSegments.getSegments().size() > 3) {
                System.err.println("Success");
                log.info("Success");
                System.exit(0);
            } else {
                throw new NotDoneException();
            }
        }), executor).exceptionally(e -> {
            System.err.println("Failure");
            log.error("Failure");
            System.exit(1);
            return null;
        }).get();
    } catch (Throwable e) {
        System.err.print("Test failed with exception: " + e.getMessage());
        System.exit(-1);
    }
    System.exit(0);
}

Example 33 with JavaSerializer

use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.

the class WatermarkingTest method recreateStreamWatermarkTest.

@Test(timeout = 120000)
public void recreateStreamWatermarkTest() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    // then delete stream and move to next iteration and verify that watermarks are generated.
    for (int i = 0; i < 2; i++) {
        String scope = "scope";
        String stream = "recreateStreamWatermarkTest";
        streamManager.createScope(scope);
        streamManager.createStream(scope, stream, config);
        // create writer
        @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
        JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
        @Cleanup EventStreamWriter<Long> writer = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
        AtomicBoolean stopFlag = new AtomicBoolean(false);
        // write events
        CompletableFuture<Void> writerFuture = writeEvents(writer, stopFlag);
        @Cleanup SynchronizerClientFactory syncClientFactory = SynchronizerClientFactory.withScope(scope, clientConfig);
        String markStream = NameUtils.getMarkStreamForStream(stream);
        @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
        LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
        fetchWatermarks(watermarkReader, watermarks, stopFlag);
        AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
        // stop run and seal and delete stream
        stopFlag.set(true);
        writerFuture.join();
        streamManager.sealStream(scope, stream);
        streamManager.deleteStream(scope, stream);
    }
}

Example 34 with JavaSerializer

use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.

the class ReadTest method testEventPointer.

@Test(timeout = 10000)
public void testEventPointer() throws ReinitializationRequiredException, NoSuchEventException {
    String endpoint = "localhost";
    String streamName = "testEventPointer";
    String readerName = "reader";
    String readerGroup = "testEventPointer-group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world ";
    String scope = "Scope1";
    StreamSegmentStore store = SERVICE_BUILDER.createStreamSegmentService();
    TableStore tableStore = SERVICE_BUILDER.createTableStoreService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, tableStore, NoOpScheduledExecutor.get());
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    @Cleanup MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, null);
    streamManager.createReaderGroup(readerGroup, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    for (int i = 0; i < 100; i++) {
        producer.writeEvent(testString + i);
    }
    producer.flush();
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroup, serializer, ReaderConfig.builder().build());
    for (int i = 0; i < 100; i++) {
        EventPointer pointer = reader.readNextEvent(5000).getEventPointer();
        String read = reader.fetchEvent(pointer);
        assertEquals(testString + i, read);
    }
}

Example 35 with JavaSerializer

use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.

the class CheckpointTest method testMoreReadersThanSegments.

@Test(timeout = 20000)
public void testMoreReadersThanSegments() throws ReinitializationRequiredException, InterruptedException, ExecutionException, TimeoutException {
    String endpoint = "localhost";
    String streamName = "testMoreReadersThanSegments";
    String readerGroupName = "testMoreReadersThanSegments-group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world\n";
    String scope = "Scope1";
    StreamSegmentStore store = SERVICE_BUILDER.createStreamSegmentService();
    TableStore tableStore = SERVICE_BUILDER.createTableStoreService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, tableStore, SERVICE_BUILDER.getLowPriorityExecutor());
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    @Cleanup MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
    streamManager.createReaderGroup(readerGroupName, groupConfig);
    @Cleanup ReaderGroup readerGroup = streamManager.getReaderGroup(readerGroupName);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    producer.writeEvent(testString);
    producer.writeEvent(testString);
    producer.writeEvent(testString);
    producer.flush();
    AtomicLong clock = new AtomicLong();
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("reader1", readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    @Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("reader2", readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    @Cleanup("shutdown") final InlineExecutor backgroundExecutor = new InlineExecutor();
    CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("Checkpoint", backgroundExecutor);
    assertFalse(checkpoint.isDone());
    EventRead<String> read = reader1.readNextEvent(60000);
    assertTrue(read.isCheckpoint());
    assertEquals("Checkpoint", read.getCheckpointName());
    assertNull(read.getEvent());
    read = reader2.readNextEvent(60000);
    assertTrue(read.isCheckpoint());
    assertEquals("Checkpoint", read.getCheckpointName());
    assertNull(read.getEvent());
    read = reader1.readNextEvent(100);
    assertFalse(read.isCheckpoint());
    assertEquals(testString, read.getEvent());
    read = reader2.readNextEvent(100);
    assertFalse(read.isCheckpoint());
    assertNull(read.getEvent());
    Checkpoint cpResult = checkpoint.get(5, TimeUnit.SECONDS);
    assertTrue(checkpoint.isDone());
    assertEquals("Checkpoint", cpResult.getName());
}