Examples with Checkpoint io.pravega.client.stream.Checkpoint used on opensource projects

Example 1 with Checkpoint

use of io.pravega.client.stream.Checkpoint in project pravega by pravega.

the class UnreadBytesTest method testUnreadBytes.

@Test(timeout = 50000)
public void testUnreadBytes() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope("unreadbytes").streamName("unreadbytes").scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("unreadbytes").get();
    controller.createStream(config).get();
    @Cleanup ClientFactory clientFactory = ClientFactory.withScope("unreadbytes", ClientConfig.builder().controllerURI(controllerUri).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter("unreadbytes", new JavaSerializer<>(), EventWriterConfig.builder().build());
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope("unreadbytes", ClientConfig.builder().controllerURI(controllerUri).build());
    ReaderGroup readerGroup = groupManager.createReaderGroup("group", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("unreadbytes/unreadbytes").build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", "group", new JavaSerializer<>(), ReaderConfig.builder().build());
    long unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    writer.writeEvent("0", "data of size 30").get();
    writer.writeEvent("0", "data of size 30").get();
    EventRead<String> firstEvent = reader.readNextEvent(15000);
    EventRead<String> secondEvent = reader.readNextEvent(15000);
    assertNotNull(firstEvent);
    assertEquals("data of size 30", firstEvent.getEvent());
    assertNotNull(secondEvent);
    assertEquals("data of size 30", secondEvent.getEvent());
    // trigger a checkpoint.
    CompletableFuture<Checkpoint> chkPointResult = readerGroup.initiateCheckpoint("test", executor);
    EventRead<String> chkpointEvent = reader.readNextEvent(15000);
    assertEquals("test", chkpointEvent.getCheckpointName());
    chkPointResult.join();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);
    writer.writeEvent("0", "data of size 30").get();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bytes: " + unreadBytes, unreadBytes == 30);
}

Example 2 with Checkpoint

use of io.pravega.client.stream.Checkpoint in project pravega by pravega.

the class CheckpointTest method testCheckpointAndRestore.

@Test(timeout = 20000)
public void testCheckpointAndRestore() throws ReinitializationRequiredException, InterruptedException, ExecutionException, TimeoutException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerName = "reader";
    String readerGroupName = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world\n";
    String scope = "Scope1";
    StreamSegmentStore store = this.serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build());
    ReaderGroup readerGroup = streamManager.createReaderGroup(readerGroupName, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    producer.writeEvent(testString);
    producer.writeEvent(testString);
    producer.writeEvent(testString);
    producer.flush();
    AtomicLong clock = new AtomicLong();
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    EventRead<String> read = reader.readNextEvent(60000);
    assertEquals(testString, read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(60000);
    assertEquals(testString, read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    @Cleanup("shutdown") final InlineExecutor backgroundExecutor = new InlineExecutor();
    CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("Checkpoint", backgroundExecutor);
    assertFalse(checkpoint.isDone());
    read = reader.readNextEvent(60000);
    assertTrue(read.isCheckpoint());
    assertEquals("Checkpoint", read.getCheckpointName());
    assertNull(read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(60000);
    assertEquals(testString, read.getEvent());
    Checkpoint cpResult = checkpoint.get(5, TimeUnit.SECONDS);
    assertTrue(checkpoint.isDone());
    assertEquals("Checkpoint", cpResult.getName());
    read = reader.readNextEvent(100);
    assertNull(read.getEvent());
    assertFalse(read.isCheckpoint());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromCheckpoint(cpResult).build());
    try {
        reader.readNextEvent(60000);
        fail();
    } catch (ReinitializationRequiredException e) {
    // Expected
    }
    reader.close();
    reader = clientFactory.createReader(readerName, readerGroupName, serializer, ReaderConfig.builder().build());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(60000);
    assertEquals(testString, read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(100);
    assertNull(read.getEvent());
    assertFalse(read.isCheckpoint());
}

Example 3 with Checkpoint

use of io.pravega.client.stream.Checkpoint in project pravega by pravega.

the class CheckpointTest method testMoreReadersThanSegments.

@Test(timeout = 20000)
public void testMoreReadersThanSegments() throws ReinitializationRequiredException, InterruptedException, ExecutionException, TimeoutException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerGroupName = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world\n";
    String scope = "Scope1";
    StreamSegmentStore store = this.serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build());
    ReaderGroup readerGroup = streamManager.createReaderGroup(readerGroupName, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    producer.writeEvent(testString);
    producer.writeEvent(testString);
    producer.writeEvent(testString);
    producer.flush();
    AtomicLong clock = new AtomicLong();
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("reader1", readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    @Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("reader2", readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    @Cleanup("shutdown") final InlineExecutor backgroundExecutor = new InlineExecutor();
    CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("Checkpoint", backgroundExecutor);
    assertFalse(checkpoint.isDone());
    EventRead<String> read = reader1.readNextEvent(60000);
    assertTrue(read.isCheckpoint());
    assertEquals("Checkpoint", read.getCheckpointName());
    assertNull(read.getEvent());
    read = reader2.readNextEvent(60000);
    assertTrue(read.isCheckpoint());
    assertEquals("Checkpoint", read.getCheckpointName());
    assertNull(read.getEvent());
    Checkpoint cpResult = checkpoint.get(5, TimeUnit.SECONDS);
    assertTrue(checkpoint.isDone());
    assertEquals("Checkpoint", cpResult.getName());
}

Example 4 with Checkpoint

use of io.pravega.client.stream.Checkpoint in project pravega by pravega.

the class ReaderGroupImpl method initiateCheckpoint.

@Override
public CompletableFuture<Checkpoint> initiateCheckpoint(String checkpointName, ScheduledExecutorService backgroundExecutor) {
    StateSynchronizer<ReaderGroupState> synchronizer = createSynchronizer();
    synchronizer.updateStateUnconditionally(new CreateCheckpoint(checkpointName));
    AtomicBoolean checkpointPending = new AtomicBoolean(true);
    return Futures.loop(checkpointPending::get, () -> {
        return Futures.delayedTask(() -> {
            synchronizer.fetchUpdates();
            checkpointPending.set(!synchronizer.getState().isCheckpointComplete(checkpointName));
            if (checkpointPending.get()) {
                log.debug("Waiting on checkpoint: {} currentState is: {}", checkpointName, synchronizer.getState());
            }
            return null;
        }, Duration.ofMillis(500), backgroundExecutor);
    }, backgroundExecutor).thenApply(v -> completeCheckpoint(checkpointName, synchronizer)).whenComplete((v, t) -> synchronizer.close());
}

Example 5 with Checkpoint

use of io.pravega.client.stream.Checkpoint in project pravega by pravega.

the class ReaderGroupImplTest method resetReadersToCheckpoint.

@Test
public void resetReadersToCheckpoint() {
    Map<Segment, Long> positions = new HashMap<>();
    IntStream.of(2).forEach(segNum -> positions.put(new Segment(SCOPE, "s1", segNum), 10L));
    Checkpoint checkpoint = new CheckpointImpl("testChkPoint", positions);
    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromCheckpoint(checkpoint).build());
    verify(synchronizer, times(1)).updateStateUnconditionally(any(Update.class));
}