Examples with CreateCheckpoint io.pravega.client.stream.impl.ReaderGroupState.CreateCheckpoint used on opensource projects

Example 1 with CreateCheckpoint

use of io.pravega.client.stream.impl.ReaderGroupState.CreateCheckpoint in project pravega by pravega.

the class ReaderGroupStateManagerTest method testSegmentsCannotBeReleasedWithoutCheckpoint.

@Test(timeout = 10000)
public void testSegmentsCannotBeReleasedWithoutCheckpoint() throws ReinitializationRequiredException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    Segment segment0 = new Segment(scope, stream, 0);
    Segment segment1 = new Segment(scope, stream, 1);
    Segment segment2 = new Segment(scope, stream, 2);
    MockController controller = new MockControllerWithSuccessors(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, new StreamSegmentsWithPredecessors(ImmutableMap.of(), ""));
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    AtomicLong clock = new AtomicLong();
    Map<Segment, Long> segments = ImmutableMap.of(segment0, 0L, segment1, 1L, segment2, 2L);
    ReaderGroupStateManager.initializeReaderGroup(stateSynchronizer, ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments);
    val readerState1 = new ReaderGroupStateManager("reader1", stateSynchronizer, controller, clock::get);
    readerState1.initializeReader(0);
    val readerState2 = new ReaderGroupStateManager("reader2", stateSynchronizer, controller, clock::get);
    readerState2.initializeReader(0);
    assertEquals(segments, stateSynchronizer.getState().getUnassignedSegments());
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP1"));
    stateSynchronizer.fetchUpdates();
    assertEquals("CP1", readerState1.getCheckpoint());
    assertEquals(Collections.emptyMap(), readerState1.acquireNewSegmentsIfNeeded(1));
    assertEquals(Collections.emptyMap(), readerState2.acquireNewSegmentsIfNeeded(2));
    assertEquals("CP1", readerState2.getCheckpoint());
    readerState1.checkpoint("CP1", new PositionImpl(Collections.emptyMap()));
    readerState2.checkpoint("CP1", new PositionImpl(Collections.emptyMap()));
    assertEquals(segments, stateSynchronizer.getState().getPositionsForCompletedCheckpoint("CP1"));
    Map<Segment, Long> segments1 = readerState1.acquireNewSegmentsIfNeeded(1);
    Map<Segment, Long> segments2 = readerState2.acquireNewSegmentsIfNeeded(2);
    assertFalse(segments1.isEmpty());
    assertFalse(segments2.isEmpty());
    assertEquals(0, stateSynchronizer.getState().getNumberOfUnassignedSegments());
    // Induce imbalance
    for (Entry<Segment, Long> entry : segments1.entrySet()) {
        stateSynchronizer.updateStateUnconditionally(new ReaderGroupState.ReleaseSegment("reader1", entry.getKey(), entry.getValue()));
        stateSynchronizer.updateStateUnconditionally(new ReaderGroupState.AcquireSegment("reader2", entry.getKey()));
    }
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP2"));
    stateSynchronizer.fetchUpdates();
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertNull(readerState1.findSegmentToReleaseIfRequired());
    assertNull(readerState2.findSegmentToReleaseIfRequired());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertFalse(readerState2.releaseSegment(segments2.keySet().iterator().next(), 20, 2));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    readerState1.checkpoint("CP2", new PositionImpl(Collections.emptyMap()));
    readerState2.checkpoint("CP2", new PositionImpl(segments));
    assertEquals(segments, stateSynchronizer.getState().getPositionsForCompletedCheckpoint("CP2"));
    Segment toRelease = readerState2.findSegmentToReleaseIfRequired();
    assertNotNull(toRelease);
    assertTrue(readerState2.releaseSegment(toRelease, 10, 1));
    assertEquals(1, stateSynchronizer.getState().getNumberOfUnassignedSegments());
}

Example 2 with CreateCheckpoint

use of io.pravega.client.stream.impl.ReaderGroupState.CreateCheckpoint in project pravega by pravega.

the class ReaderGroupImpl method initiateCheckpoint.

@Override
public CompletableFuture<Checkpoint> initiateCheckpoint(String checkpointName, ScheduledExecutorService backgroundExecutor) {
    StateSynchronizer<ReaderGroupState> synchronizer = createSynchronizer();
    synchronizer.updateStateUnconditionally(new CreateCheckpoint(checkpointName));
    AtomicBoolean checkpointPending = new AtomicBoolean(true);
    return Futures.loop(checkpointPending::get, () -> {
        return Futures.delayedTask(() -> {
            synchronizer.fetchUpdates();
            checkpointPending.set(!synchronizer.getState().isCheckpointComplete(checkpointName));
            if (checkpointPending.get()) {
                log.debug("Waiting on checkpoint: {} currentState is: {}", checkpointName, synchronizer.getState());
            }
            return null;
        }, Duration.ofMillis(500), backgroundExecutor);
    }, backgroundExecutor).thenApply(v -> completeCheckpoint(checkpointName, synchronizer)).whenComplete((v, t) -> synchronizer.close());
}

Example 3 with CreateCheckpoint

use of io.pravega.client.stream.impl.ReaderGroupState.CreateCheckpoint in project pravega by pravega.

the class ReaderGroupStateManager method getCheckpoint.

String getCheckpoint() throws ReinitializationRequiredException {
    fetchUpdatesIfNeeded();
    ReaderGroupState state = sync.getState();
    long automaticCpInterval = state.getConfig().getAutomaticCheckpointIntervalMillis();
    if (!state.isReaderOnline(readerId)) {
        throw new ReinitializationRequiredException();
    }
    String checkpoint = state.getCheckpointForReader(readerId);
    if (checkpoint != null) {
        checkpointTimer.reset(Duration.ofMillis(automaticCpInterval));
        return checkpoint;
    }
    if (automaticCpInterval <= 0 || checkpointTimer.hasRemaining() || state.hasOngoingCheckpoint()) {
        return null;
    }
    sync.updateState(s -> s.hasOngoingCheckpoint() ? null : ImmutableList.of(new CreateCheckpoint()));
    checkpointTimer.reset(Duration.ofMillis(automaticCpInterval));
    return state.getCheckpointForReader(readerId);
}

Example 4 with CreateCheckpoint

use of io.pravega.client.stream.impl.ReaderGroupState.CreateCheckpoint in project pravega by pravega.

the class ReaderGroupStateManagerTest method testCheckpointContainsAllShards.

@Test(timeout = 10000)
public void testCheckpointContainsAllShards() throws ReinitializationRequiredException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    Segment segment0 = new Segment(scope, stream, 0);
    Segment segment1 = new Segment(scope, stream, 1);
    Segment segment2 = new Segment(scope, stream, 2);
    MockController controller = new MockControllerWithSuccessors(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, new StreamSegmentsWithPredecessors(ImmutableMap.of(), ""));
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    Map<Segment, Long> segments = ImmutableMap.of(segment0, 0L, segment1, 1L, segment2, 2L);
    ReaderGroupStateManager.initializeReaderGroup(stateSynchronizer, ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments);
    val readerState1 = new ReaderGroupStateManager("reader1", stateSynchronizer, controller, null);
    readerState1.initializeReader(0);
    val readerState2 = new ReaderGroupStateManager("reader2", stateSynchronizer, controller, null);
    readerState2.initializeReader(0);
    assertEquals(segments, stateSynchronizer.getState().getUnassignedSegments());
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP1"));
    stateSynchronizer.fetchUpdates();
    assertEquals("CP1", readerState1.getCheckpoint());
    assertEquals(Collections.emptyMap(), readerState1.acquireNewSegmentsIfNeeded(1));
    assertEquals(Collections.emptyMap(), readerState2.acquireNewSegmentsIfNeeded(2));
    assertEquals("CP1", readerState2.getCheckpoint());
    readerState1.checkpoint("CP1", new PositionImpl(Collections.emptyMap()));
    readerState2.checkpoint("CP1", new PositionImpl(Collections.emptyMap()));
    assertTrue(stateSynchronizer.getState().isCheckpointComplete("CP1"));
    assertEquals(segments, stateSynchronizer.getState().getPositionsForCompletedCheckpoint("CP1"));
}

Example 5 with CreateCheckpoint

use of io.pravega.client.stream.impl.ReaderGroupState.CreateCheckpoint in project pravega by pravega.

the class ReaderGroupStateManagerTest method testCheckpoint.

@Test(timeout = 10000)
public void testCheckpoint() throws ReinitializationRequiredException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    Segment initialSegment = new Segment(scope, stream, 0);
    Segment successorA = new Segment(scope, stream, 1);
    Segment successorB = new Segment(scope, stream, 2);
    MockController controller = new MockControllerWithSuccessors(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, new StreamSegmentsWithPredecessors(ImmutableMap.of(new SegmentWithRange(successorA, 0.0, 0.5), singletonList(0), new SegmentWithRange(successorB, 0.5, 1.0), singletonList(0)), ""));
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    Map<Segment, Long> segments = new HashMap<>();
    segments.put(initialSegment, 1L);
    ReaderGroupStateManager.initializeReaderGroup(stateSynchronizer, ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments);
    val readerState = new ReaderGroupStateManager("testReader", stateSynchronizer, controller, null);
    readerState.initializeReader(0);
    assertNull(readerState.getCheckpoint());
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP1"));
    stateSynchronizer.fetchUpdates();
    assertEquals("CP1", readerState.getCheckpoint());
    assertEquals("CP1", readerState.getCheckpoint());
    readerState.checkpoint("CP1", new PositionImpl(Collections.emptyMap()));
    assertNull(readerState.getCheckpoint());
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP2"));
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP3"));
    stateSynchronizer.fetchUpdates();
    assertEquals("CP2", readerState.getCheckpoint());
    readerState.checkpoint("CP2", new PositionImpl(Collections.emptyMap()));
    assertEquals("CP3", readerState.getCheckpoint());
    readerState.checkpoint("CP3", new PositionImpl(Collections.emptyMap()));
    assertNull(readerState.getCheckpoint());
}