Examples with SynchronizerClientFactory io.pravega.client.SynchronizerClientFactory used on opensource projects

Example 6 with SynchronizerClientFactory

use of io.pravega.client.SynchronizerClientFactory in project pravega by pravega.

the class ReaderGroupStateManagerTest method testCompaction.

@Test(timeout = 10000)
public void testCompaction() throws ReaderNotInReaderGroupException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> state1 = createState(stream, clientFactory, config);
    Segment s1 = new Segment(scope, stream, 1);
    Segment s2 = new Segment(scope, stream, 2);
    Map<SegmentWithRange, Long> segments = new HashMap<>();
    segments.put(new SegmentWithRange(s1, 0.0, 0.5), 1L);
    segments.put(new SegmentWithRange(s2, 0.5, 1.0), 2L);
    AtomicLong clock = new AtomicLong();
    state1.initialize(new ReaderGroupState.ReaderGroupStateInit(ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments, Collections.emptyMap(), false));
    ReaderGroupStateManager r1 = new ReaderGroupStateManager(scope, stream, "r1", state1, controller, clock::get);
    r1.initializeReader(0);
    r1.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap()));
    assertTrue(state1.getState().getUnassignedSegments().isEmpty());
    assertEquals(state1.getState().getAssignedSegments("r1"), segments);
    state1.compact(s -> new ReaderGroupState.CompactReaderGroupState(s));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    r1.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments));
    state1.compact(s -> new ReaderGroupState.CompactReaderGroupState(s));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    @Cleanup StateSynchronizer<ReaderGroupState> state2 = createState(stream, clientFactory, config);
    ReaderGroupStateManager r2 = new ReaderGroupStateManager(scope, stream, "r2", state2, controller, clock::get);
    r2.initializeReader(0);
    assertEquals(state1.getState().getPositions(), state2.getState().getPositions());
    state1.fetchUpdates();
    assertTrue(r1.releaseSegment(s1, 1, 1, new PositionImpl(segments)));
    state2.fetchUpdates();
    assertFalse(state2.getState().getUnassignedSegments().isEmpty());
    assertFalse(r2.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap())).isEmpty());
    state2.fetchUpdates();
    assertTrue(state2.getState().getUnassignedSegments().isEmpty());
    assertEquals(ImmutableMap.of(new SegmentWithRange(s2, 0.5, 1.0), 2L), state2.getState().getAssignedSegments("r1"));
    assertEquals(ImmutableMap.of(new SegmentWithRange(s1, 0.0, 0.5), 1L), state2.getState().getAssignedSegments("r2"));
    state2.compact(s -> new ReaderGroupState.CompactReaderGroupState(s));
    r1.findSegmentToReleaseIfRequired();
    r1.acquireNewSegmentsIfNeeded(0, new PositionImpl(ImmutableMap.of(new SegmentWithRange(s2, 0.5, 1.0), 2L)));
    r2.getCheckpoint();
    @Cleanup StateSynchronizer<ReaderGroupState> state3 = createState(stream, clientFactory, config);
    state3.fetchUpdates();
    assertEquals(state3.getState().getPositions(), state1.getState().getPositions());
    assertEquals(state3.getState().getPositions(), state2.getState().getPositions());
    assertEquals(segments, state3.getState().getLastReadPositions(Stream.of(scope, stream)));
}

Example 7 with SynchronizerClientFactory

use of io.pravega.client.SynchronizerClientFactory in project pravega by pravega.

the class ReaderGroupStateManagerTest method testAcquireRace.

@Test(timeout = 5000)
public void testAcquireRace() throws ReaderNotInReaderGroupException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> state1 = createState(stream, clientFactory, config);
    @Cleanup StateSynchronizer<ReaderGroupState> state2 = createState(stream, clientFactory, config);
    AtomicLong clock = new AtomicLong();
    Map<SegmentWithRange, Long> segments = new HashMap<>();
    segments.put(new SegmentWithRange(new Segment(scope, stream, 0), 0.0, 0.25), 0L);
    segments.put(new SegmentWithRange(new Segment(scope, stream, 1), 0.25, 0.5), 1L);
    segments.put(new SegmentWithRange(new Segment(scope, stream, 2), 0.5, 0.75), 2L);
    segments.put(new SegmentWithRange(new Segment(scope, stream, 3), 0.75, 1.0), 3L);
    state1.initialize(new ReaderGroupState.ReaderGroupStateInit(ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments, Collections.emptyMap(), false));
    ReaderGroupStateManager reader1 = new ReaderGroupStateManager(scope, stream, "reader1", state1, controller, clock::get);
    reader1.initializeReader(0);
    ReaderGroupStateManager reader2 = new ReaderGroupStateManager(scope, stream, "reader2", state2, controller, clock::get);
    reader2.initializeReader(0);
    Map<SegmentWithRange, Long> segments1 = reader1.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap()));
    assertFalse(segments1.isEmpty());
    assertEquals(2, segments1.size());
    assertTrue(reader1.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments1)).isEmpty());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertNull(reader1.findSegmentToReleaseIfRequired());
    Map<SegmentWithRange, Long> segments2 = reader2.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap()));
    assertFalse(segments2.isEmpty());
    assertEquals(2, segments2.size());
    assertNull(reader2.findSegmentToReleaseIfRequired());
    segments1 = reader1.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments2));
    assertTrue(segments1.isEmpty());
}

Example 8 with SynchronizerClientFactory

use of io.pravega.client.SynchronizerClientFactory in project pravega by pravega.

the class ReaderGroupStateManagerTest method testCheckpointUpdatesAssignedSegments.

@Test(timeout = 10000)
public void testCheckpointUpdatesAssignedSegments() throws ReaderNotInReaderGroupException {
    String scope = "scope";
    String stream = "stream";
    String checkpointId = "checkpoint";
    String reader1 = "reader1";
    String reader2 = "reader2";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    SegmentWithRange segment0 = new SegmentWithRange(new Segment(scope, stream, 0), 0.0, 0.33);
    SegmentWithRange segment1 = new SegmentWithRange(new Segment(scope, stream, 1), 0.33, 0.66);
    SegmentWithRange segment2 = new SegmentWithRange(new Segment(scope, stream, 2), 0.66, 1.0);
    MockController controller = new MockControllerWithSuccessors(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, new StreamSegmentsWithPredecessors(ImmutableMap.of(), ""));
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    Map<SegmentWithRange, Long> segments = ImmutableMap.of(segment0, 0L, segment1, 1L, segment2, 2L);
    stateSynchronizer.initialize(new ReaderGroupState.ReaderGroupStateInit(ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments, Collections.emptyMap(), false));
    val readerState1 = new ReaderGroupStateManager(scope, stream, reader1, stateSynchronizer, controller, null);
    readerState1.initializeReader(0);
    val readerState2 = new ReaderGroupStateManager(scope, stream, reader2, stateSynchronizer, controller, null);
    readerState2.initializeReader(0);
    // Assert that readers initially got no assigned segments.
    assertNull(readerState1.findSegmentToReleaseIfRequired());
    assertNull(readerState2.findSegmentToReleaseIfRequired());
    // Assert that both readers have acquired all the segments.
    assertEquals(segments.size(), readerState1.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap())).size() + readerState2.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap())).size());
    assertEquals(Collections.emptyMap(), stateSynchronizer.getState().getUnassignedSegments());
    // Initialize checkpoint in state synchronizer.
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint(checkpointId));
    stateSynchronizer.fetchUpdates();
    assertEquals(checkpointId, readerState1.getCheckpoint());
    assertEquals(checkpointId, readerState2.getCheckpoint());
    // Create some positions for all the segments in the stream > than the initial ones.
    Map<SegmentWithRange, Long> checkpointPositions = new HashMap<>();
    checkpointPositions.put(segment0, 10L);
    checkpointPositions.put(segment1, 10L);
    checkpointPositions.put(segment2, 10L);
    // This should update assigned segments offsets with the checkpoint positions.
    readerState1.checkpoint(checkpointId, new PositionImpl(checkpointPositions));
    readerState2.checkpoint(checkpointId, new PositionImpl(checkpointPositions));
    assertTrue(stateSynchronizer.getState().isCheckpointComplete(checkpointId));
    // Verify that assigned getPositions() retrieves the updated segment offsets.
    Map<Stream, Map<SegmentWithRange, Long>> readergroupPositions = new HashMap<>();
    readergroupPositions.put(Stream.of(scope, stream), checkpointPositions);
    assertEquals(stateSynchronizer.getState().getPositions(), readergroupPositions);
}

Example 9 with SynchronizerClientFactory

use of io.pravega.client.SynchronizerClientFactory in project pravega by pravega.

the class ReaderGroupStateManagerTest method testReachEnd.

@Test(timeout = 10000)
public void testReachEnd() throws ReaderNotInReaderGroupException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, SynchronizerConfig.builder().build());
    Map<SegmentWithRange, Long> segments = new HashMap<>();
    SegmentWithRange segment = new SegmentWithRange(new Segment(scope, stream, 0), 0.0, 1.0);
    segments.put(segment, 1L);
    StreamCutImpl start = new StreamCutImpl(Stream.of(scope, stream), ImmutableMap.of(segment.getSegment(), 0L));
    StreamCutImpl end = new StreamCutImpl(Stream.of(scope, stream), ImmutableMap.of(segment.getSegment(), 100L));
    ReaderGroupConfig config = ReaderGroupConfig.builder().stream(Stream.of(scope, stream), start, end).build();
    stateSynchronizer.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, ReaderGroupImpl.getEndSegmentsForStreams(config), false));
    ReaderGroupStateManager readerState = new ReaderGroupStateManager(scope, stream, "testReader", stateSynchronizer, controller, null);
    readerState.initializeReader(0);
    Segment toRelease = readerState.findSegmentToReleaseIfRequired();
    assertNull(toRelease);
    Map<SegmentWithRange, Long> newSegments = readerState.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments));
    assertFalse(newSegments.isEmpty());
    assertEquals(1, newSegments.size());
    assertTrue(newSegments.containsKey(segment));
    assertTrue(readerState.handleEndOfSegment(segment));
}

Example 10 with SynchronizerClientFactory

use of io.pravega.client.SynchronizerClientFactory in project pravega by pravega.

the class ReaderGroupStateManagerTest method testSegmentMerge.

@Test(timeout = 20000)
public void testSegmentMerge() throws ReaderNotInReaderGroupException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    SegmentWithRange initialSegmentA = new SegmentWithRange(new Segment(scope, stream, 0L), 0.0, 0.5);
    SegmentWithRange initialSegmentB = new SegmentWithRange(new Segment(scope, stream, 1L), 0.5, 1.0);
    SegmentWithRange successor = new SegmentWithRange(new Segment(scope, stream, 2L), 0.0, 1.0);
    MockController controller = new MockControllerWithSuccessors(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, new StreamSegmentsWithPredecessors(Collections.singletonMap(successor, ImmutableList.of(0L, 1L)), ""));
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    Map<SegmentWithRange, Long> segments = new HashMap<>();
    segments.put(initialSegmentA, 1L);
    segments.put(initialSegmentB, 2L);
    stateSynchronizer.initialize(new ReaderGroupState.ReaderGroupStateInit(ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments, Collections.emptyMap(), false));
    val readerState = new ReaderGroupStateManager(scope, stream, "testReader", stateSynchronizer, controller, null);
    readerState.initializeReader(0);
    Map<SegmentWithRange, Long> newSegments = readerState.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap()));
    assertEquals(2, newSegments.size());
    assertEquals(Long.valueOf(1), newSegments.get(initialSegmentA));
    assertEquals(Long.valueOf(2), newSegments.get(initialSegmentB));
    readerState.handleEndOfSegment(initialSegmentA);
    newSegments = readerState.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments));
    assertTrue(newSegments.isEmpty());
    readerState.handleEndOfSegment(initialSegmentB);
    newSegments = readerState.acquireNewSegmentsIfNeeded(0, new PositionImpl(newSegments));
    assertEquals(1, newSegments.size());
    assertEquals(Long.valueOf(0), newSegments.get(successor));
    newSegments = readerState.acquireNewSegmentsIfNeeded(0, new PositionImpl(newSegments));
    assertTrue(newSegments.isEmpty());
}