Examples with SynchronizerConfig io.pravega.client.state.SynchronizerConfig used on opensource projects

Example 6 with SynchronizerConfig

use of io.pravega.client.state.SynchronizerConfig in project pravega by pravega.

the class ReaderGroupStateManagerTest method testSegmentsAssigned.

@Test(timeout = 10000)
public void testSegmentsAssigned() throws ReinitializationRequiredException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    AtomicLong clock = new AtomicLong();
    Map<Segment, Long> segments = new HashMap<>();
    segments.put(new Segment(scope, stream, 0), 0L);
    segments.put(new Segment(scope, stream, 1), 1L);
    segments.put(new Segment(scope, stream, 2), 2L);
    segments.put(new Segment(scope, stream, 3), 3L);
    ReaderGroupStateManager.initializeReaderGroup(stateSynchronizer, ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments);
    ReaderGroupStateManager reader1 = new ReaderGroupStateManager("reader1", stateSynchronizer, controller, clock::get);
    reader1.initializeReader(0);
    ReaderGroupStateManager reader2 = new ReaderGroupStateManager("reader2", stateSynchronizer, controller, clock::get);
    reader2.initializeReader(0);
    Map<Segment, Long> segments1 = reader1.acquireNewSegmentsIfNeeded(0);
    assertFalse(segments1.isEmpty());
    assertEquals(2, segments1.size());
    assertTrue(reader1.acquireNewSegmentsIfNeeded(0).isEmpty());
    assertNull(reader1.findSegmentToReleaseIfRequired());
    Map<Segment, Long> segments2 = reader2.acquireNewSegmentsIfNeeded(0);
    assertFalse(segments2.isEmpty());
    assertEquals(2, segments2.size());
    assertTrue(reader2.acquireNewSegmentsIfNeeded(0).isEmpty());
    assertNull(reader2.findSegmentToReleaseIfRequired());
    assertTrue(Sets.intersection(segments1.keySet(), segments2.keySet()).isEmpty());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertFalse(reader1.releaseSegment(new Segment(scope, stream, 0), 0, 0));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertTrue(reader1.acquireNewSegmentsIfNeeded(0).isEmpty());
    assertNull(reader1.findSegmentToReleaseIfRequired());
    assertTrue(reader2.acquireNewSegmentsIfNeeded(0).isEmpty());
    assertNull(reader2.findSegmentToReleaseIfRequired());
    reader1.readerShutdown(new PositionImpl(segments1));
    Map<Segment, Long> segmentsRecovered = reader2.acquireNewSegmentsIfNeeded(0);
    assertFalse(segmentsRecovered.isEmpty());
    assertEquals(2, segmentsRecovered.size());
    assertEquals(segments1, segmentsRecovered);
    assertTrue(reader2.acquireNewSegmentsIfNeeded(0).isEmpty());
    assertNull(reader2.findSegmentToReleaseIfRequired());
    segments2.putAll(segmentsRecovered);
    reader2.readerShutdown(new PositionImpl(segments2));
    reader1.initializeReader(0);
    segments1 = reader1.acquireNewSegmentsIfNeeded(0);
    assertEquals(4, segments1.size());
    assertEquals(segments2, segments1);
}

Example 7 with SynchronizerConfig

use of io.pravega.client.state.SynchronizerConfig in project pravega by pravega.

the class ReaderGroupStateManagerTest method testReleaseWhenReadersAdded.

// (timeout = 20000)
@Test
public void testReleaseWhenReadersAdded() throws ReinitializationRequiredException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    AtomicLong clock = new AtomicLong();
    Map<Segment, Long> segments = new HashMap<>();
    segments.put(new Segment(scope, stream, 0), 0L);
    segments.put(new Segment(scope, stream, 1), 1L);
    segments.put(new Segment(scope, stream, 2), 2L);
    segments.put(new Segment(scope, stream, 3), 3L);
    segments.put(new Segment(scope, stream, 4), 4L);
    segments.put(new Segment(scope, stream, 5), 5L);
    ReaderGroupStateManager.initializeReaderGroup(stateSynchronizer, ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments);
    ReaderGroupStateManager reader1 = new ReaderGroupStateManager("reader1", stateSynchronizer, controller, clock::get);
    reader1.initializeReader(0);
    Map<Segment, Long> segments1 = reader1.acquireNewSegmentsIfNeeded(0);
    assertEquals(6, segments1.size());
    ReaderGroupStateManager reader2 = new ReaderGroupStateManager("reader2", stateSynchronizer, controller, clock::get);
    reader2.initializeReader(0);
    assertTrue(reader2.acquireNewSegmentsIfNeeded(0).isEmpty());
    assertNull(reader1.findSegmentToReleaseIfRequired());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertNotNull(reader1.findSegmentToReleaseIfRequired());
    reader1.releaseSegment(new Segment(scope, stream, 3), 3, 0);
    assertNull(reader1.findSegmentToReleaseIfRequired());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertNotNull(reader1.findSegmentToReleaseIfRequired());
    reader1.releaseSegment(new Segment(scope, stream, 4), 4, 0);
    assertNull(reader1.findSegmentToReleaseIfRequired());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertNotNull(reader1.findSegmentToReleaseIfRequired());
    reader1.releaseSegment(new Segment(scope, stream, 5), 5, 0);
    assertNull(reader1.findSegmentToReleaseIfRequired());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertNull(reader1.findSegmentToReleaseIfRequired());
    Map<Segment, Long> segments2 = reader2.acquireNewSegmentsIfNeeded(0);
    assertEquals(3, segments2.size());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    ReaderGroupStateManager reader3 = new ReaderGroupStateManager("reader3", stateSynchronizer, controller, clock::get);
    reader3.initializeReader(0);
    assertTrue(reader3.acquireNewSegmentsIfNeeded(0).isEmpty());
    assertNotNull(reader1.findSegmentToReleaseIfRequired());
    reader1.releaseSegment(new Segment(scope, stream, 0), 0, 0);
    assertNull(reader1.findSegmentToReleaseIfRequired());
    assertNotNull(reader2.findSegmentToReleaseIfRequired());
    reader2.releaseSegment(new Segment(scope, stream, 3), 3, 0);
    assertNull(reader2.findSegmentToReleaseIfRequired());
    Map<Segment, Long> segments3 = reader3.acquireNewSegmentsIfNeeded(0);
    assertEquals(2, segments3.size());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertTrue(reader3.acquireNewSegmentsIfNeeded(0).isEmpty());
    assertNull(reader1.findSegmentToReleaseIfRequired());
    assertNull(reader2.findSegmentToReleaseIfRequired());
    assertNull(reader3.findSegmentToReleaseIfRequired());
}

Example 8 with SynchronizerConfig

use of io.pravega.client.state.SynchronizerConfig in project pravega by pravega.

the class ReaderGroupStateManagerTest method testCompaction.

@Test(timeout = 10000)
public void testCompaction() throws ReinitializationRequiredException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> state1 = createState(stream, clientFactory, config);
    Segment s1 = new Segment(scope, stream, 1);
    Segment s2 = new Segment(scope, stream, 2);
    Map<Segment, Long> segments = new HashMap<>();
    segments.put(s1, 1L);
    segments.put(s2, 2L);
    AtomicLong clock = new AtomicLong();
    ReaderGroupStateManager.initializeReaderGroup(state1, ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments);
    ReaderGroupStateManager r1 = new ReaderGroupStateManager("r1", state1, controller, clock::get);
    r1.initializeReader(0);
    r1.acquireNewSegmentsIfNeeded(0);
    assertTrue(state1.getState().getUnassignedSegments().isEmpty());
    state1.compact(s -> new ReaderGroupState.CompactReaderGroupState(s));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    r1.acquireNewSegmentsIfNeeded(0);
    state1.compact(s -> new ReaderGroupState.CompactReaderGroupState(s));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    @Cleanup StateSynchronizer<ReaderGroupState> state2 = createState(stream, clientFactory, config);
    ReaderGroupStateManager r2 = new ReaderGroupStateManager("r2", state2, controller, clock::get);
    r2.initializeReader(0);
    assertEquals(state1.getState().getPositions(), state2.getState().getPositions());
    state1.fetchUpdates();
    assertTrue(r1.releaseSegment(s1, 1, 1));
    state2.fetchUpdates();
    assertFalse(state2.getState().getUnassignedSegments().isEmpty());
    assertFalse(r2.acquireNewSegmentsIfNeeded(0).isEmpty());
    state2.fetchUpdates();
    assertTrue(state2.getState().getUnassignedSegments().isEmpty());
    assertEquals(Collections.singleton(s2), state2.getState().getSegments("r1"));
    assertEquals(Collections.singleton(s1), state2.getState().getSegments("r2"));
    state2.compact(s -> new ReaderGroupState.CompactReaderGroupState(s));
    r1.findSegmentToReleaseIfRequired();
    r1.acquireNewSegmentsIfNeeded(0);
    r2.getCheckpoint();
    @Cleanup StateSynchronizer<ReaderGroupState> state3 = createState(stream, clientFactory, config);
    state3.fetchUpdates();
    assertEquals(state3.getState().getPositions(), state1.getState().getPositions());
    assertEquals(state3.getState().getPositions(), state2.getState().getPositions());
}

Example 9 with SynchronizerConfig

use of io.pravega.client.state.SynchronizerConfig in project pravega by pravega.

the class ReaderGroupStateManagerTest method testSegmentMerge.

@Test(timeout = 20000)
public void testSegmentMerge() throws ReinitializationRequiredException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    Segment initialSegmentA = new Segment(scope, stream, 0);
    Segment initialSegmentB = new Segment(scope, stream, 1);
    Segment successor = new Segment(scope, stream, 2);
    MockController controller = new MockControllerWithSuccessors(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, new StreamSegmentsWithPredecessors(Collections.singletonMap(new SegmentWithRange(successor, 0.0, 1.0), ImmutableList.of(0, 1)), ""));
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    Map<Segment, Long> segments = new HashMap<>();
    segments.put(initialSegmentA, 1L);
    segments.put(initialSegmentB, 2L);
    ReaderGroupStateManager.initializeReaderGroup(stateSynchronizer, ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments);
    val readerState = new ReaderGroupStateManager("testReader", stateSynchronizer, controller, null);
    readerState.initializeReader(0);
    Map<Segment, Long> newSegments = readerState.acquireNewSegmentsIfNeeded(0);
    assertEquals(2, newSegments.size());
    assertEquals(Long.valueOf(1), newSegments.get(initialSegmentA));
    assertEquals(Long.valueOf(2), newSegments.get(initialSegmentB));
    readerState.handleEndOfSegment(initialSegmentA);
    newSegments = readerState.acquireNewSegmentsIfNeeded(0);
    assertTrue(newSegments.isEmpty());
    readerState.handleEndOfSegment(initialSegmentB);
    newSegments = readerState.acquireNewSegmentsIfNeeded(0);
    assertEquals(1, newSegments.size());
    assertEquals(Long.valueOf(0), newSegments.get(successor));
    newSegments = readerState.acquireNewSegmentsIfNeeded(0);
    assertTrue(newSegments.isEmpty());
}

Example 10 with SynchronizerConfig

use of io.pravega.client.state.SynchronizerConfig in project pravega by pravega.

the class RevisionedStreamClientTest method testConditionalWrite.

@Test
public void testConditionalWrite() {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    @Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    @Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    RevisionedStreamClient<String> client = clientFactory.createRevisionedStreamClient(stream, new JavaSerializer<>(), config);
    client.writeUnconditionally("a");
    Revision revision = client.fetchLatestRevision();
    Revision newRevision = client.writeConditionally(revision, "b");
    assertNotNull(newRevision);
    assertTrue(newRevision.compareTo(revision) > 0);
    assertEquals(newRevision, client.fetchLatestRevision());
    Revision failed = client.writeConditionally(revision, "fail");
    assertNull(failed);
    assertEquals(newRevision, client.fetchLatestRevision());
    Iterator<Entry<Revision, String>> iter = client.readFrom(revision);
    assertTrue(iter.hasNext());
    Entry<Revision, String> entry = iter.next();
    assertEquals(newRevision, entry.getKey());
    assertEquals("b", entry.getValue());
    assertFalse(iter.hasNext());
}