Search in sources :

Example 56 with MockSegmentStreamFactory

use of io.pravega.client.stream.mock.MockSegmentStreamFactory in project pravega by pravega.

the class ReaderGroupStateManagerTest method testAddReader.

@Test(timeout = 10000)
public void testAddReader() throws ReaderNotInReaderGroupException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    Map<SegmentWithRange, Long> segments = new HashMap<>();
    SegmentWithRange segment = new SegmentWithRange(new Segment(scope, stream, 0), 0.0, 1.0);
    segments.put(segment, 1L);
    stateSynchronizer.initialize(new ReaderGroupState.ReaderGroupStateInit(ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments, Collections.emptyMap(), false));
    ReaderGroupStateManager readerState = new ReaderGroupStateManager(scope, stream, "testReader", stateSynchronizer, controller, null);
    readerState.initializeReader(0);
    Segment toRelease = readerState.findSegmentToReleaseIfRequired();
    assertNull(toRelease);
    Map<SegmentWithRange, Long> newSegments = readerState.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap()));
    assertFalse(newSegments.isEmpty());
    assertEquals(1, newSegments.size());
    assertTrue(newSegments.containsKey(segment));
    assertEquals(1, newSegments.get(segment).longValue());
}
Also used : HashMap(java.util.HashMap) MockSegmentStreamFactory(io.pravega.client.stream.mock.MockSegmentStreamFactory) Cleanup(lombok.Cleanup) Segment(io.pravega.client.segment.impl.Segment) SynchronizerClientFactory(io.pravega.client.SynchronizerClientFactory) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) AtomicLong(java.util.concurrent.atomic.AtomicLong) MockController(io.pravega.client.stream.mock.MockController) SynchronizerConfig(io.pravega.client.state.SynchronizerConfig) Test(org.junit.Test)

Example 57 with MockSegmentStreamFactory

use of io.pravega.client.stream.mock.MockSegmentStreamFactory in project pravega by pravega.

the class ReaderGroupStateManagerTest method testReleaseAndAcquireTimes.

@Test(timeout = 5000)
public void testReleaseAndAcquireTimes() throws ReaderNotInReaderGroupException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> state = createState(stream, clientFactory, config);
    AtomicLong clock = new AtomicLong();
    Map<SegmentWithRange, Long> segments = new HashMap<>();
    segments.put(new SegmentWithRange(new Segment(scope, stream, 0), 0.0, 0.25), 0L);
    segments.put(new SegmentWithRange(new Segment(scope, stream, 1), 0.25, 0.5), 1L);
    segments.put(new SegmentWithRange(new Segment(scope, stream, 2), 0.5, 0.75), 2L);
    segments.put(new SegmentWithRange(new Segment(scope, stream, 3), 0.65, 1.0), 3L);
    state.initialize(new ReaderGroupState.ReaderGroupStateInit(ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments, Collections.emptyMap(), false));
    ReaderGroupStateManager reader1 = new ReaderGroupStateManager(scope, stream, "reader1", state, controller, clock::get);
    reader1.initializeReader(100);
    ReaderGroupStateManager reader2 = new ReaderGroupStateManager(scope, stream, "reader2", state, controller, clock::get);
    reader2.initializeReader(100);
    Map<SegmentWithRange, Long> newSegments = reader1.acquireNewSegmentsIfNeeded(123, new PositionImpl(Collections.emptyMap()));
    assertEquals(0, newSegments.size());
    newSegments = reader2.acquireNewSegmentsIfNeeded(123, new PositionImpl(Collections.emptyMap()));
    assertEquals(0, newSegments.size());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    newSegments = reader1.acquireNewSegmentsIfNeeded(123, new PositionImpl(Collections.emptyMap()));
    assertEquals(2, newSegments.size());
    Duration r1aqt = ReaderGroupStateManager.calculateAcquireTime("reader1", state.getState());
    Duration r2aqt = ReaderGroupStateManager.calculateAcquireTime("reader2", state.getState());
    assertTrue(r1aqt.toMillis() > r2aqt.toMillis());
    Duration r1rlt = ReaderGroupStateManager.calculateReleaseTime("reader1", state.getState());
    Duration r2rlt = ReaderGroupStateManager.calculateReleaseTime("reader2", state.getState());
    assertTrue(r1rlt.toMillis() < r2rlt.toMillis());
    reader1.releaseSegment(newSegments.keySet().iterator().next().getSegment(), 0, 123, new PositionImpl(Collections.emptyMap()));
    newSegments = reader2.acquireNewSegmentsIfNeeded(123, new PositionImpl(Collections.emptyMap()));
    assertEquals(2, newSegments.size());
    r1aqt = ReaderGroupStateManager.calculateAcquireTime("reader1", state.getState());
    r2aqt = ReaderGroupStateManager.calculateAcquireTime("reader2", state.getState());
    assertTrue(r1aqt.toMillis() < r2aqt.toMillis());
    r1rlt = ReaderGroupStateManager.calculateReleaseTime("reader1", state.getState());
    r2rlt = ReaderGroupStateManager.calculateReleaseTime("reader2", state.getState());
    assertTrue(r1rlt.toMillis() > r2rlt.toMillis());
}
Also used : HashMap(java.util.HashMap) MockSegmentStreamFactory(io.pravega.client.stream.mock.MockSegmentStreamFactory) Duration(java.time.Duration) Cleanup(lombok.Cleanup) Segment(io.pravega.client.segment.impl.Segment) SynchronizerClientFactory(io.pravega.client.SynchronizerClientFactory) AtomicLong(java.util.concurrent.atomic.AtomicLong) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) AtomicLong(java.util.concurrent.atomic.AtomicLong) MockController(io.pravega.client.stream.mock.MockController) SynchronizerConfig(io.pravega.client.state.SynchronizerConfig) Test(org.junit.Test)

Example 58 with MockSegmentStreamFactory

use of io.pravega.client.stream.mock.MockSegmentStreamFactory in project pravega by pravega.

the class ReaderGroupStateManagerTest method testCheckpoint.

@Test(timeout = 10000)
public void testCheckpoint() throws ReaderNotInReaderGroupException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    SegmentWithRange initialSegment = new SegmentWithRange(new Segment(scope, stream, 0), 0, 1);
    SegmentWithRange successorA = new SegmentWithRange(new Segment(scope, stream, 1), 0.0, 0.5);
    SegmentWithRange successorB = new SegmentWithRange(new Segment(scope, stream, 2), 0.5, 1.0);
    MockController controller = new MockControllerWithSuccessors(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, new StreamSegmentsWithPredecessors(ImmutableMap.of(successorA, singletonList(0L), successorB, singletonList(0L)), ""));
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    Map<SegmentWithRange, Long> segments = new HashMap<>();
    segments.put(initialSegment, 1L);
    stateSynchronizer.initialize(new ReaderGroupState.ReaderGroupStateInit(ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).disableAutomaticCheckpoints().build(), segments, Collections.emptyMap(), false));
    val readerState = new ReaderGroupStateManager(scope, stream, "testReader", stateSynchronizer, controller, null);
    readerState.initializeReader(0);
    assertNull(readerState.getCheckpoint());
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP1"));
    stateSynchronizer.fetchUpdates();
    assertEquals("CP1", readerState.getCheckpoint());
    assertEquals("CP1", readerState.getCheckpoint());
    readerState.checkpoint("CP1", new PositionImpl(Collections.emptyMap()));
    assertNull(readerState.getCheckpoint());
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP2"));
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP3"));
    stateSynchronizer.fetchUpdates();
    assertEquals("CP2", readerState.getCheckpoint());
    readerState.checkpoint("CP2", new PositionImpl(Collections.emptyMap()));
    assertEquals("CP3", readerState.getCheckpoint());
    readerState.checkpoint("CP3", new PositionImpl(Collections.emptyMap()));
    assertNull(readerState.getCheckpoint());
    // Checking idempotency (state should resolved inconsistency)
    readerState.checkpoint("CP3", new PositionImpl(Collections.emptyMap()));
    assertNull(readerState.getCheckpoint());
}
Also used : lombok.val(lombok.val) HashMap(java.util.HashMap) MockSegmentStreamFactory(io.pravega.client.stream.mock.MockSegmentStreamFactory) Cleanup(lombok.Cleanup) Segment(io.pravega.client.segment.impl.Segment) SynchronizerClientFactory(io.pravega.client.SynchronizerClientFactory) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) AtomicLong(java.util.concurrent.atomic.AtomicLong) CreateCheckpoint(io.pravega.client.stream.impl.ReaderGroupState.CreateCheckpoint) MockController(io.pravega.client.stream.mock.MockController) SynchronizerConfig(io.pravega.client.state.SynchronizerConfig) Test(org.junit.Test)

Aggregations

MockSegmentStreamFactory (io.pravega.client.stream.mock.MockSegmentStreamFactory)58 Cleanup (lombok.Cleanup)55 Test (org.junit.Test)55 MockController (io.pravega.client.stream.mock.MockController)48 Segment (io.pravega.client.segment.impl.Segment)46 AtomicLong (java.util.concurrent.atomic.AtomicLong)39 MockConnectionFactoryImpl (io.pravega.client.stream.mock.MockConnectionFactoryImpl)31 PravegaNodeUri (io.pravega.shared.protocol.netty.PravegaNodeUri)30 SynchronizerClientFactory (io.pravega.client.SynchronizerClientFactory)27 SegmentOutputStream (io.pravega.client.segment.impl.SegmentOutputStream)24 SynchronizerConfig (io.pravega.client.state.SynchronizerConfig)23 Controller (io.pravega.client.control.impl.Controller)19 HashMap (java.util.HashMap)16 ByteBuffer (java.nio.ByteBuffer)11 ClientFactoryImpl (io.pravega.client.stream.impl.ClientFactoryImpl)8 MockClientFactory (io.pravega.client.stream.mock.MockClientFactory)8 lombok.val (lombok.val)8 EventSegmentReader (io.pravega.client.segment.impl.EventSegmentReader)7 InOrder (org.mockito.InOrder)7 ReaderGroupConfig (io.pravega.client.stream.ReaderGroupConfig)6