Examples with Segment io.pravega.client.segment.impl.Segment used on opensource projects

Example 71 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class PositionImplTest method getSegments.

private List<Segment> getSegments() {
    List<Segment> segments = new ArrayList<>();
    segments.add(new Segment("scope", "s0", 0));
    segments.add(new Segment("scope", "s1", 0));
    segments.add(new Segment("scope", "s2", 0));
    segments.add(new Segment("scope", "s3", 0));
    return segments;
}

Example 72 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class PositionImplTest method getSegmentOffsetUpdates.

private List<Map.Entry<Segment, Long>> getSegmentOffsetUpdates(List<Segment> segments) {
    List<Map.Entry<Segment, Long>> offsetUpdates = new ArrayList<>();
    int index = 1;
    for (Segment s : segments) {
        for (long i = 1; i <= index; i++) {
            offsetUpdates.add(new AbstractMap.SimpleEntry<>(s, i));
        }
        index++;
    }
    return offsetUpdates;
}

Example 73 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class PositionImplTest method getSegmentRanges.

private Map<Segment, SegmentWithRange.Range> getSegmentRanges(List<Segment> segments) {
    double rangeShare = 1.0 / segments.size();
    double rangeLow = 0.0;
    Map<Segment, SegmentWithRange.Range> segmentRanges = new HashMap<>();
    for (Segment s : segments) {
        segmentRanges.put(s, new SegmentWithRange.Range(rangeLow, rangeLow + rangeShare));
        rangeLow += rangeShare;
    }
    return segmentRanges;
}

Example 74 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class ReaderGroupStateManagerTest method testSegmentsAssigned.

@Test(timeout = 10000)
public void testSegmentsAssigned() throws ReaderNotInReaderGroupException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    AtomicLong clock = new AtomicLong();
    Map<SegmentWithRange, Long> segments = new HashMap<>();
    segments.put(new SegmentWithRange(new Segment(scope, stream, 0), 0.0, 0.25), 0L);
    segments.put(new SegmentWithRange(new Segment(scope, stream, 1), 0.25, 0.5), 1L);
    segments.put(new SegmentWithRange(new Segment(scope, stream, 2), 0.5, 0.75), 2L);
    segments.put(new SegmentWithRange(new Segment(scope, stream, 3), 0.75, 1.0), 3L);
    stateSynchronizer.initialize(new ReaderGroupState.ReaderGroupStateInit(ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments, Collections.emptyMap(), false));
    ReaderGroupStateManager reader1 = new ReaderGroupStateManager(scope, stream, "reader1", stateSynchronizer, controller, clock::get);
    reader1.initializeReader(0);
    ReaderGroupStateManager reader2 = new ReaderGroupStateManager(scope, stream, "reader2", stateSynchronizer, controller, clock::get);
    reader2.initializeReader(0);
    Map<SegmentWithRange, Long> segments1 = reader1.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap()));
    assertFalse(segments1.isEmpty());
    assertEquals(2, segments1.size());
    assertTrue(reader1.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments1)).isEmpty());
    assertNull(reader1.findSegmentToReleaseIfRequired());
    Map<SegmentWithRange, Long> segments2 = reader2.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap()));
    assertFalse(segments2.isEmpty());
    assertEquals(2, segments2.size());
    assertTrue(reader2.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments2)).isEmpty());
    assertNull(reader2.findSegmentToReleaseIfRequired());
    assertTrue(Sets.intersection(segments1.keySet(), segments2.keySet()).isEmpty());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertTrue(reader1.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments1)).isEmpty());
    assertNull(reader1.findSegmentToReleaseIfRequired());
    assertTrue(reader2.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments2)).isEmpty());
    assertNull(reader2.findSegmentToReleaseIfRequired());
    reader1.readerShutdown(new PositionImpl(segments1));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    Map<SegmentWithRange, Long> segmentsRecovered = reader2.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments2));
    assertFalse(segmentsRecovered.isEmpty());
    assertEquals(2, segmentsRecovered.size());
    assertEquals(segments1, segmentsRecovered);
    assertTrue(reader2.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments)).isEmpty());
    assertNull(reader2.findSegmentToReleaseIfRequired());
    segments2.putAll(segmentsRecovered);
    reader2.readerShutdown(new PositionImpl(segments2));
    reader1.initializeReader(0);
    segments1 = reader1.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap()));
    assertEquals(4, segments1.size());
    assertEquals(segments2, segments1);
}

Example 75 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class ReaderGroupStateManagerTest method testCompaction.

@Test(timeout = 10000)
public void testCompaction() throws ReaderNotInReaderGroupException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> state1 = createState(stream, clientFactory, config);
    Segment s1 = new Segment(scope, stream, 1);
    Segment s2 = new Segment(scope, stream, 2);
    Map<SegmentWithRange, Long> segments = new HashMap<>();
    segments.put(new SegmentWithRange(s1, 0.0, 0.5), 1L);
    segments.put(new SegmentWithRange(s2, 0.5, 1.0), 2L);
    AtomicLong clock = new AtomicLong();
    state1.initialize(new ReaderGroupState.ReaderGroupStateInit(ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments, Collections.emptyMap(), false));
    ReaderGroupStateManager r1 = new ReaderGroupStateManager(scope, stream, "r1", state1, controller, clock::get);
    r1.initializeReader(0);
    r1.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap()));
    assertTrue(state1.getState().getUnassignedSegments().isEmpty());
    assertEquals(state1.getState().getAssignedSegments("r1"), segments);
    state1.compact(s -> new ReaderGroupState.CompactReaderGroupState(s));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    r1.acquireNewSegmentsIfNeeded(0, new PositionImpl(segments));
    state1.compact(s -> new ReaderGroupState.CompactReaderGroupState(s));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    @Cleanup StateSynchronizer<ReaderGroupState> state2 = createState(stream, clientFactory, config);
    ReaderGroupStateManager r2 = new ReaderGroupStateManager(scope, stream, "r2", state2, controller, clock::get);
    r2.initializeReader(0);
    assertEquals(state1.getState().getPositions(), state2.getState().getPositions());
    state1.fetchUpdates();
    assertTrue(r1.releaseSegment(s1, 1, 1, new PositionImpl(segments)));
    state2.fetchUpdates();
    assertFalse(state2.getState().getUnassignedSegments().isEmpty());
    assertFalse(r2.acquireNewSegmentsIfNeeded(0, new PositionImpl(Collections.emptyMap())).isEmpty());
    state2.fetchUpdates();
    assertTrue(state2.getState().getUnassignedSegments().isEmpty());
    assertEquals(ImmutableMap.of(new SegmentWithRange(s2, 0.5, 1.0), 2L), state2.getState().getAssignedSegments("r1"));
    assertEquals(ImmutableMap.of(new SegmentWithRange(s1, 0.0, 0.5), 1L), state2.getState().getAssignedSegments("r2"));
    state2.compact(s -> new ReaderGroupState.CompactReaderGroupState(s));
    r1.findSegmentToReleaseIfRequired();
    r1.acquireNewSegmentsIfNeeded(0, new PositionImpl(ImmutableMap.of(new SegmentWithRange(s2, 0.5, 1.0), 2L)));
    r2.getCheckpoint();
    @Cleanup StateSynchronizer<ReaderGroupState> state3 = createState(stream, clientFactory, config);
    state3.fetchUpdates();
    assertEquals(state3.getState().getPositions(), state1.getState().getPositions());
    assertEquals(state3.getState().getPositions(), state2.getState().getPositions());
    assertEquals(segments, state3.getState().getLastReadPositions(Stream.of(scope, stream)));
}