Examples with Segment io.pravega.client.segment.impl.Segment used on opensource projects

Example 41 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class EventStreamWriterTest method testEndOfSegmentBackgroundRefresh.

@Test
public void testEndOfSegmentBackgroundRefresh() {
    String scope = "scope";
    String streamName = "stream";
    String routingKey = "RoutingKey";
    StreamImpl stream = new StreamImpl(scope, streamName);
    Segment segment1 = new Segment(scope, streamName, 0);
    Segment segment2 = new Segment(scope, streamName, 1);
    EventWriterConfig config = EventWriterConfig.builder().build();
    SegmentOutputStreamFactory streamFactory = Mockito.mock(SegmentOutputStreamFactory.class);
    Controller controller = Mockito.mock(Controller.class);
    FakeSegmentOutputStream outputStream1 = new FakeSegmentOutputStream(segment1);
    FakeSegmentOutputStream outputStream2 = new FakeSegmentOutputStream(segment2);
    Mockito.when(streamFactory.createOutputStreamForSegment(eq(segment1), any(), any(), any())).thenAnswer(i -> {
        outputStream1.callBackForSealed = i.getArgument(1);
        return outputStream1;
    });
    Mockito.when(streamFactory.createOutputStreamForSegment(eq(segment2), any(), any(), any())).thenAnswer(i -> {
        outputStream2.callBackForSealed = i.getArgument(1);
        return outputStream2;
    });
    JavaSerializer<String> serializer = new JavaSerializer<>();
    Mockito.when(controller.getCurrentSegments(scope, streamName)).thenReturn(getSegmentsFuture(segment1)).thenReturn(getSegmentsFuture(segment2));
    @Cleanup EventStreamWriter<String> writer = new EventStreamWriterImpl<>(stream, controller, streamFactory, serializer, config, new InlineExecutor());
    writer.writeEvent(routingKey, "Foo");
    Mockito.when(controller.getCurrentSegments(scope, streamName)).thenReturn(getSegmentsFuture(segment2));
    Mockito.when(controller.getSuccessors(segment1)).thenReturn(getReplacement(segment1, segment2));
    writer.writeEvent(routingKey, "Bar");
    Mockito.verify(controller, Mockito.times(1)).getCurrentSegments(any(), any());
    assertEquals(2, outputStream1.getUnackedEventsOnSeal().size());
    assertEquals("Foo", serializer.deserialize(outputStream1.getUnackedEventsOnSeal().get(0).getData()));
    assertEquals("Bar", serializer.deserialize(outputStream1.getUnackedEventsOnSeal().get(1).getData()));
    // simulate a segment sealed callback.
    outputStream1.invokeSealedCallBack();
    writer.writeEvent(routingKey, "TestData");
    // This time the actual handleLogSealed is invoked and the resend method resends data to outputStream2.
    assertEquals(3, outputStream2.getUnackedEventsOnSeal().size());
    assertEquals("Foo", serializer.deserialize(outputStream2.getUnackedEventsOnSeal().get(0).getData()));
    assertEquals("Bar", serializer.deserialize(outputStream2.getUnackedEventsOnSeal().get(1).getData()));
    assertEquals("TestData", serializer.deserialize(outputStream2.getUnackedEventsOnSeal().get(2).getData()));
}

Example 42 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class EventStreamWriterTest method testRetryFlushSegmentSealed.

@Test
public void testRetryFlushSegmentSealed() throws EndOfSegmentException, SegmentTruncatedException {
    String scope = "scope";
    String streamName = "stream";
    StreamImpl stream = new StreamImpl(scope, streamName);
    Segment segment1 = new Segment(scope, streamName, 0);
    Segment segment2 = new Segment(scope, streamName, 1);
    EventWriterConfig config = EventWriterConfig.builder().build();
    SegmentOutputStreamFactory streamFactory = Mockito.mock(SegmentOutputStreamFactory.class);
    Controller controller = Mockito.mock(Controller.class);
    SealedSegmentOutputStream outputStream = new SealedSegmentOutputStream(segment1);
    Mockito.when(controller.getCurrentSegments(scope, streamName)).thenReturn(getSegmentsFuture(segment1));
    Mockito.when(controller.getSuccessors(segment1)).thenReturn(getReplacement(segment1, segment2));
    Mockito.when(streamFactory.createOutputStreamForSegment(eq(segment1), any(), any(), any())).thenAnswer(i -> {
        outputStream.callBackForSealed = i.getArgument(1);
        return outputStream;
    });
    JavaSerializer<String> serializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<String> writer = new EventStreamWriterImpl<>(stream, controller, streamFactory, serializer, config, new InlineExecutor());
    writer.writeEvent("Foo");
    Mockito.verify(controller).getCurrentSegments(any(), any());
    assertTrue(outputStream.getUnackedEventsOnSeal().size() > 0);
    MockSegmentIoStreams outputStream2 = new MockSegmentIoStreams(segment2);
    Mockito.when(streamFactory.createOutputStreamForSegment(eq(segment2), any(), any(), any())).thenReturn(outputStream2);
    Async.testBlocking(() -> {
        // blocking on flush.
        writer.flush();
    }, () -> {
        // trigger release with a segmentSealedException.
        outputStream.releaseFlush();
        // trigger Sealed Segment call back.
        outputStream.invokeSealedCallBack();
    });
    Mockito.verify(controller, Mockito.times(1)).getCurrentSegments(any(), any());
    assertTrue(outputStream2.fetchCurrentSegmentLength() > 0);
    assertEquals(serializer.serialize("Foo"), outputStream2.read());
}

Example 43 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class AutoScaleProcessor method triggerScaleDown.

private void triggerScaleDown(String streamSegmentName, boolean silent) {
    if (initialized.get()) {
        Pair<Long, Long> pair = cache.getIfPresent(streamSegmentName);
        long lastRequestTs = 0;
        if (pair != null && pair.getValue() != null) {
            lastRequestTs = pair.getValue();
        }
        long timestamp = System.currentTimeMillis();
        if (timestamp - lastRequestTs > configuration.getMuteDuration().toMillis()) {
            log.info("sending request for scale down for {}", streamSegmentName);
            Segment segment = Segment.fromScopedName(streamSegmentName);
            AutoScaleEvent event = new AutoScaleEvent(segment.getScope(), segment.getStreamName(), segment.getSegmentNumber(), AutoScaleEvent.DOWN, timestamp, 0, silent);
            writeRequest(event).thenAccept(x -> {
                if (!silent) {
                    // mute only scale downs
                    cache.put(streamSegmentName, new ImmutablePair<>(0L, timestamp));
                }
            });
        }
    }
}

Example 44 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class AutoScaleProcessor method triggerScaleUp.

private void triggerScaleUp(String streamSegmentName, int numOfSplits) {
    if (initialized.get()) {
        Pair<Long, Long> pair = cache.getIfPresent(streamSegmentName);
        long lastRequestTs = 0;
        if (pair != null && pair.getKey() != null) {
            lastRequestTs = pair.getKey();
        }
        long timestamp = System.currentTimeMillis();
        if (timestamp - lastRequestTs > configuration.getMuteDuration().toMillis()) {
            log.info("sending request for scale up for {}", streamSegmentName);
            Segment segment = Segment.fromScopedName(streamSegmentName);
            AutoScaleEvent event = new AutoScaleEvent(segment.getScope(), segment.getStreamName(), segment.getSegmentNumber(), AutoScaleEvent.UP, timestamp, numOfSplits, false);
            // Mute scale for timestamp for both scale up and down
            writeRequest(event).thenAccept(x -> cache.put(streamSegmentName, new ImmutablePair<>(timestamp, timestamp)));
        }
    }
}

Example 45 with Segment

use of io.pravega.client.segment.impl.Segment in project pravega by pravega.

the class ReaderGroupStateManagerTest method testSegmentsCannotBeReleasedWithoutCheckpoint.

@Test(timeout = 10000)
public void testSegmentsCannotBeReleasedWithoutCheckpoint() throws ReinitializationRequiredException {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    Segment segment0 = new Segment(scope, stream, 0);
    Segment segment1 = new Segment(scope, stream, 1);
    Segment segment2 = new Segment(scope, stream, 2);
    MockController controller = new MockControllerWithSuccessors(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, new StreamSegmentsWithPredecessors(ImmutableMap.of(), ""));
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup StateSynchronizer<ReaderGroupState> stateSynchronizer = createState(stream, clientFactory, config);
    AtomicLong clock = new AtomicLong();
    Map<Segment, Long> segments = ImmutableMap.of(segment0, 0L, segment1, 1L, segment2, 2L);
    ReaderGroupStateManager.initializeReaderGroup(stateSynchronizer, ReaderGroupConfig.builder().stream(Stream.of(scope, stream)).build(), segments);
    val readerState1 = new ReaderGroupStateManager("reader1", stateSynchronizer, controller, clock::get);
    readerState1.initializeReader(0);
    val readerState2 = new ReaderGroupStateManager("reader2", stateSynchronizer, controller, clock::get);
    readerState2.initializeReader(0);
    assertEquals(segments, stateSynchronizer.getState().getUnassignedSegments());
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP1"));
    stateSynchronizer.fetchUpdates();
    assertEquals("CP1", readerState1.getCheckpoint());
    assertEquals(Collections.emptyMap(), readerState1.acquireNewSegmentsIfNeeded(1));
    assertEquals(Collections.emptyMap(), readerState2.acquireNewSegmentsIfNeeded(2));
    assertEquals("CP1", readerState2.getCheckpoint());
    readerState1.checkpoint("CP1", new PositionImpl(Collections.emptyMap()));
    readerState2.checkpoint("CP1", new PositionImpl(Collections.emptyMap()));
    assertEquals(segments, stateSynchronizer.getState().getPositionsForCompletedCheckpoint("CP1"));
    Map<Segment, Long> segments1 = readerState1.acquireNewSegmentsIfNeeded(1);
    Map<Segment, Long> segments2 = readerState2.acquireNewSegmentsIfNeeded(2);
    assertFalse(segments1.isEmpty());
    assertFalse(segments2.isEmpty());
    assertEquals(0, stateSynchronizer.getState().getNumberOfUnassignedSegments());
    // Induce imbalance
    for (Entry<Segment, Long> entry : segments1.entrySet()) {
        stateSynchronizer.updateStateUnconditionally(new ReaderGroupState.ReleaseSegment("reader1", entry.getKey(), entry.getValue()));
        stateSynchronizer.updateStateUnconditionally(new ReaderGroupState.AcquireSegment("reader2", entry.getKey()));
    }
    stateSynchronizer.updateStateUnconditionally(new CreateCheckpoint("CP2"));
    stateSynchronizer.fetchUpdates();
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertNull(readerState1.findSegmentToReleaseIfRequired());
    assertNull(readerState2.findSegmentToReleaseIfRequired());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    assertFalse(readerState2.releaseSegment(segments2.keySet().iterator().next(), 20, 2));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    readerState1.checkpoint("CP2", new PositionImpl(Collections.emptyMap()));
    readerState2.checkpoint("CP2", new PositionImpl(segments));
    assertEquals(segments, stateSynchronizer.getState().getPositionsForCompletedCheckpoint("CP2"));
    Segment toRelease = readerState2.findSegmentToReleaseIfRequired();
    assertNotNull(toRelease);
    assertTrue(readerState2.releaseSegment(toRelease, 10, 1));
    assertEquals(1, stateSynchronizer.getState().getNumberOfUnassignedSegments());
}