Examples with SegmentOutputStream io.pravega.client.segment.impl.SegmentOutputStream used on opensource projects

Example 36 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamWriterImpl method resend.

@GuardedBy("writeSealLock")
private void resend(List<PendingEvent> toResend) {
    while (!toResend.isEmpty()) {
        List<PendingEvent> unsent = new ArrayList<>();
        boolean sendFailed = false;
        log.info("Resending {} events", toResend.size());
        for (PendingEvent event : toResend) {
            if (sendFailed) {
                unsent.add(event);
            } else {
                SegmentOutputStream segmentWriter = selector.getSegmentOutputStreamForKey(event.getRoutingKey());
                if (segmentWriter == null) {
                    log.info("No writer for segment during resend.");
                    unsent.addAll(selector.refreshSegmentEventWriters(segmentSealedCallBack));
                    sendFailed = true;
                } else {
                    segmentWriter.write(event);
                }
            }
        }
        toResend = unsent;
    }
}

Example 37 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamWriterImpl method getSegmentWriter.

private SegmentOutputStream getSegmentWriter(String routingKey) {
    SegmentOutputStream segmentWriter = selector.getSegmentOutputStreamForKey(routingKey);
    while (segmentWriter == null) {
        log.info("Don't have a writer for segment: {}", selector.getSegmentForEvent(routingKey));
        handleMissingLog();
        segmentWriter = selector.getSegmentOutputStreamForKey(routingKey);
    }
    return segmentWriter;
}

Example 38 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class RevisionedStreamClientTest method testSegmentSealedFromSegmentOutputStreamError.

@Test
public void testSegmentSealedFromSegmentOutputStreamError() {
    String scope = "scope";
    String stream = "stream";
    // Setup Environment
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    @Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    @Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    // Setup mock
    SegmentOutputStreamFactory outFactory = mock(SegmentOutputStreamFactory.class);
    SegmentOutputStream out = mock(SegmentOutputStream.class);
    when(outFactory.createOutputStreamForSegment(eq(new Segment(scope, stream, 0)), any(), any(), any(DelegationTokenProvider.class))).thenReturn(out);
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, outFactory, streamFactory, streamFactory);
    CompletableFuture<Void> writeFuture = new CompletableFuture<>();
    PendingEvent event1 = PendingEvent.withoutHeader("key", ByteBufferUtils.EMPTY, writeFuture);
    PendingEvent event2 = PendingEvent.withoutHeader("key", ByteBufferUtils.EMPTY, null);
    // Two events are returned when the callback invokes getUnackedEventsOnSeal
    when(out.getUnackedEventsOnSeal()).thenReturn(Arrays.asList(event1, event2));
    @Cleanup RevisionedStreamClient<String> client = clientFactory.createRevisionedStreamClient(stream, new JavaSerializer<>(), SynchronizerConfig.builder().build());
    // simulate invocation of handleSegmentSealed by Segment writer.
    ((RevisionedStreamClientImpl) client).handleSegmentSealed();
    // Verify SegmentOutputStream#getUnackedEventsOnSeal is invoked.
    verify(out, times(1)).getUnackedEventsOnSeal();
    assertTrue(writeFuture.isCompletedExceptionally());
    assertThrows(SegmentSealedException.class, writeFuture::get);
}

Example 39 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamReaderTest method testPositionsContainSealedSegments.

@Test
public void testPositionsContainSealedSegments() throws SegmentSealedException {
    String scope = "scope";
    String stream = "stream";
    String groupName = "readerGroup";
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", -1);
    @Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    @Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    // Mock for the two SegmentInputStreams.
    Segment segment1 = new Segment(scope, stream, 0);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    writeInt(stream1, 1);
    Segment segment2 = new Segment(scope, stream, 1);
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    @Cleanup StateSynchronizer<ReaderGroupState> sync = createStateSynchronizerForReaderGroup(connectionFactory, controller, segmentStreamFactory, Stream.of(scope, stream), "reader1", clock, 2, groupName);
    @Cleanup EventStreamReaderImpl<byte[]> reader = createReader(controller, segmentStreamFactory, "reader1", sync, clock, scope, groupName);
    EventRead<byte[]> event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    assertEquals(ImmutableSet.of(), event.getPosition().asImpl().getCompletedSegments());
    assertEquals(ImmutableSet.of(segment1, segment2), event.getPosition().asImpl().getOwnedSegments());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(1, readInt(event));
    assertEquals(ImmutableSet.of(), event.getPosition().asImpl().getCompletedSegments());
    assertEquals(ImmutableSet.of(segment1, segment2), event.getPosition().asImpl().getOwnedSegments());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    assertEquals(ImmutableSet.of(), event.getPosition().asImpl().getCompletedSegments());
    assertEquals(ImmutableSet.of(segment1, segment2), event.getPosition().asImpl().getOwnedSegments());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    assertEquals(ImmutableSet.of(segment1), event.getPosition().asImpl().getCompletedSegments());
    assertEquals(ImmutableSet.of(segment1, segment2), event.getPosition().asImpl().getOwnedSegments());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(10);
    assertNull(event.getEvent());
    assertEquals(ImmutableSet.of(segment1, segment2), event.getPosition().asImpl().getCompletedSegments());
    assertEquals(ImmutableSet.of(segment1, segment2), event.getPosition().asImpl().getOwnedSegments());
}

Example 40 with SegmentOutputStream

use of io.pravega.client.segment.impl.SegmentOutputStream in project pravega by pravega.

the class EventStreamReaderTest method testRestore.

@Test(timeout = 10000)
public void testRestore() throws SegmentSealedException, ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(new SegmentWithRange(segment, 0, 1), 0L)).thenReturn(Collections.emptyMap());
    @Cleanup SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    writeInt(stream, 1);
    Mockito.when(groupState.getCheckpoint()).thenThrow(new ReinitializationRequiredException());
    assertThrows(ReinitializationRequiredException.class, () -> reader.readNextEvent(0));
    assertTrue(reader.getReaders().isEmpty());
    reader.close();
}