Examples with SegmentInputStream io.pravega.client.segment.impl.SegmentInputStream used on opensource projects

Example 1 with SegmentInputStream

use of io.pravega.client.segment.impl.SegmentInputStream in project pravega by pravega.

the class ClientFactoryImpl method createRevisionedStreamClient.

@Override
public <T> RevisionedStreamClient<T> createRevisionedStreamClient(String streamName, Serializer<T> serializer, SynchronizerConfig config) {
    log.info("Creating revisioned stream client for stream: {} with synchronizer configuration: {}", streamName, config);
    Segment segment = new Segment(scope, streamName, 0);
    SegmentInputStream in = inFactory.createInputStreamForSegment(segment);
    // Segment sealed is not expected for Revisioned Stream Client.
    Consumer<Segment> segmentSealedCallBack = s -> {
        throw new IllegalStateException("RevisionedClient: Segmentsealed exception observed for segment:" + s);
    };
    String delegationToken = Futures.getAndHandleExceptions(controller.getOrRefreshDelegationTokenFor(segment.getScope(), segment.getStreamName()), RuntimeException::new);
    SegmentOutputStream out = outFactory.createOutputStreamForSegment(segment, segmentSealedCallBack, config.getEventWriterConfig(), delegationToken);
    SegmentMetadataClient meta = metaFactory.createSegmentMetadataClient(segment, delegationToken);
    return new RevisionedStreamClientImpl<>(segment, in, out, meta, serializer, controller, delegationToken);
}

Example 2 with SegmentInputStream

use of io.pravega.client.segment.impl.SegmentInputStream in project pravega by pravega.

the class EventStreamReaderImpl method acquireSegmentsIfNeeded.

@GuardedBy("readers")
private void acquireSegmentsIfNeeded() throws ReinitializationRequiredException {
    Map<Segment, Long> newSegments = groupState.acquireNewSegmentsIfNeeded(getLag());
    if (!newSegments.isEmpty()) {
        log.info("{} acquiring segments {}", this, newSegments);
        for (Entry<Segment, Long> newSegment : newSegments.entrySet()) {
            SegmentInputStream in = inputStreamFactory.createInputStreamForSegment(newSegment.getKey());
            in.setOffset(newSegment.getValue());
            readers.add(in);
        }
    }
}

Example 3 with SegmentInputStream

use of io.pravega.client.segment.impl.SegmentInputStream in project pravega by pravega.

the class EventStreamReaderImpl method readNextEvent.

@Override
public EventRead<Type> readNextEvent(long timeout) throws ReinitializationRequiredException, TruncatedDataException {
    synchronized (readers) {
        Preconditions.checkState(!closed, "Reader is closed");
        long waitTime = Math.min(timeout, ReaderGroupStateManager.TIME_UNIT.toMillis());
        Timer timer = new Timer();
        Segment segment = null;
        long offset = -1;
        ByteBuffer buffer;
        do {
            String checkpoint = updateGroupStateIfNeeded();
            if (checkpoint != null) {
                return createEmptyEvent(checkpoint);
            }
            SegmentInputStream segmentReader = orderer.nextSegment(readers);
            if (segmentReader == null) {
                Exceptions.handleInterrupted(() -> Thread.sleep(waitTime));
                buffer = null;
            } else {
                segment = segmentReader.getSegmentId();
                offset = segmentReader.getOffset();
                try {
                    buffer = segmentReader.read(waitTime);
                } catch (EndOfSegmentException e) {
                    handleEndOfSegment(segmentReader);
                    buffer = null;
                } catch (SegmentTruncatedException e) {
                    handleSegmentTruncated(segmentReader);
                    buffer = null;
                }
            }
        } while (buffer == null && timer.getElapsedMillis() < timeout);
        if (buffer == null) {
            return createEmptyEvent(null);
        }
        lastRead = Sequence.create(segment.getSegmentNumber(), offset);
        int length = buffer.remaining() + WireCommands.TYPE_PLUS_LENGTH_SIZE;
        return new EventReadImpl<>(lastRead, deserializer.deserialize(buffer), getPosition(), new EventPointerImpl(segment, offset, length), null);
    }
}

Example 4 with SegmentInputStream

use of io.pravega.client.segment.impl.SegmentInputStream in project pravega by pravega.

the class EventStreamReaderTest method testReleaseSegment.

@Test(timeout = 10000)
public void testReleaseSegment() throws SegmentSealedException, ReinitializationRequiredException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build());
    Segment segment1 = Segment.fromScopedName("Foo/Bar/0");
    Segment segment2 = Segment.fromScopedName("Foo/Bar/1");
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(0L)).thenReturn(ImmutableMap.of(segment1, 0L, segment2, 0L)).thenReturn(Collections.emptyMap());
    SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, "");
    SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, "");
    writeInt(stream1, 1);
    writeInt(stream2, 2);
    reader.readNextEvent(0);
    List<SegmentInputStream> readers = reader.getReaders();
    assertEquals(2, readers.size());
    Assert.assertEquals(segment1, readers.get(0).getSegmentId());
    Assert.assertEquals(segment2, readers.get(1).getSegmentId());
    Mockito.when(groupState.getCheckpoint()).thenReturn("checkpoint");
    assertTrue(reader.readNextEvent(0).isCheckpoint());
    Mockito.when(groupState.getCheckpoint()).thenReturn(null);
    Mockito.when(groupState.findSegmentToReleaseIfRequired()).thenReturn(segment2);
    Mockito.when(groupState.releaseSegment(Mockito.eq(segment2), Mockito.anyLong(), Mockito.anyLong())).thenReturn(true);
    assertFalse(reader.readNextEvent(0).isCheckpoint());
    Mockito.verify(groupState).releaseSegment(Mockito.eq(segment2), Mockito.anyLong(), Mockito.anyLong());
    readers = reader.getReaders();
    assertEquals(1, readers.size());
    Assert.assertEquals(segment1, readers.get(0).getSegmentId());
    reader.close();
}

Example 5 with SegmentInputStream

use of io.pravega.client.segment.impl.SegmentInputStream in project pravega by pravega.

the class EventStreamReaderTest method testAcquireSegment.

@SuppressWarnings("unused")
@Test(timeout = 10000)
public void testAcquireSegment() throws SegmentSealedException, ReinitializationRequiredException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build());
    Segment segment1 = Segment.fromScopedName("Foo/Bar/0");
    Segment segment2 = Segment.fromScopedName("Foo/Bar/1");
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(0L)).thenReturn(ImmutableMap.of(segment1, 0L)).thenReturn(ImmutableMap.of(segment2, 0L)).thenReturn(Collections.emptyMap());
    SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, "");
    SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, "");
    writeInt(stream1, 1);
    writeInt(stream1, 2);
    writeInt(stream2, 3);
    writeInt(stream2, 4);
    reader.readNextEvent(0);
    List<SegmentInputStream> readers = reader.getReaders();
    assertEquals(1, readers.size());
    Assert.assertEquals(segment1, readers.get(0).getSegmentId());
    reader.readNextEvent(0);
    readers = reader.getReaders();
    assertEquals(2, readers.size());
    Assert.assertEquals(segment1, readers.get(0).getSegmentId());
    Assert.assertEquals(segment2, readers.get(1).getSegmentId());
    reader.close();
}