Examples with SegmentMetadataClient io.pravega.client.segment.impl.SegmentMetadataClient used on opensource projects

Example 11 with SegmentMetadataClient

use of io.pravega.client.segment.impl.SegmentMetadataClient in project pravega by pravega.

the class EventStreamReaderTest method testTruncatedSegmentDeleted.

/**
 * This test tests a scenario where a lagging reader's SegmentInputStream receives a SegmentTruncatedException and
 * when the reader tries to fetch the start offset of this segment the SegmentStore returns a NoSuchSegmentException.
 */
@Test
public void testTruncatedSegmentDeleted() throws Exception {
    AtomicLong clock = new AtomicLong();
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    SegmentWithRange rangedSegment = new SegmentWithRange(segment, 0, 1);
    // Setup mock.
    SegmentInputStreamFactory segInputStreamFactory = Mockito.mock(SegmentInputStreamFactory.class);
    SegmentMetadataClientFactory segmentMetadataClientFactory = Mockito.mock(SegmentMetadataClientFactory.class);
    SegmentMetadataClient metadataClient = Mockito.mock(SegmentMetadataClient.class);
    EventSegmentReader segmentInputStream = Mockito.mock(EventSegmentReader.class);
    Mockito.when(segmentMetadataClientFactory.createSegmentMetadataClient(any(Segment.class), any())).thenReturn(metadataClient);
    Mockito.when(segmentInputStream.getSegmentId()).thenReturn(segment);
    Mockito.when(segInputStreamFactory.createEventReaderForSegment(any(Segment.class), anyInt(), any(Semaphore.class), anyLong())).thenReturn(segmentInputStream);
    // Ensure segmentInputStream.read() returns SegmentTruncatedException.
    Mockito.when(segmentInputStream.isSegmentReady()).thenReturn(true);
    Mockito.when(segmentInputStream.read(anyLong())).thenThrow(SegmentTruncatedException.class);
    // Ensure SegmentInfo returns NoSuchSegmentException.
    Mockito.when(metadataClient.getSegmentInfo()).thenThrow(NoSuchSegmentException.class);
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segInputStreamFactory, segmentMetadataClientFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(rangedSegment, 0L)).thenReturn(Collections.emptyMap());
    InOrder inOrder = Mockito.inOrder(groupState, segmentInputStream);
    // Validate that TruncatedDataException is thrown.
    AssertExtensions.assertThrows(TruncatedDataException.class, () -> reader.readNextEvent(0));
    inOrder.verify(groupState).getCheckpoint();
    // Ensure this segment is closed.
    inOrder.verify(segmentInputStream, Mockito.times(1)).close();
    // Ensure groupstate is not updated before the checkpoint.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(rangedSegment);
    Mockito.when(groupState.getCheckpoint()).thenReturn("Foo").thenReturn(null);
    EventRead<byte[]> event = reader.readNextEvent(0);
    assertTrue(event.isCheckpoint());
    assertEquals("Foo", event.getCheckpointName());
    inOrder.verify(groupState).getCheckpoint();
    // Verify groupstate is not updated before the checkpoint, but is after.
    inOrder.verify(groupState, Mockito.times(0)).handleEndOfSegment(rangedSegment);
    event = reader.readNextEvent(0);
    assertFalse(event.isCheckpoint());
    assertNull(event.getEvent());
    // Now it is called.
    inOrder.verify(groupState).handleEndOfSegment(rangedSegment);
}

Example 12 with SegmentMetadataClient

use of io.pravega.client.segment.impl.SegmentMetadataClient in project pravega by pravega.

the class ClientFactoryImpl method createRevisionedStreamClient.

private <T> RevisionedStreamClient<T> createRevisionedStreamClient(Segment segment, Serializer<T> serializer, SynchronizerConfig config) {
    EventSegmentReader in = inFactory.createEventReaderForSegment(segment, config.getReadBufferSize());
    DelegationTokenProvider delegationTokenProvider = DelegationTokenProviderFactory.create(controller, segment, AccessOperation.READ_WRITE);
    ConditionalOutputStream cond = condFactory.createConditionalOutputStream(segment, delegationTokenProvider, config.getEventWriterConfig());
    SegmentMetadataClient meta = metaFactory.createSegmentMetadataClient(segment, delegationTokenProvider);
    return new RevisionedStreamClientImpl<>(segment, in, outFactory, cond, meta, serializer, config.getEventWriterConfig(), delegationTokenProvider);
}

Example 13 with SegmentMetadataClient

use of io.pravega.client.segment.impl.SegmentMetadataClient in project pravega by pravega.

the class RevisionedStreamClientTest method testRetryOnTimeout.

@Test
public void testRetryOnTimeout() throws ConnectionFailedException {
    String scope = "scope";
    String stream = "stream";
    Segment segment = new Segment(scope, stream, 0L);
    // Setup Environment
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    // Setup Mocks
    JavaSerializer<String> serializer = new JavaSerializer<>();
    @Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    @Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    // Setup client connection.
    ClientConnection c = mock(ClientConnection.class);
    connectionFactory.provideConnection(endpoint, c);
    // Create Scope and Stream.
    createScopeAndStream(scope, stream, controller);
    // Create mock ClientFactory.
    SegmentInputStreamFactory segInputFactory = new SegmentInputStreamFactoryImpl(controller, connectionFactory);
    SegmentOutputStreamFactory segOutputFactory = mock(SegmentOutputStreamFactory.class);
    ConditionalOutputStreamFactory condOutputFactory = new ConditionalOutputStreamFactoryImpl(controller, connectionFactory);
    SegmentMetadataClientFactory segMetaFactory = mock(SegmentMetadataClientFactory.class);
    SegmentMetadataClient segMetaClient = mock(SegmentMetadataClient.class);
    when(segMetaFactory.createSegmentMetadataClient(eq(segment), any(DelegationTokenProvider.class))).thenReturn(segMetaClient);
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, segInputFactory, segOutputFactory, condOutputFactory, segMetaFactory);
    RevisionedStreamClientImpl<String> client = spy((RevisionedStreamClientImpl<String>) clientFactory.createRevisionedStreamClient(stream, serializer, SynchronizerConfig.builder().build()));
    // Override the readTimeout value for RevisionedClient to 1 second.
    doReturn(1000L).when(client).getReadTimeout();
    // Setup the SegmentMetadataClient mock.
    doReturn(CompletableFuture.completedFuture(new SegmentInfo(segment, 0L, 30L, false, 1L))).when(segMetaClient).getSegmentInfo();
    // Get the iterator from Revisioned Stream Client.
    Iterator<Entry<Revision, String>> iterator = client.readFrom(new RevisionImpl(segment, 15, 1));
    // since we are trying to read @ offset 15 and the writeOffset is 30L a true is returned for hasNext().
    assertTrue(iterator.hasNext());
    // Setup mock to validate a retry.
    doNothing().doAnswer(i -> {
        WireCommands.ReadSegment request = i.getArgument(0);
        ReplyProcessor rp = connectionFactory.getProcessor(endpoint);
        WireCommands.Event event = new WireCommands.Event(Unpooled.wrappedBuffer(serializer.serialize("A")));
        ByteArrayOutputStream bout = new ByteArrayOutputStream();
        event.writeFields(new DataOutputStream(bout));
        ByteBuf eventData = Unpooled.wrappedBuffer(bout.toByteArray());
        // Invoke Reply processor to simulate a successful read.
        rp.process(new WireCommands.SegmentRead(request.getSegment(), 15L, true, true, eventData, request.getRequestId()));
        return null;
    }).when(c).send(any(WireCommands.ReadSegment.class));
    Entry<Revision, String> r = iterator.next();
    assertEquals("A", r.getValue());
    // Verify retries have been performed.
    verify(c, times(3)).send(any(WireCommands.ReadSegment.class));
}

Example 14 with SegmentMetadataClient

use of io.pravega.client.segment.impl.SegmentMetadataClient in project pravega by pravega.

the class EventStreamReaderTest method testDataTruncated.

@Test(timeout = 10000)
public void testDataTruncated() throws SegmentSealedException, ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, orderer, clock::get, ReaderConfig.builder().build(), createWatermarkReaders(), Mockito.mock(Controller.class));
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any())).thenReturn(ImmutableMap.of(new SegmentWithRange(segment, 0, 1), 0L)).thenReturn(Collections.emptyMap());
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    @Cleanup SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    @Cleanup SegmentMetadataClient metadataClient = segmentStreamFactory.createSegmentMetadataClient(segment, DelegationTokenProviderFactory.createWithEmptyToken());
    ByteBuffer buffer1 = writeInt(stream, 1);
    ByteBuffer buffer2 = writeInt(stream, 2);
    writeInt(stream, 3);
    long length = metadataClient.fetchCurrentSegmentLength().join();
    assertEquals(0, length % 3);
    EventRead<byte[]> event1 = reader.readNextEvent(0);
    assertEquals(buffer1, ByteBuffer.wrap(event1.getEvent()));
    metadataClient.truncateSegment(length / 3).join();
    assertEquals(buffer2, ByteBuffer.wrap(reader.readNextEvent(0).getEvent()));
    metadataClient.truncateSegment(length).join();
    ByteBuffer buffer4 = writeInt(stream, 4);
    assertThrows(TruncatedDataException.class, () -> reader.readNextEvent(0));
    assertEquals(buffer4, ByteBuffer.wrap(reader.readNextEvent(0).getEvent()));
    assertNull(reader.readNextEvent(0).getEvent());
    assertThrows(NoSuchEventException.class, () -> reader.fetchEvent(event1.getEventPointer()));
    reader.close();
}

Example 15 with SegmentMetadataClient

use of io.pravega.client.segment.impl.SegmentMetadataClient in project pravega by pravega.

the class EndToEndTruncationTest method testTruncationOffsets.

@Test(timeout = 7000)
public void testTruncationOffsets() throws InterruptedException, ExecutionException, TimeoutException, TruncatedDataException, ReinitializationRequiredException {
    String scope = "scope";
    String streamName = "testTruncationOffsets";
    String testString = "Hello world\n";
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, StreamConfiguration.builder().build()).join();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    Serializer<String> serializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    Future<Void> ack = producer.writeEvent(testString);
    ack.get(5, TimeUnit.SECONDS);
    SegmentMetadataClientFactory metadataClientFactory = new SegmentMetadataClientFactoryImpl(controller, clientFactory.getConnectionPool());
    @Cleanup ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
    Segment segment = new Segment(scope, streamName, 0);
    @Cleanup SegmentMetadataClient metadataClient = metadataClientFactory.createSegmentMetadataClient(segment, DelegationTokenProviderFactory.createWithEmptyToken());
    assertEquals(0, metadataClient.getSegmentInfo().join().getStartingOffset());
    long writeOffset = metadataClient.getSegmentInfo().join().getWriteOffset();
    assertEquals(writeOffset, metadataClient.fetchCurrentSegmentLength().join().longValue());
    assertTrue(metadataClient.getSegmentInfo().join().getWriteOffset() > testString.length());
    metadataClient.truncateSegment(writeOffset).join();
    assertEquals(writeOffset, metadataClient.getSegmentInfo().join().getStartingOffset());
    assertEquals(writeOffset, metadataClient.getSegmentInfo().join().getWriteOffset());
    assertEquals(writeOffset, metadataClient.fetchCurrentSegmentLength().join().longValue());
    ack = producer.writeEvent(testString);
    ack.get(5, TimeUnit.SECONDS);
    String group = "testTruncationOffsets-group";
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(new StreamImpl(scope, streamName)).build();
    readerGroupManager.createReaderGroup(group, groupConfig);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("reader", group, serializer, ReaderConfig.builder().build());
    AssertExtensions.assertThrows(TruncatedDataException.class, () -> reader.readNextEvent(2000));
    EventRead<String> event = reader.readNextEvent(2000);
    assertEquals(testString, event.getEvent());
    event = reader.readNextEvent(100);
    assertEquals(null, event.getEvent());
}