Examples with InlineExecutor io.pravega.test.common.InlineExecutor used on opensource projects

Example 36 with InlineExecutor

use of io.pravega.test.common.InlineExecutor in project pravega by pravega.

the class ControllerImplLBTest method testDiscoverySuccess.

@Test
public void testDiscoverySuccess() throws Exception {
    final int serverPort1 = testRPCServer1.getPort();
    final int serverPort2 = testRPCServer2.getPort();
    // Use 2 servers to discover all the servers.
    @Cleanup("shutdown") InlineExecutor executor = new InlineExecutor();
    final ControllerImpl controllerClient = new ControllerImpl(ControllerImplConfig.builder().clientConfig(ClientConfig.builder().controllerURI(URI.create("pravega://localhost:" + serverPort1 + ",localhost:" + serverPort2)).build()).retryAttempts(1).build(), executor);
    final Set<PravegaNodeUri> uris = fetchFromServers(controllerClient, 3);
    // Verify we could reach all 3 controllers.
    Assert.assertEquals(3, uris.size());
}

Example 37 with InlineExecutor

use of io.pravega.test.common.InlineExecutor in project pravega by pravega.

the class EventStreamReaderTest method testTimeWindow.

@Test
public void testTimeWindow() throws SegmentSealedException {
    String scope = "scope";
    String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    String groupName = "readerGroup";
    String readerGroupStream = NameUtils.getStreamForReaderGroup(groupName);
    String markStream = NameUtils.getMarkStreamForStream(streamName);
    // Create factories
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, segmentStreamFactory);
    MockController controller = (MockController) clientFactory.getController();
    @Cleanup InlineExecutor executor = new InlineExecutor();
    // Create streams
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).build());
    controller.createStream(scope, readerGroupStream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
    // Reader group state synchronizer
    ReaderGroupConfig config = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(stream).build();
    StateSynchronizer<ReaderGroupState> sync = clientFactory.createStateSynchronizer(readerGroupStream, new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
    // Watermark reader/writer
    @Cleanup RevisionedStreamClient<Watermark> markWriter = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    @Cleanup WatermarkReaderImpl markReader = new WatermarkReaderImpl(stream, markWriter, executor);
    // Initialize reader group state
    Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
    sync.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
    // Data segment writers
    Segment segment1 = new Segment(scope, streamName, 0);
    Segment segment2 = new Segment(scope, streamName, 1);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    // Write stream data
    writeInt(stream1, 1);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    // Write mark data
    val r1 = new SegmentWithRange(segment1, 0, 0.5).convert();
    val r2 = new SegmentWithRange(segment2, 0.5, 1).convert();
    markWriter.writeUnconditionally(new Watermark(0L, 99L, ImmutableMap.of(r1, 0L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(100L, 199L, ImmutableMap.of(r1, 12L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(200L, 299L, ImmutableMap.of(r1, 12L, r2, 12L)));
    markWriter.writeUnconditionally(new Watermark(300L, 399L, ImmutableMap.of(r1, 12L, r2, 24L)));
    markWriter.writeUnconditionally(new Watermark(400L, 499L, ImmutableMap.of(r1, 12L, r2, 36L)));
    // Create reader
    AtomicLong clock = new AtomicLong();
    ReaderGroupStateManager groupState = new ReaderGroupStateManager(scope, groupName, "reader1", sync, controller, clock::get);
    groupState.initializeReader(0);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, new Orderer(), clock::get, ReaderConfig.builder().build(), ImmutableMap.of(stream, markReader), Mockito.mock(Controller.class));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    EventRead<byte[]> event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    TimeWindow timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(199, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(1, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(299, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(200, timeWindow.getLowerTimeBound().longValue());
    assertEquals(399, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(300, timeWindow.getLowerTimeBound().longValue());
    assertEquals(499, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(null, event.getEvent());
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(400, timeWindow.getLowerTimeBound().longValue());
    assertEquals(null, timeWindow.getUpperTimeBound());
}

Example 38 with InlineExecutor

use of io.pravega.test.common.InlineExecutor in project pravega by pravega.

the class SegmentMetadataClientTest method testExceptionOnSend.

@Test(timeout = 10000)
public void testExceptionOnSend() throws ConnectionFailedException {
    Segment segment = new Segment("scope", "testRetry", 4);
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", 0);
    @Cleanup("shutdown") InlineExecutor executor = new InlineExecutor();
    @Cleanup ConnectionPool cf = Mockito.mock(ConnectionPool.class);
    Mockito.when(cf.getInternalExecutor()).thenReturn(executor);
    @Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), cf, true);
    ClientConnection connection1 = mock(ClientConnection.class);
    ClientConnection connection2 = mock(ClientConnection.class);
    AtomicReference<ReplyProcessor> processor = new AtomicReference<>();
    Mockito.doAnswer(invocation -> {
        final CompletableFuture<ClientConnection> future = invocation.getArgument(3);
        future.completeExceptionally(new ConnectionFailedException(new RuntimeException("Mock error")));
        return null;
    }).doAnswer(invocation -> {
        final CompletableFuture<ClientConnection> future = invocation.getArgument(3);
        future.complete(connection1);
        return null;
    }).doAnswer(invocation -> {
        final CompletableFuture<ClientConnection> future = invocation.getArgument(3);
        processor.set(invocation.getArgument(2));
        future.complete(connection2);
        return null;
    }).when(cf).getClientConnection(Mockito.any(Flow.class), Mockito.eq(endpoint), Mockito.any(ReplyProcessor.class), Mockito.<CompletableFuture<ClientConnection>>any());
    final List<Long> requestIds = new ArrayList<>();
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            WireCommands.GetStreamSegmentInfo request = invocation.getArgument(0);
            requestIds.add(request.getRequestId());
            if (requestIds.size() == 1) {
                throw new ConnectionFailedException();
            } else {
                processor.get().process(new StreamSegmentInfo(request.getRequestId(), segment.getScopedName(), true, false, false, 0, 123, 121));
            }
            return null;
        }
    }).when(connection1).send(any(WireCommands.GetStreamSegmentInfo.class));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            WireCommands.GetStreamSegmentInfo request = invocation.getArgument(0);
            requestIds.add(request.getRequestId());
            processor.get().process(new StreamSegmentInfo(request.getRequestId(), segment.getScopedName(), true, false, false, 0, 123, 121));
            return null;
        }
    }).when(connection2).send(any(WireCommands.GetStreamSegmentInfo.class));
    @Cleanup SegmentMetadataClientImpl client = new SegmentMetadataClientImpl(segment, controller, cf, "");
    InOrder order = Mockito.inOrder(connection1, connection2, cf);
    long length = client.fetchCurrentSegmentLength().join();
    order.verify(cf, Mockito.times(2)).getClientConnection(Mockito.any(Flow.class), Mockito.eq(endpoint), Mockito.any(), Mockito.<CompletableFuture<ClientConnection>>any());
    order.verify(connection1).send(Mockito.eq(new WireCommands.GetStreamSegmentInfo(requestIds.get(0), segment.getScopedName(), "")));
    order.verify(connection1).close();
    order.verify(cf).getClientConnection(Mockito.any(Flow.class), Mockito.eq(endpoint), Mockito.any(), Mockito.<CompletableFuture<ClientConnection>>any());
    order.verify(connection2).send(Mockito.eq(new WireCommands.GetStreamSegmentInfo(requestIds.get(1), segment.getScopedName(), "")));
    order.verifyNoMoreInteractions();
    assertEquals(123, length);
}

Example 39 with InlineExecutor

use of io.pravega.test.common.InlineExecutor in project pravega by pravega.

the class EndToEndCBRTest method testReaderGroupAutoRetention.

@Test(timeout = 60000)
public void testReaderGroupAutoRetention() throws Exception {
    String scope = "test";
    String streamName = "testReaderGroupAutoRetention";
    String groupName = "testReaderGroupAutoRetention-group";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).retentionPolicy(RetentionPolicy.bySizeBytes(10, Long.MAX_VALUE)).build();
    LocalController controller = (LocalController) PRAVEGA.getLocalController();
    controller.createScope(scope).get();
    controller.createStream(scope, streamName, config).get();
    Stream stream = Stream.of(scope, streamName);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    // write events
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    writer.writeEvent("1", "e1").join();
    writer.writeEvent("2", "e2").join();
    // Create a ReaderGroup
    @Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(scope, controller, clientFactory);
    groupManager.createReaderGroup(groupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().retentionType(ReaderGroupConfig.StreamDataRetention.AUTOMATIC_RELEASE_AT_LAST_CHECKPOINT).stream(stream).build());
    // Create a Reader
    AtomicLong clock = new AtomicLong();
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("reader1", groupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    EventRead<String> read = reader.readNextEvent(60000);
    assertEquals("e1", read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    @Cleanup("shutdown") final InlineExecutor backgroundExecutor = new InlineExecutor();
    ReaderGroup readerGroup = groupManager.getReaderGroup(groupName);
    CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("Checkpoint", backgroundExecutor);
    assertFalse(checkpoint.isDone());
    read = reader.readNextEvent(60000);
    assertTrue(read.isCheckpoint());
    assertEquals("Checkpoint", read.getCheckpointName());
    assertNull(read.getEvent());
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    read = reader.readNextEvent(60000);
    assertEquals("e2", read.getEvent());
    Checkpoint cpResult = checkpoint.get(5, TimeUnit.SECONDS);
    assertTrue(checkpoint.isDone());
    assertEquals("Checkpoint", cpResult.getName());
    read = reader.readNextEvent(100);
    assertNull(read.getEvent());
    assertFalse(read.isCheckpoint());
    AssertExtensions.assertEventuallyEquals(true, () -> controller.getSegmentsAtTime(new StreamImpl(scope, streamName), 0L).join().values().stream().anyMatch(off -> off > 0), 30 * 1000L);
    String group2 = groupName + "2";
    groupManager.createReaderGroup(group2, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(NameUtils.getScopedStreamName(scope, streamName)).build());
    EventStreamReader<String> reader2 = clientFactory.createReader("reader2", group2, serializer, ReaderConfig.builder().build());
    EventRead<String> eventRead2 = reader2.readNextEvent(10000);
    assertEquals("e2", eventRead2.getEvent());
}

Example 40 with InlineExecutor

use of io.pravega.test.common.InlineExecutor in project pravega by pravega.

the class EndToEndReaderGroupTest method testGenerateStreamCuts.

@Test(timeout = 30000)
public void testGenerateStreamCuts() throws Exception {
    String streamName = "testGenerateStreamCuts";
    final Stream stream = Stream.of(SCOPE, streamName);
    final String group = "testGenerateStreamCuts-group";
    createScope(SCOPE);
    createStream(SCOPE, streamName, ScalingPolicy.fixed(1));
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    // Prep the stream with data.
    // 1.Write events with event size of 30
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(1)).join();
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(2)).join();
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(3)).join();
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(4)).join();
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, PRAVEGA.getControllerURI());
    groupManager.createReaderGroup(group, ReaderGroupConfig.builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(1000).stream(stream).build());
    ReaderGroup readerGroup = groupManager.getReaderGroup(group);
    // Create a reader
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader("readerId", group, serializer, ReaderConfig.builder().build());
    readAndVerify(reader, 1);
    @Cleanup("shutdown") InlineExecutor backgroundExecutor = new InlineExecutor();
    CompletableFuture<Map<Stream, StreamCut>> sc = readerGroup.generateStreamCuts(backgroundExecutor);
    // The reader group state will be updated after 1 second.
    TimeUnit.SECONDS.sleep(1);
    EventRead<String> data = reader.readNextEvent(15000);
    // wait until the streamCut is obtained.
    assertTrue(Futures.await(sc));
    // expected segment 0 offset is 30L.
    Map<Segment, Long> expectedOffsetMap = ImmutableMap.of(getSegment(streamName, 0, 0), 30L);
    Map<Stream, StreamCut> scMap = sc.join();
    assertEquals("StreamCut for a single stream expected", 1, scMap.size());
    assertEquals("StreamCut pointing ot offset 30L expected", new StreamCutImpl(stream, expectedOffsetMap), scMap.get(stream));
}