Examples with MockClientFactory io.pravega.client.stream.mock.MockClientFactory used on opensource projects

Example 26 with MockClientFactory

use of io.pravega.client.stream.mock.MockClientFactory in project pravega by pravega.

the class SynchronizerTest method testConsistency.

@Test(timeout = 20000)
public void testConsistency() {
    String streamName = "streamName";
    String scope = "scope";
    MockSegmentStreamFactory ioFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, ioFactory);
    createScopeAndStream(streamName, scope, clientFactory.getController());
    @Cleanup StateSynchronizer<RevisionedImpl> syncA = clientFactory.createStateSynchronizer(streamName, new JavaSerializer<>(), new JavaSerializer<>(), SynchronizerConfig.builder().build());
    @Cleanup StateSynchronizer<RevisionedImpl> syncB = clientFactory.createStateSynchronizer(streamName, new JavaSerializer<>(), new JavaSerializer<>(), SynchronizerConfig.builder().build());
    syncA.initialize(new RegularUpdate("Foo"));
    AtomicInteger callCount = new AtomicInteger(0);
    syncB.updateState((state, updates) -> {
        callCount.incrementAndGet();
        updates.add(new RegularUpdate("Bar"));
    });
    assertEquals(1, callCount.get());
    assertEquals("Foo", syncA.getState().value);
    syncA.updateState((state, updates) -> {
        callCount.incrementAndGet();
        updates.add(new RegularUpdate("Baz"));
    });
    assertEquals(3, callCount.get());
    assertEquals("Baz", syncA.getState().value);
    syncB.fetchUpdates();
    assertEquals("Baz", syncA.getState().value);
    syncB.updateState((state, updates) -> {
        callCount.incrementAndGet();
        updates.add(new RegularUpdate("Bat"));
    });
    assertEquals(4, callCount.get());
    assertEquals("Baz", syncA.getState().value);
    syncA.fetchUpdates();
    assertEquals("Bat", syncA.getState().value);
}

Example 27 with MockClientFactory

use of io.pravega.client.stream.mock.MockClientFactory in project pravega by pravega.

the class SynchronizerTest method testCompactionShrinksSet.

@Test(timeout = 20000)
public void testCompactionShrinksSet() {
    String streamName = "testCompactionShrinksSet";
    String scope = "scope";
    MockSegmentStreamFactory ioFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, ioFactory);
    createScopeAndStream(streamName, scope, clientFactory.getController());
    SetSynchronizer<String> set = SetSynchronizer.createNewSet(streamName, clientFactory);
    RevisionedStreamClient<byte[]> rsc = clientFactory.createRevisionedStreamClient(streamName, new ByteArraySerializer(), SynchronizerConfig.builder().build());
    set.add("Foo");
    assertNull(rsc.getMark());
    set.add("Bar");
    assertNull(rsc.getMark());
    set.clear();
    assertNotNull(rsc.getMark());
    Iterator<?> iter = rsc.readFrom(rsc.getMark());
    assertTrue(iter.hasNext());
    iter.next();
    assertFalse(iter.hasNext());
    set.add("Foo2");
    assertNotNull(rsc.getMark());
    assertEquals(1, set.getCurrentSize());
}

Example 28 with MockClientFactory

use of io.pravega.client.stream.mock.MockClientFactory in project pravega by pravega.

the class EventStreamReaderTest method testTimeWindow.

@Test
public void testTimeWindow() throws SegmentSealedException {
    String scope = "scope";
    String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    String groupName = "readerGroup";
    String readerGroupStream = NameUtils.getStreamForReaderGroup(groupName);
    String markStream = NameUtils.getMarkStreamForStream(streamName);
    // Create factories
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, segmentStreamFactory);
    MockController controller = (MockController) clientFactory.getController();
    @Cleanup InlineExecutor executor = new InlineExecutor();
    // Create streams
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).build());
    controller.createStream(scope, readerGroupStream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
    // Reader group state synchronizer
    ReaderGroupConfig config = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(stream).build();
    StateSynchronizer<ReaderGroupState> sync = clientFactory.createStateSynchronizer(readerGroupStream, new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
    // Watermark reader/writer
    @Cleanup RevisionedStreamClient<Watermark> markWriter = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    @Cleanup WatermarkReaderImpl markReader = new WatermarkReaderImpl(stream, markWriter, executor);
    // Initialize reader group state
    Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
    sync.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
    // Data segment writers
    Segment segment1 = new Segment(scope, streamName, 0);
    Segment segment2 = new Segment(scope, streamName, 1);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    // Write stream data
    writeInt(stream1, 1);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    // Write mark data
    val r1 = new SegmentWithRange(segment1, 0, 0.5).convert();
    val r2 = new SegmentWithRange(segment2, 0.5, 1).convert();
    markWriter.writeUnconditionally(new Watermark(0L, 99L, ImmutableMap.of(r1, 0L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(100L, 199L, ImmutableMap.of(r1, 12L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(200L, 299L, ImmutableMap.of(r1, 12L, r2, 12L)));
    markWriter.writeUnconditionally(new Watermark(300L, 399L, ImmutableMap.of(r1, 12L, r2, 24L)));
    markWriter.writeUnconditionally(new Watermark(400L, 499L, ImmutableMap.of(r1, 12L, r2, 36L)));
    // Create reader
    AtomicLong clock = new AtomicLong();
    ReaderGroupStateManager groupState = new ReaderGroupStateManager(scope, groupName, "reader1", sync, controller, clock::get);
    groupState.initializeReader(0);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, new Orderer(), clock::get, ReaderConfig.builder().build(), ImmutableMap.of(stream, markReader), Mockito.mock(Controller.class));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    EventRead<byte[]> event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    TimeWindow timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(199, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(1, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(299, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(200, timeWindow.getLowerTimeBound().longValue());
    assertEquals(399, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(300, timeWindow.getLowerTimeBound().longValue());
    assertEquals(499, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(null, event.getEvent());
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(400, timeWindow.getLowerTimeBound().longValue());
    assertEquals(null, timeWindow.getUpperTimeBound());
}

Example 29 with MockClientFactory

use of io.pravega.client.stream.mock.MockClientFactory in project pravega by pravega.

the class EndToEndTransactionOrderTest method setUp.

@Before
public void setUp() throws Exception {
    zkTestServer = new TestingServerStarter().start();
    int port = Config.SERVICE_PORT;
    controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), false, controllerPort, serviceHost, servicePort, Config.HOST_STORE_CONTAINER_COUNT);
    controller = controllerWrapper.getController();
    connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, connectionFactory);
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    TableStore tableStore = serviceBuilder.createTableStoreService();
    controllerWrapper.getControllerService().createScope(NameUtils.INTERNAL_SCOPE_NAME, 0L).get();
    autoScaleMonitor = new AutoScaleMonitor(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).build());
    server = new PravegaConnectionListener(false, false, "localhost", servicePort, store, tableStore, autoScaleMonitor.getStatsRecorder(), autoScaleMonitor.getTableSegmentStatsRecorder(), null, null, null, true, serviceBuilder.getLowPriorityExecutor(), SecurityConfigDefaults.TLS_PROTOCOL_VERSION);
    server.startListening();
    controllerWrapper.awaitRunning();
    controllerWrapper.getControllerService().createScope("test", 0L).get();
    controller.createStream("test", "test", config).get();
    clientFactory = new MockClientFactory("test", controller, internalCF.getConnectionPool());
    readerGroupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
    readerGroupManager.createReaderGroup("readergrp", ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(2000).groupRefreshTimeMillis(1000).stream("test/test").build());
    reader = clientFactory.createReader("1", "readergrp", new IntegerSerializer(), ReaderConfig.builder().build());
}

Example 30 with MockClientFactory

use of io.pravega.client.stream.mock.MockClientFactory in project pravega by pravega.

the class AutoCheckpointTest method testCheckpointsOccur.

@Test(timeout = 30000)
public void testCheckpointsOccur() throws ReinitializationRequiredException, DurableDataLogException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerName = "reader";
    String readerGroup = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world: ";
    String scope = "Scope1";
    @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, mock(TableStore.class), serviceBuilder.getLowPriorityExecutor());
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    @Cleanup MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(10000).stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, null);
    streamManager.createReaderGroup(readerGroup, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    populateEvents(streamName, testString, clientFactory, serializer);
    AtomicLong fakeClock = new AtomicLong(0);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
    int numRead = 0;
    int checkpointCount = 0;
    while (numRead < 100) {
        fakeClock.addAndGet(NANOS_PER_SECOND);
        EventRead<String> event = reader.readNextEvent(1000);
        if (event.isCheckpoint()) {
            checkpointCount++;
        } else {
            String message = event.getEvent();
            assertEquals(testString + numRead, message);
            numRead++;
        }
    }
    assertTrue("Count was " + checkpointCount, checkpointCount > 5);
    assertTrue("Count was " + checkpointCount, checkpointCount < 20);
}