Examples with SocketConnectionFactoryImpl io.pravega.client.connection.impl.SocketConnectionFactoryImpl used on opensource projects

Example 11 with SocketConnectionFactoryImpl

use of io.pravega.client.connection.impl.SocketConnectionFactoryImpl in project pravega by pravega.

the class EndToEndAutoScaleUpWithTxnTest method main.

public static void main(String[] args) throws Exception {
    try {
        @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
        int port = Config.SERVICE_PORT;
        @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port);
        Controller controller = controllerWrapper.getController();
        controllerWrapper.getControllerService().createScope(NameUtils.INTERNAL_SCOPE_NAME, 0L).get();
        @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
        @Cleanup ClientFactoryImpl internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, connectionFactory);
        @Cleanup("shutdownNow") val executor = ExecutorServiceHelpers.newScheduledThreadPool(1, "test");
        @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
        serviceBuilder.initialize();
        StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
        TableStore tableStore = serviceBuilder.createTableStoreService();
        @Cleanup AutoScaleMonitor autoScaleMonitor = new AutoScaleMonitor(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).build());
        @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, false, "localhost", 12345, store, tableStore, autoScaleMonitor.getStatsRecorder(), autoScaleMonitor.getTableSegmentStatsRecorder(), null, null, null, true, serviceBuilder.getLowPriorityExecutor(), Config.TLS_PROTOCOL_VERSION.toArray(new String[Config.TLS_PROTOCOL_VERSION.size()]));
        server.startListening();
        controllerWrapper.awaitRunning();
        controllerWrapper.getControllerService().createScope("test", 0L).get();
        controller.createStream("test", "test", CONFIG).get();
        @Cleanup MockClientFactory clientFactory = new MockClientFactory("test", controller, internalCF.getConnectionPool());
        // Mocking pravega service by putting scale up and scale down requests for the stream
        EventWriterConfig writerConfig = EventWriterConfig.builder().transactionTimeoutTime(30000).build();
        TransactionalEventStreamWriter<String> test = clientFactory.createTransactionalEventWriter("writer", "test", new UTF8StringSerializer(), writerConfig);
        // region Successful commit tests
        Transaction<String> txn1 = test.beginTxn();
        txn1.writeEvent("1");
        txn1.flush();
        Map<Double, Double> map = new HashMap<>();
        map.put(0.0, 1.0 / 3.0);
        map.put(1.0 / 3.0, 2.0 / 3.0);
        map.put(2.0 / 3.0, 1.0);
        Stream stream = new StreamImpl("test", "test");
        controller.startScale(stream, Collections.singletonList(0L), map).get();
        Transaction<String> txn2 = test.beginTxn();
        txn2.writeEvent("2");
        txn2.flush();
        txn2.commit();
        txn1.commit();
        Thread.sleep(1000);
        @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl("test", controller, clientFactory);
        readerGroupManager.createReaderGroup("readergrp", ReaderGroupConfig.builder().stream("test/test").build());
        final EventStreamReader<String> reader = clientFactory.createReader("1", "readergrp", new JavaSerializer<>(), ReaderConfig.builder().build());
        String event1 = reader.readNextEvent(SECONDS.toMillis(60)).getEvent();
        String event2 = reader.readNextEvent(SECONDS.toMillis(60)).getEvent();
        assert event1.equals("1");
        assert event2.equals("2");
        final AtomicBoolean done = new AtomicBoolean(false);
        startWriter(test, done);
        Retry.withExpBackoff(10, 10, 100, 10000).retryingOn(NotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments("test", "test").thenAccept(streamSegments -> {
            if (streamSegments.getSegments().stream().anyMatch(x -> NameUtils.getEpoch(x.getSegmentId()) > 5)) {
                System.err.println("Success");
                log.info("Success");
                System.exit(0);
            } else {
                throw new NotDoneException();
            }
        }), executor).exceptionally(e -> {
            System.err.println("Failure");
            log.error("Failure");
            System.exit(1);
            return null;
        }).get();
    } catch (Throwable e) {
        System.err.print("Test failed with exception: " + e.getMessage());
        log.error("Test failed with exception: {}", e);
        System.exit(-1);
    }
    System.exit(0);
}

Example 12 with SocketConnectionFactoryImpl

use of io.pravega.client.connection.impl.SocketConnectionFactoryImpl in project pravega by pravega.

the class RetentionTest method retentionTest.

private CompletableFuture<Void> retentionTest(String streamName, boolean sizeBased) throws Exception {
    return CompletableFuture.runAsync(() -> {
        final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
        @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
        ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
        @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
        log.info("Invoking Writer test with Controller URI: {}", controllerURI);
        // create a writer
        @Cleanup EventStreamWriter<Serializable> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
        // write an event
        String writeEvent = "event";
        writer.writeEvent(writeEvent);
        if (sizeBased) {
            // since truncation always happens at an event boundary, for size based, we will write two events,
            // so that truncation can happen at the first event.
            writer.writeEvent(writeEvent);
        }
        writer.flush();
        log.debug("Writing event: {} ", writeEvent);
        // sleep for 5 mins -- retention frequency is set to 2 minutes. So in 5 minutes we should definitely have
        // 2 retention cycles, with a stream cut being computed in first cycle and truncation happening on the
        // previously computed streamcut in second cycle.
        // for time based retention, we wrote one event, which would get truncated.
        // for size based retention we wrote two events such that stream would retain at least 1 byte as prescribed by
        // the policy
        Exceptions.handleInterrupted(() -> Thread.sleep(5 * 60 * 1000));
        // create a reader
        ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
        String groupName = READER_GROUP + streamName;
        groupManager.createReaderGroup(groupName, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).build());
        EventStreamReader<String> reader = clientFactory.createReader(UUID.randomUUID().toString(), groupName, new JavaSerializer<>(), ReaderConfig.builder().build());
        if (sizeBased) {
            // we should read one write event back from the stream.
            String event = reader.readNextEvent(6000).getEvent();
            assertEquals(event, writeEvent);
        }
        // verify reader functionality is unaffected post truncation
        String event = "newEvent";
        writer.writeEvent(event);
        log.info("Writing event: {}", event);
        Assert.assertEquals(event, reader.readNextEvent(6000).getEvent());
        log.debug("The stream is already truncated.Simple retention test passed.");
    });
}

Example 13 with SocketConnectionFactoryImpl

use of io.pravega.client.connection.impl.SocketConnectionFactoryImpl in project pravega by pravega.

the class MultiReaderTxnWriterWithFailoverTest method setup.

@Before
public void setup() {
    // Get zk details to verify if controller, SSS are running
    Service zkService = Utils.createZookeeperService();
    List<URI> zkUris = zkService.getServiceDetails();
    log.debug("Zookeeper service details: {}", zkUris);
    // get the zk ip details and pass it to  host, controller
    URI zkUri = zkUris.get(0);
    // Verify controller is running.
    controllerInstance = Utils.createPravegaControllerService(zkUri);
    assertTrue(controllerInstance.isRunning());
    List<URI> conURIs = controllerInstance.getServiceDetails();
    log.info("Pravega Controller service instance details: {}", conURIs);
    // Fetch all the RPC endpoints and construct the client URIs.
    final List<String> uris = conURIs.stream().filter(ISGRPC).map(URI::getAuthority).collect(Collectors.toList());
    controllerURIDirect = URI.create((Utils.TLS_AND_AUTH_ENABLED ? TLS : TCP) + String.join(",", uris));
    log.info("Controller Service direct URI: {}", controllerURIDirect);
    // Verify segment store is running.
    segmentStoreInstance = Utils.createPravegaSegmentStoreService(zkUri, controllerURIDirect);
    assertTrue(segmentStoreInstance.isRunning());
    log.info("Pravega Segmentstore service instance details: {}", segmentStoreInstance.getServiceDetails());
    // executor service
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(NUM_READERS + NUM_WRITERS + 2, "MultiReaderTxnWriterWithFailoverTest-main");
    controllerExecutorService = ExecutorServiceHelpers.newScheduledThreadPool(2, "MultiReaderTxnWriterWithFailoverTest-controller");
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURIDirect);
    // get Controller Uri
    controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).maxBackoffMillis(5000).build(), controllerExecutorService);
    testState = new TestState(true);
    // read and write count variables
    streamManager = new StreamManagerImpl(clientConfig);
    createScopeAndStream(scope, STREAM_NAME, config, streamManager);
    log.info("Scope passed to client factory {}", scope);
    clientFactory = new ClientFactoryImpl(scope, controller, new SocketConnectionFactoryImpl(clientConfig));
    readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
}

Example 14 with SocketConnectionFactoryImpl

use of io.pravega.client.connection.impl.SocketConnectionFactoryImpl in project pravega by pravega.

the class OffsetTruncationTest method offsetTruncationTest.

/**
 * This test verifies that truncation works specifying an offset that applies to multiple segments. To this end,
 * the test first writes a set of events on a Stream (with multiple segments) and truncates it at a specified offset
 * (truncatedEvents). The tests asserts that readers first get a TruncatedDataException as they are attempting to
 * read a truncated segment, and then they only read the remaining events that have not been truncated.
 */
@Test
public void offsetTruncationTest() {
    final int totalEvents = 200;
    final int truncatedEvents = 50;
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    log.info("Invoking offsetTruncationTest test with Controller URI: {}", controllerURI);
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().stream(Stream.of(SCOPE, STREAM)).build());
    @Cleanup ReaderGroup readerGroup = groupManager.getReaderGroup(READER_GROUP);
    // Write events to the Stream.
    writeEvents(clientFactory, STREAM, totalEvents);
    // Instantiate readers to consume from Stream up to truncatedEvents.
    List<CompletableFuture<Integer>> futures = readEventFutures(clientFactory, READER_GROUP, PARALLELISM, truncatedEvents);
    Futures.allOf(futures).join();
    // Ensure that we have read all the events required before initiating the checkpoint.
    assertEquals("Number of events read is not the expected one.", (Integer) truncatedEvents, futures.stream().map(f -> Futures.getAndHandleExceptions(f, RuntimeException::new)).reduce(Integer::sum).get());
    // Perform truncation on stream segment.
    Checkpoint cp = readerGroup.initiateCheckpoint("truncationCheckpoint", executor).join();
    StreamCut streamCut = cp.asImpl().getPositions().values().iterator().next();
    StreamCut alternativeStreamCut = readerGroup.generateStreamCuts(executor).join().get(Stream.of(SCOPE, STREAM));
    assertEquals("StreamCuts for reader group differ depending on how they are generated.", streamCut, alternativeStreamCut);
    assertTrue(streamManager.truncateStream(SCOPE, STREAM, streamCut));
    // Just after the truncation, read events from the offset defined in truncate call onwards.
    final String newGroupName = READER_GROUP + "new";
    groupManager.createReaderGroup(newGroupName, ReaderGroupConfig.builder().stream(Stream.of(SCOPE, STREAM)).build());
    futures = readEventFutures(clientFactory, newGroupName, PARALLELISM);
    Futures.allOf(futures).join();
    assertEquals("Expected read events: ", totalEvents - truncatedEvents, (int) futures.stream().map(CompletableFuture::join).reduce(Integer::sum).get());
    log.debug("The stream has been successfully truncated at event {}. Offset truncation test passed.", truncatedEvents);
}

Example 15 with SocketConnectionFactoryImpl

use of io.pravega.client.connection.impl.SocketConnectionFactoryImpl in project pravega by pravega.

the class WatermarkingTest method watermarkTest.

@Test(timeout = 120000)
public void watermarkTest() throws Exception {
    Controller controller = PRAVEGA.getLocalController();
    String scope = "scope";
    String stream = "watermarkTest";
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(5)).build();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(PRAVEGA.getControllerURI()).build();
    @Cleanup StreamManager streamManager = StreamManager.create(clientConfig);
    streamManager.createScope(scope);
    streamManager.createStream(scope, stream, config);
    Stream streamObj = Stream.of(scope, stream);
    // create 2 writers
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
    JavaSerializer<Long> javaSerializer = new JavaSerializer<>();
    @Cleanup EventStreamWriter<Long> writer1 = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
    @Cleanup EventStreamWriter<Long> writer2 = clientFactory.createEventWriter(stream, javaSerializer, EventWriterConfig.builder().build());
    AtomicBoolean stopFlag = new AtomicBoolean(false);
    // write events
    CompletableFuture<Void> writer1Future = writeEvents(writer1, stopFlag);
    CompletableFuture<Void> writer2Future = writeEvents(writer2, stopFlag);
    // scale the stream several times so that we get complex positions
    scale(controller, streamObj, config);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    @Cleanup ClientFactoryImpl syncClientFactory = new ClientFactoryImpl(scope, new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor()), connectionFactory);
    String markStream = NameUtils.getMarkStreamForStream(stream);
    @Cleanup RevisionedStreamClient<Watermark> watermarkReader = syncClientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    LinkedBlockingQueue<Watermark> watermarks = new LinkedBlockingQueue<>();
    fetchWatermarks(watermarkReader, watermarks, stopFlag);
    AssertExtensions.assertEventuallyEquals(true, () -> watermarks.size() >= 2, 100000);
    stopFlag.set(true);
    writer1Future.join();
    writer2Future.join();
    // read events from the stream
    @Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(scope, controller, syncClientFactory);
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    Map<Segment, Long> positionMap1 = watermark1.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(scope, stream, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutFirst = new StreamCutImpl(streamObj, positionMap0);
    StreamCut streamCutSecond = new StreamCutImpl(streamObj, positionMap1);
    Map<Stream, StreamCut> firstMarkStreamCut = Collections.singletonMap(streamObj, streamCutFirst);
    Map<Stream, StreamCut> secondMarkStreamCut = Collections.singletonMap(streamObj, streamCutSecond);
    // read from stream cut of first watermark
    String readerGroup = "watermarkTest-group";
    readerGroupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().stream(streamObj).startingStreamCuts(firstMarkStreamCut).endingStreamCuts(secondMarkStreamCut).disableAutomaticCheckpoints().build());
    @Cleanup final EventStreamReader<Long> reader = clientFactory.createReader("myreader", readerGroup, javaSerializer, ReaderConfig.builder().build());
    EventRead<Long> event = reader.readNextEvent(10000L);
    TimeWindow currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    while (event.getEvent() != null && currentTimeWindow.getLowerTimeBound() == null && currentTimeWindow.getUpperTimeBound() == null) {
        event = reader.readNextEvent(10000L);
        currentTimeWindow = reader.getCurrentTimeWindow(streamObj);
    }
    assertNotNull(currentTimeWindow.getUpperTimeBound());
    // read all events and verify that all events are below the bounds
    while (event.getEvent() != null) {
        Long time = event.getEvent();
        log.info("timewindow = {} event = {}", currentTimeWindow, time);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || time >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || time <= currentTimeWindow.getUpperTimeBound());
        TimeWindow nextTimeWindow = reader.getCurrentTimeWindow(streamObj);
        assertTrue(currentTimeWindow.getLowerTimeBound() == null || nextTimeWindow.getLowerTimeBound() >= currentTimeWindow.getLowerTimeBound());
        assertTrue(currentTimeWindow.getUpperTimeBound() == null || nextTimeWindow.getUpperTimeBound() >= currentTimeWindow.getUpperTimeBound());
        currentTimeWindow = nextTimeWindow;
        event = reader.readNextEvent(10000L);
        if (event.isCheckpoint()) {
            event = reader.readNextEvent(10000L);
        }
    }
    assertNotNull(currentTimeWindow.getLowerTimeBound());
}