Examples with ControllerImpl io.pravega.client.control.impl.ControllerImpl used on opensource projects

Example 21 with ControllerImpl

use of io.pravega.client.control.impl.ControllerImpl in project pravega by pravega.

the class ByteClientTest method createClientFactory.

ByteStreamClientFactory createClientFactory(String scope) {
    ClientConfig config = ClientConfig.builder().build();
    ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(config);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(Utils.buildClientConfig(controllerURI)).build(), connectionFactory.getInternalExecutor());
    ConnectionPool pool = new ConnectionPoolImpl(config, connectionFactory);
    val inputStreamFactory = new SegmentInputStreamFactoryImpl(controller, pool);
    val outputStreamFactory = new SegmentOutputStreamFactoryImpl(controller, pool);
    val metaStreamFactory = new SegmentMetadataClientFactoryImpl(controller, pool);
    return new ByteStreamClientImpl(scope, controller, pool, inputStreamFactory, outputStreamFactory, metaStreamFactory);
}

Example 22 with ControllerImpl

use of io.pravega.client.control.impl.ControllerImpl in project pravega by pravega.

the class BatchClientSimpleTest method batchClientSimpleTest.

/**
 * This test verifies the basic functionality of {@link BatchClientFactory}, including stream metadata checks, segment
 * counts, parallel segment reads and reads with offsets using stream cuts.
 */
@Test
@SuppressWarnings("deprecation")
public void batchClientSimpleTest() {
    final int totalEvents = RG_PARALLELISM * 100;
    final int offsetEvents = RG_PARALLELISM * 20;
    final int batchIterations = 4;
    final Stream stream = Stream.of(SCOPE, STREAM);
    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(clientConfig);
    ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), connectionFactory.getInternalExecutor());
    @Cleanup ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
    @Cleanup BatchClientFactory batchClient = BatchClientFactory.withScope(SCOPE, clientConfig);
    log.info("Invoking batchClientSimpleTest test with Controller URI: {}", controllerURI);
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, clientConfig);
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(SCOPE + "/" + STREAM).build());
    ReaderGroup readerGroup = groupManager.getReaderGroup(READER_GROUP);
    log.info("Writing events to stream");
    // Write events to the Stream.
    writeEvents(clientFactory, STREAM, totalEvents);
    // Instantiate readers to consume from Stream up to truncatedEvents.
    List<CompletableFuture<Integer>> futures = readEventFutures(clientFactory, READER_GROUP, RG_PARALLELISM, offsetEvents);
    Futures.allOf(futures).join();
    // Create a stream cut on the specified offset position.
    Checkpoint cp = readerGroup.initiateCheckpoint("batchClientCheckpoint", executor).join();
    StreamCut streamCut = cp.asImpl().getPositions().values().iterator().next();
    // Instantiate the batch client and assert it provides correct stream info.
    log.debug("Creating batch client.");
    StreamInfo streamInfo = streamManager.getStreamInfo(SCOPE, stream.getStreamName());
    log.debug("Validating stream metadata fields.");
    assertEquals("Expected Stream name: ", STREAM, streamInfo.getStreamName());
    assertEquals("Expected Scope name: ", SCOPE, streamInfo.getScope());
    // Test that we can read events from parallel segments from an offset onwards.
    log.debug("Reading events from stream cut onwards in parallel.");
    List<SegmentRange> ranges = Lists.newArrayList(batchClient.getSegments(stream, streamCut, StreamCut.UNBOUNDED).getIterator());
    assertEquals("Expected events read: ", totalEvents - offsetEvents, readFromRanges(ranges, batchClient));
    // Emulate the behavior of Hadoop client: i) Get tail of Stream, ii) Read from current point until tail, iii) repeat.
    log.debug("Reading in batch iterations.");
    StreamCut currentTailStreamCut = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getTailStreamCut();
    int readEvents = 0;
    for (int i = 0; i < batchIterations; i++) {
        writeEvents(clientFactory, STREAM, totalEvents);
        // Read all the existing events in parallel segments from the previous tail to the current one.
        ranges = Lists.newArrayList(batchClient.getSegments(stream, currentTailStreamCut, StreamCut.UNBOUNDED).getIterator());
        assertEquals("Expected number of segments: ", RG_PARALLELISM, ranges.size());
        readEvents += readFromRanges(ranges, batchClient);
        log.debug("Events read in parallel so far: {}.", readEvents);
        currentTailStreamCut = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getTailStreamCut();
    }
    assertEquals("Expected events read: .", totalEvents * batchIterations, readEvents);
    // Truncate the stream in first place.
    log.debug("Truncating stream at event {}.", offsetEvents);
    assertTrue(controller.truncateStream(SCOPE, STREAM, streamCut).join());
    // Test the batch client when we select to start reading a Stream from a truncation point.
    StreamCut initialPosition = streamManager.getStreamInfo(SCOPE, stream.getStreamName()).getHeadStreamCut();
    List<SegmentRange> newRanges = Lists.newArrayList(batchClient.getSegments(stream, initialPosition, StreamCut.UNBOUNDED).getIterator());
    assertEquals("Expected events read: ", (totalEvents - offsetEvents) + totalEvents * batchIterations, readFromRanges(newRanges, batchClient));
    log.debug("Events correctly read from Stream: simple batch client test passed.");
}

Example 23 with ControllerImpl

use of io.pravega.client.control.impl.ControllerImpl in project pravega by pravega.

the class ControllerFailoverTest method failoverTest.

@Test
public void failoverTest() throws InterruptedException, ExecutionException {
    String scope = "testFailoverScope" + RandomStringUtils.randomAlphabetic(5);
    String stream = "testFailoverStream" + RandomStringUtils.randomAlphabetic(5);
    int initialSegments = 1;
    List<Long> segmentsToSeal = Collections.singletonList(0L);
    Map<Double, Double> newRangesToCreate = new HashMap<>();
    newRangesToCreate.put(0.0, 1.0);
    ClientConfig clientConfig = Utils.buildClientConfig(controllerURIDirect);
    // Connect with first controller instance.
    final Controller controller1 = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConfig).build(), executorService);
    // Create scope, stream, and a transaction with high timeout value.
    controller1.createScope(scope).join();
    log.info("Scope {} created successfully", scope);
    createStream(controller1, scope, stream, ScalingPolicy.fixed(initialSegments));
    log.info("Stream {}/{} created successfully", scope, stream);
    long txnCreationTimestamp = System.nanoTime();
    StreamImpl stream1 = new StreamImpl(scope, stream);
    // Initiate scale operation. It will block until ongoing transaction is complete.
    controller1.startScale(stream1, segmentsToSeal, newRangesToCreate).join();
    // Now stop the controller instance executing scale operation.
    Futures.getAndHandleExceptions(controllerService.scaleService(0), ExecutionException::new);
    log.info("Successfully stopped one instance of controller service");
    // restart controller service
    Futures.getAndHandleExceptions(controllerService.scaleService(1), ExecutionException::new);
    log.info("Successfully stopped one instance of controller service");
    List<URI> controllerUris = controllerService.getServiceDetails();
    // Fetch all the RPC endpoints and construct the client URIs.
    final List<String> uris = controllerUris.stream().filter(ISGRPC).map(URI::getAuthority).collect(Collectors.toList());
    controllerURIDirect = URI.create((Utils.TLS_AND_AUTH_ENABLED ? TLS : TCP) + String.join(",", uris));
    log.info("Controller Service direct URI: {}", controllerURIDirect);
    ClientConfig clientConf = Utils.buildClientConfig(controllerURIDirect);
    // Connect to another controller instance.
    @Cleanup final Controller controller2 = new ControllerImpl(ControllerImplConfig.builder().clientConfig(clientConf).build(), executorService);
    // Note: if scale does not complete within desired time, test will timeout.
    boolean scaleStatus = controller2.checkScaleStatus(stream1, 0).join();
    while (!scaleStatus) {
        scaleStatus = controller2.checkScaleStatus(stream1, 0).join();
        Thread.sleep(30000);
    }
    segmentsToSeal = Collections.singletonList(NameUtils.computeSegmentId(1, 1));
    newRangesToCreate = new HashMap<>();
    newRangesToCreate.put(0.0, 0.5);
    newRangesToCreate.put(0.5, 1.0);
    controller2.scaleStream(stream1, segmentsToSeal, newRangesToCreate, executorService).getFuture().join();
    log.info("Checking whether scale operation succeeded by fetching current segments");
    StreamSegments streamSegments = controller2.getCurrentSegments(scope, stream).join();
    log.info("Current segment count= {}", streamSegments.getSegments().size());
    Assert.assertEquals(2, streamSegments.getSegments().size());
}

Example 24 with ControllerImpl

use of io.pravega.client.control.impl.ControllerImpl in project pravega by pravega.

the class LargeEventTest method largeEventSimpleTest.

/**
 * Invoke the largeEventSimpleTest, ensure we are able to produce  events.
 * The test fails incase of exceptions while writing to the stream.
 */
@Test
public void largeEventSimpleTest() {
    Service conService = Utils.createPravegaControllerService(null);
    List<URI> ctlURIs = conService.getServiceDetails();
    URI controllerUri = ctlURIs.get(0);
    log.info("Invoking create stream with Controller URI: {}", controllerUri);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(Utils.buildClientConfig(controllerUri));
    @Cleanup ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(Utils.buildClientConfig(controllerUri)).build(), connectionFactory.getInternalExecutor());
    assertTrue(controller.createScope(STREAM_SCOPE).join());
    assertTrue(controller.createStream(STREAM_SCOPE, STREAM_NAME, config).join());
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(STREAM_SCOPE, Utils.buildClientConfig(controllerUri));
    log.info("Invoking Writer test with Controller URI: {}", controllerUri);
    @Cleanup EventStreamWriter<ByteBuffer> writer = clientFactory.createEventWriter(STREAM_NAME, new ByteBufferSerializer(), EventWriterConfig.builder().build());
    byte[] payload = new byte[Serializer.MAX_EVENT_SIZE];
    for (int i = 0; i < NUM_EVENTS; i++) {
        log.debug("Producing event: {} ", i);
        // any exceptions while writing the event will fail the test.
        writer.writeEvent("", ByteBuffer.wrap(payload));
        writer.flush();
    }
    log.info("Invoking Reader test.");
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(STREAM_SCOPE, Utils.buildClientConfig(controllerUri));
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().stream(Stream.of(STREAM_SCOPE, STREAM_NAME)).build());
    @Cleanup EventStreamReader<ByteBuffer> reader = clientFactory.createReader(UUID.randomUUID().toString(), READER_GROUP, new ByteBufferSerializer(), ReaderConfig.builder().build());
    int readCount = 0;
    EventRead<ByteBuffer> event = null;
    do {
        event = reader.readNextEvent(10_000);
        log.debug("Read event: {}.", event.getEvent());
        if (event.getEvent() != null) {
            readCount++;
        }
    // try reading until all the written events are read, else the test will timeout.
    } while ((event.getEvent() != null || event.isCheckpoint()) && readCount < NUM_EVENTS);
    assertEquals("Read count should be equal to write count", NUM_EVENTS, readCount);
}

Example 25 with ControllerImpl

use of io.pravega.client.control.impl.ControllerImpl in project pravega by pravega.

the class MetadataScalabilityLargeScalesTest method largeNumScalesScalability.

@Test
public void largeNumScalesScalability() {
    testState = new TestState(false);
    ControllerImpl controller = getController();
    List<List<Segment>> listOfEpochs = scale(controller);
    truncation(controller, listOfEpochs);
    sealAndDeleteStream(controller);
}