Examples with EventStreamClientFactory io.pravega.client.EventStreamClientFactory used on opensource projects

Example 41 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class AutoScaleProcessorTest method writerCreationTest.

@Test(timeout = 10000)
public void writerCreationTest() throws Exception {
    EventStreamClientFactory clientFactory = mock(EventStreamClientFactory.class);
    CompletableFuture<Void> createWriterLatch = new CompletableFuture<>();
    doAnswer(x -> {
        createWriterLatch.complete(null);
        throw new RuntimeException();
    }).when(clientFactory).createEventWriter(any(), any(), any());
    TestAutoScaleProcessor failingWriterProcessor = new TestAutoScaleProcessor(AutoScalerConfig.builder().with(AutoScalerConfig.CONTROLLER_URI, "tcp://localhost:9090").build(), clientFactory, executorService());
    String segmentStreamName = "scope/myStreamSegment/0.#epoch.0";
    failingWriterProcessor.notifyCreated(segmentStreamName);
    assertFalse(failingWriterProcessor.isInitializeStarted());
    AtomicReference<EventStreamWriter<AutoScaleEvent>> w = new AtomicReference<>();
    AssertExtensions.assertThrows("Bootstrap should not be initiated until isInitializeStarted is true", () -> failingWriterProcessor.bootstrapOnce(clientFactory, w), e -> Exceptions.unwrap(e) instanceof RuntimeException);
    // report but since the cooldown time hasnt elapsed, no scale event should be attempted. So no writer should be initialized yet.
    failingWriterProcessor.report(segmentStreamName, 1, 0L, 10.0, 10.0, 10.0, 10.0);
    assertFalse(failingWriterProcessor.isInitializeStarted());
    failingWriterProcessor.setTimeMillis(20 * 60000L);
    failingWriterProcessor.report(segmentStreamName, 1, 0L, 10.0, 10.0, 10.0, 10.0);
    // the above should initiate the bootstrap.
    assertTrue(failingWriterProcessor.isInitializeStarted());
    // since we are throwing on writer creation, wait until the writer is invoked once at least
    createWriterLatch.join();
    // now close the processor. The writer future should get cancelled.
    failingWriterProcessor.close();
    assertTrue(failingWriterProcessor.getWriterFuture().isCancelled());
    // create new processor and let the writer get created
    TestAutoScaleProcessor processor = new TestAutoScaleProcessor(AutoScalerConfig.builder().with(AutoScalerConfig.CONTROLLER_URI, "tcp://localhost:9090").build(), clientFactory, executorService());
    LinkedBlockingQueue<AutoScaleEvent> queue = new LinkedBlockingQueue<>();
    EventStreamWriter<AutoScaleEvent> writerMock = createWriter(queue::add);
    doAnswer(x -> writerMock).when(clientFactory).createEventWriter(any(), any(), any());
    processor.notifyCreated(segmentStreamName);
    // report a low rate to trigger a scale down
    processor.setTimeMillis(21 * 60000L);
    processor.report(segmentStreamName, 10, 0L, 1.0, 1.0, 1.0, 1.0);
    assertTrue(processor.isInitializeStarted());
    AssertExtensions.assertEventuallyEquals(writerMock, () -> processor.getWriterFuture().join(), 10000L);
    AutoScaleEvent event = queue.take();
    assertEquals(event.getDirection(), AutoScaleEvent.DOWN);
    processor.close();
    // create third writer, this time supply the writer directly
    EventStreamWriter<AutoScaleEvent> writer = spy(createWriter(e -> {
    }));
    // verify that when writer is set, we are able to get the processor initialized
    TestAutoScaleProcessor processor2 = new TestAutoScaleProcessor(writer, AutoScalerConfig.builder().with(AutoScalerConfig.CONTROLLER_URI, "tcp://localhost:9090").build(), executorService());
    processor2.notifyCreated(segmentStreamName);
    assertFalse(processor2.isInitializeStarted());
    processor2.setTimeMillis(20 * 60000L);
    processor2.report(segmentStreamName, 1, 0L, 10.0, 10.0, 10.0, 10.0);
    // the above should create a writer future.
    assertTrue(processor2.isInitializeStarted());
    assertTrue(Futures.isSuccessful(processor2.getWriterFuture()));
    processor2.close();
    verify(writer, times(1)).close();
}

Example 42 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class AutoScaleProcessorTest method testSteadyStateExpiry.

@Test
public void testSteadyStateExpiry() {
    HashMap<String, Pair<Long, Long>> map = new HashMap<>();
    HashMap<String, Long> lastAccessedTime = new HashMap<>();
    List<String> evicted = new ArrayList<>();
    @SuppressWarnings("unchecked") SimpleCache<String, Pair<Long, Long>> simpleCache = mock(SimpleCache.class);
    AtomicLong clock = new AtomicLong(0L);
    Function<Void, Void> cleanup = m -> {
        for (Map.Entry<String, Long> e : lastAccessedTime.entrySet()) {
            if (e.getValue() < clock.get()) {
                lastAccessedTime.remove(e.getKey());
                map.remove(e.getKey());
                evicted.add(e.getKey());
            }
        }
        // remove all that should have expired.
        return null;
    };
    doAnswer(x -> {
        cleanup.apply(null);
        return map.get(x.getArgument(0));
    }).when(simpleCache).get(anyString());
    doAnswer(x -> {
        cleanup.apply(null);
        map.put(x.getArgument(0), x.getArgument(1));
        return map.get(x.getArgument(0));
    }).when(simpleCache).put(anyString(), any());
    doAnswer(x -> cleanup.apply(null)).when(simpleCache).cleanUp();
    AutoScalerConfig config = AutoScalerConfig.builder().with(AutoScalerConfig.CONTROLLER_URI, "tcp://localhost:9090").with(AutoScalerConfig.TLS_ENABLED, false).build();
    ClientConfig objectUnderTest = AutoScaleProcessor.prepareClientConfig(config);
    @Cleanup EventStreamClientFactory eventStreamClientFactory = EventStreamClientFactory.withScope(SCOPE, objectUnderTest);
    @Cleanup TestAutoScaleProcessor monitor = new TestAutoScaleProcessor(AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).with(AutoScalerConfig.AUTH_ENABLED, authEnabled).with(AutoScalerConfig.CACHE_CLEANUP_IN_SECONDS, 150).with(AutoScalerConfig.CACHE_EXPIRY_IN_SECONDS, 60).build(), eventStreamClientFactory, executorService(), simpleCache);
    String streamSegmentName1 = NameUtils.getQualifiedStreamSegmentName(SCOPE, STREAM1, 0L);
    monitor.setTimeMillis(0L);
    clock.set(0L);
    monitor.notifyCreated(streamSegmentName1);
    monitor.put(streamSegmentName1, new ImmutablePair<>(5L, 5L));
    monitor.setTimeMillis(30 * 1000L);
    clock.set(30L);
    monitor.report(streamSegmentName1, 10L, 0L, 10D, 10D, 10D, 10D);
    monitor.setTimeMillis(80 * 1000L);
    clock.set(80L);
    simpleCache.cleanUp();
    assertNotNull(monitor.get(streamSegmentName1));
    assertNotNull(simpleCache.get(streamSegmentName1));
    assertTrue(evicted.isEmpty());
    AssertExtensions.assertThrows("NPE should be thrown", () -> new AutoScaleProcessor(null, config, executorService()), e -> e instanceof NullPointerException);
    AssertExtensions.assertThrows("NPE should be thrown", () -> new AutoScaleProcessor(null, eventStreamClientFactory, executorService()), e -> e instanceof NullPointerException);
    AssertExtensions.assertThrows("NPE should be thrown", () -> new AutoScaleProcessor(AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).with(AutoScalerConfig.AUTH_ENABLED, authEnabled).with(AutoScalerConfig.CACHE_CLEANUP_IN_SECONDS, 150).with(AutoScalerConfig.CACHE_EXPIRY_IN_SECONDS, 60).build(), eventStreamClientFactory, null), e -> e instanceof NullPointerException);
    AssertExtensions.assertThrows("NPE should be thrown", () -> new AutoScaleProcessor(AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).with(AutoScalerConfig.AUTH_ENABLED, authEnabled).with(AutoScalerConfig.CACHE_CLEANUP_IN_SECONDS, 150).with(AutoScalerConfig.CACHE_EXPIRY_IN_SECONDS, 60).build(), eventStreamClientFactory, null, simpleCache), e -> e instanceof NullPointerException);
    AssertExtensions.assertThrows("NPE should be thrown", () -> new AutoScaleProcessor(null, eventStreamClientFactory, executorService(), simpleCache), e -> e instanceof NullPointerException);
    monitor.notifySealed(streamSegmentName1);
}

Example 43 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class PravegaTest method simpleTest.

/**
 * Invoke the simpleTest, ensure we are able to produce  events.
 * The test fails incase of exceptions while writing to the stream.
 */
@Test
public void simpleTest() {
    Service conService = Utils.createPravegaControllerService(null);
    List<URI> ctlURIs = conService.getServiceDetails();
    URI controllerUri = ctlURIs.get(0);
    log.info("Invoking create stream with Controller URI: {}", controllerUri);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(Utils.buildClientConfig(controllerUri));
    @Cleanup ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(Utils.buildClientConfig(controllerUri)).build(), connectionFactory.getInternalExecutor());
    assertTrue(controller.createScope(STREAM_SCOPE).join());
    assertTrue(controller.createStream(STREAM_SCOPE, STREAM_NAME, config).join());
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(STREAM_SCOPE, Utils.buildClientConfig(controllerUri));
    log.info("Invoking Writer test with Controller URI: {}", controllerUri);
    @Cleanup EventStreamWriter<Serializable> writer = clientFactory.createEventWriter(STREAM_NAME, new JavaSerializer<>(), EventWriterConfig.builder().build());
    for (int i = 0; i < NUM_EVENTS; i++) {
        String event = "Publish " + i + "\n";
        log.debug("Producing event: {} ", event);
        // any exceptions while writing the event will fail the test.
        writer.writeEvent("", event);
        writer.flush();
    }
    log.info("Invoking Reader test.");
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(STREAM_SCOPE, Utils.buildClientConfig(controllerUri));
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().stream(Stream.of(STREAM_SCOPE, STREAM_NAME)).build());
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(UUID.randomUUID().toString(), READER_GROUP, new JavaSerializer<>(), ReaderConfig.builder().build());
    int readCount = 0;
    EventRead<String> event = null;
    do {
        event = reader.readNextEvent(10_000);
        log.debug("Read event: {}.", event.getEvent());
        if (event.getEvent() != null) {
            readCount++;
        }
    // try reading until all the written events are read, else the test will timeout.
    } while ((event.getEvent() != null || event.isCheckpoint()) && readCount < NUM_EVENTS);
    assertEquals("Read count should be equal to write count", NUM_EVENTS, readCount);
}

Example 44 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class LargeEventTest method largeEventSimpleTest.

/**
 * Invoke the largeEventSimpleTest, ensure we are able to produce  events.
 * The test fails incase of exceptions while writing to the stream.
 */
@Test
public void largeEventSimpleTest() {
    Service conService = Utils.createPravegaControllerService(null);
    List<URI> ctlURIs = conService.getServiceDetails();
    URI controllerUri = ctlURIs.get(0);
    log.info("Invoking create stream with Controller URI: {}", controllerUri);
    @Cleanup ConnectionFactory connectionFactory = new SocketConnectionFactoryImpl(Utils.buildClientConfig(controllerUri));
    @Cleanup ControllerImpl controller = new ControllerImpl(ControllerImplConfig.builder().clientConfig(Utils.buildClientConfig(controllerUri)).build(), connectionFactory.getInternalExecutor());
    assertTrue(controller.createScope(STREAM_SCOPE).join());
    assertTrue(controller.createStream(STREAM_SCOPE, STREAM_NAME, config).join());
    @Cleanup EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(STREAM_SCOPE, Utils.buildClientConfig(controllerUri));
    log.info("Invoking Writer test with Controller URI: {}", controllerUri);
    @Cleanup EventStreamWriter<ByteBuffer> writer = clientFactory.createEventWriter(STREAM_NAME, new ByteBufferSerializer(), EventWriterConfig.builder().build());
    byte[] payload = new byte[Serializer.MAX_EVENT_SIZE];
    for (int i = 0; i < NUM_EVENTS; i++) {
        log.debug("Producing event: {} ", i);
        // any exceptions while writing the event will fail the test.
        writer.writeEvent("", ByteBuffer.wrap(payload));
        writer.flush();
    }
    log.info("Invoking Reader test.");
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(STREAM_SCOPE, Utils.buildClientConfig(controllerUri));
    groupManager.createReaderGroup(READER_GROUP, ReaderGroupConfig.builder().stream(Stream.of(STREAM_SCOPE, STREAM_NAME)).build());
    @Cleanup EventStreamReader<ByteBuffer> reader = clientFactory.createReader(UUID.randomUUID().toString(), READER_GROUP, new ByteBufferSerializer(), ReaderConfig.builder().build());
    int readCount = 0;
    EventRead<ByteBuffer> event = null;
    do {
        event = reader.readNextEvent(10_000);
        log.debug("Read event: {}.", event.getEvent());
        if (event.getEvent() != null) {
            readCount++;
        }
    // try reading until all the written events are read, else the test will timeout.
    } while ((event.getEvent() != null || event.isCheckpoint()) && readCount < NUM_EVENTS);
    assertEquals("Read count should be equal to write count", NUM_EVENTS, readCount);
}

Example 45 with EventStreamClientFactory

use of io.pravega.client.EventStreamClientFactory in project pravega by pravega.

the class ReaderCheckpointTest method writeEvents.

private <T extends Serializable> void writeEvents(final String scope, final List<T> events) {
    ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);
    try (EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
        EventStreamWriter<T> writer = clientFactory.createEventWriter(STREAM, new JavaSerializer<T>(), EventWriterConfig.builder().build())) {
        for (T event : events) {
            String routingKey = String.valueOf(event);
            log.info("Writing message: {} with routing-key: {} to stream {}", event, routingKey, STREAM);
            writer.writeEvent(routingKey, event);
        }
    }
}