use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.
the class ReaderGroupNotificationTest method testSegmentNotifications.
@Test(timeout = 40000)
public void testSegmentNotifications() throws Exception {
final String streamName = "stream1";
StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(streamName).scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
Controller controller = controllerWrapper.getController();
controllerWrapper.getControllerService().createScope(SCOPE).get();
controller.createStream(config).get();
@Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
@Cleanup ClientFactory clientFactory = new ClientFactoryImpl(SCOPE, controller, connectionFactory);
@Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
writer.writeEvent("0", "data1").get();
// scale
Stream stream = new StreamImpl(SCOPE, streamName);
Map<Double, Double> map = new HashMap<>();
map.put(0.0, 0.5);
map.put(0.5, 1.0);
Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
assertTrue(result);
writer.writeEvent("0", "data2").get();
@Cleanup ReaderGroupManager groupManager = new ReaderGroupManagerImpl(SCOPE, controller, clientFactory, connectionFactory);
ReaderGroup readerGroup = groupManager.createReaderGroup("reader", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, streamName)).build());
@Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "reader", new JavaSerializer<>(), ReaderConfig.builder().build());
// Add segment event listener
Listener<SegmentNotification> l1 = notification -> {
listenerInvoked.set(true);
numberOfReaders.set(notification.getNumOfReaders());
numberOfSegments.set(notification.getNumOfSegments());
listenerLatch.release();
};
readerGroup.getSegmentNotifier(executor).registerListener(l1);
EventRead<String> event1 = reader1.readNextEvent(15000);
EventRead<String> event2 = reader1.readNextEvent(15000);
assertNotNull(event1);
assertEquals("data1", event1.getEvent());
assertNotNull(event2);
assertEquals("data2", event2.getEvent());
listenerLatch.await();
assertTrue("Listener invoked", listenerInvoked.get());
assertEquals(2, numberOfSegments.get());
assertEquals(1, numberOfReaders.get());
}
use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.
the class StateSynchronizerTest method testStateTracker.
@Test(timeout = 20000)
public void testStateTracker() {
String endpoint = "localhost";
String stateName = "abc";
int port = TestUtils.getAvailableListenPort();
StreamSegmentStore store = this.serviceBuilder.createStreamSegmentService();
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
server.startListening();
@Cleanup MockStreamManager streamManager = new MockStreamManager("scope", endpoint, port);
streamManager.createScope("scope");
streamManager.createStream("scope", stateName, null);
JavaSerializer<TestUpdate> serializer = new JavaSerializer<TestUpdate>();
@Cleanup val a = streamManager.getClientFactory().createStateSynchronizer(stateName, serializer, serializer, SynchronizerConfig.builder().build());
@Cleanup val b = streamManager.getClientFactory().createStateSynchronizer(stateName, serializer, serializer, SynchronizerConfig.builder().build());
a.initialize(new TestUpdate("init"));
b.fetchUpdates();
assertEquals("init", b.getState().value);
assertEquals(1, update(a, "already up to date 1"));
assertEquals(2, update(b, "fail Initially 2"));
assertEquals("already up to date 1", a.getState().value);
assertEquals("fail Initially 2", b.getState().value);
assertEquals(1, update(b, "already up to date 3"));
assertEquals("already up to date 1", a.getState().value);
a.fetchUpdates();
assertEquals("already up to date 3", a.getState().value);
assertEquals(1, update(a, "already up to date 4"));
assertEquals("already up to date 4", a.getState().value);
assertEquals("already up to date 3", b.getState().value);
assertEquals(2, update(b, "fail Initially 5"));
assertEquals("already up to date 4", a.getState().value);
a.fetchUpdates();
assertEquals("fail Initially 5", a.getState().value);
a.fetchUpdates();
b.fetchUpdates();
assertEquals("fail Initially 5", a.getState().value);
assertEquals("fail Initially 5", b.getState().value);
}
use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.
the class EventProcessorTest method testEventProcessor.
@Test(timeout = 60000)
public void testEventProcessor() throws Exception {
@Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
int servicePort = TestUtils.getAvailableListenPort();
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, servicePort, store);
server.startListening();
int controllerPort = TestUtils.getAvailableListenPort();
@Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), true, controllerPort, "localhost", servicePort, 4);
controllerWrapper.awaitRunning();
Controller controller = controllerWrapper.getController();
// Create controller object for testing against a separate controller process.
// ControllerImpl controller = new ControllerImpl("localhost", 9090);
final String host = "host";
final String scope = "controllerScope";
final String streamName = "stream1";
final String readerGroup = "readerGroup";
final CompletableFuture<Boolean> createScopeStatus = controller.createScope(scope);
if (!createScopeStatus.join()) {
throw new RuntimeException("Scope already existed");
}
final StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build();
System.err.println(String.format("Creating stream (%s, %s)", scope, streamName));
CompletableFuture<Boolean> createStatus = controller.createStream(config);
if (!createStatus.get()) {
System.err.println("Stream alrady existed, exiting");
return;
}
@Cleanup ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
@Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
@Cleanup EventStreamWriter<TestEvent> producer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
int expectedSum = input.length * (input.length + 1) / 2;
for (int i = 0; i < input.length; i++) {
producer.writeEvent("key", new TestEvent(input[i]));
}
producer.writeEvent("key", new TestEvent(-1));
producer.flush();
EventProcessorSystem system = new EventProcessorSystemImpl("Controller", host, scope, new ClientFactoryImpl(scope, controller, connectionFactory), new ReaderGroupManagerImpl(scope, controller, clientFactory, connectionFactory));
CheckpointConfig.CheckpointPeriod period = CheckpointConfig.CheckpointPeriod.builder().numEvents(1).numSeconds(1).build();
CheckpointConfig checkpointConfig = CheckpointConfig.builder().type(CheckpointConfig.Type.Periodic).checkpointPeriod(period).build();
EventProcessorGroupConfig eventProcessorGroupConfig = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(readerGroup).streamName(streamName).checkpointConfig(checkpointConfig).build();
CompletableFuture<Long> result = new CompletableFuture<>();
// Test case 1. Actor does not throw any exception during normal operation.
EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(false, result)).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(eventProcessorGroupConfig).build();
@Cleanup EventProcessorGroup<TestEvent> eventProcessorGroup = system.createEventProcessorGroup(eventProcessorConfig, CheckpointStoreFactory.createInMemoryStore());
Long value = result.join();
Assert.assertEquals(expectedSum, value.longValue());
log.info("SUCCESS: received expected sum = " + expectedSum);
}
use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.
the class EndToEndAutoScaleUpWithTxnTest method main.
public static void main(String[] args) throws Exception {
try {
@Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
int port = Config.SERVICE_PORT;
@Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port);
Controller controller = controllerWrapper.getController();
controllerWrapper.getControllerService().createScope(NameUtils.INTERNAL_SCOPE_NAME).get();
@Cleanup ConnectionFactory connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
@Cleanup ClientFactory internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, connectionFactory);
ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
@Cleanup SegmentStatsFactory segmentStatsFactory = new SegmentStatsFactory();
SegmentStatsRecorder statsRecorder = segmentStatsFactory.createSegmentStatsRecorder(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).build());
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, "localhost", 12345, store, statsRecorder, null, null, null);
server.startListening();
controllerWrapper.awaitRunning();
controllerWrapper.getControllerService().createScope("test").get();
controller.createStream(CONFIG).get();
@Cleanup MockClientFactory clientFactory = new MockClientFactory("test", controller);
// Mocking pravega service by putting scale up and scale down requests for the stream
EventWriterConfig writerConfig = EventWriterConfig.builder().transactionTimeoutTime(30000).transactionTimeoutScaleGracePeriod(30000).build();
EventStreamWriter<String> test = clientFactory.createEventWriter("test", new JavaSerializer<>(), writerConfig);
// region Successful commit tests
Transaction<String> txn1 = test.beginTxn();
txn1.writeEvent("1");
txn1.flush();
Map<Double, Double> map = new HashMap<>();
map.put(0.0, 1.0 / 3.0);
map.put(1.0 / 3.0, 2.0 / 3.0);
map.put(2.0 / 3.0, 1.0);
Stream stream = new StreamImpl("test", "test");
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
Transaction<String> txn2 = test.beginTxn();
txn2.writeEvent("2");
txn2.flush();
txn2.commit();
txn1.commit();
Thread.sleep(1000);
@Cleanup ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl("test", controller, clientFactory, connectionFactory);
readerGroupManager.createReaderGroup("readergrp", ReaderGroupConfig.builder().stream("test").build());
final EventStreamReader<String> reader = clientFactory.createReader("1", "readergrp", new JavaSerializer<>(), ReaderConfig.builder().build());
String event1 = reader.readNextEvent(SECONDS.toMillis(60)).getEvent();
String event2 = reader.readNextEvent(SECONDS.toMillis(60)).getEvent();
assert event1.equals("1");
assert event2.equals("2");
final AtomicBoolean done = new AtomicBoolean(false);
startWriter(test, done);
Retry.withExpBackoff(10, 10, 100, 10000).retryingOn(NotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments("test", "test").thenAccept(streamSegments -> {
if (streamSegments.getSegments().size() > 3) {
System.err.println("Success");
log.info("Success");
System.exit(0);
} else {
throw new NotDoneException();
}
}), Executors.newSingleThreadScheduledExecutor()).exceptionally(e -> {
System.err.println("Failure");
log.error("Failure");
System.exit(1);
return null;
}).get();
} catch (Throwable e) {
System.err.print("Test failed with exception: " + e.getMessage());
log.error("Test failed with exception: {}", e);
System.exit(-1);
}
System.exit(0);
}
use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.
the class AutoCheckpointTest method testOnlyOneOutstanding.
@Test(timeout = 30000)
public void testOnlyOneOutstanding() throws ReinitializationRequiredException, DurableDataLogException {
String endpoint = "localhost";
String streamName = "abc";
String readerGroup = "group";
int port = TestUtils.getAvailableListenPort();
String testString = "Hello world: ";
String scope = "Scope1";
@Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
server.startListening();
@Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
MockClientFactory clientFactory = streamManager.getClientFactory();
ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(1000).stream(Stream.of(scope, streamName)).build();
streamManager.createScope(scope);
streamManager.createStream(scope, streamName, null);
streamManager.createReaderGroup(readerGroup, groupConfig);
JavaSerializer<String> serializer = new JavaSerializer<>();
populateEvents(streamName, testString, clientFactory, serializer);
AtomicLong fakeClock = new AtomicLong(0);
@Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("reader1", readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
@Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("reader2", readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
int numRead = 0;
int checkpointCount = 0;
while (numRead < 100) {
fakeClock.addAndGet(NANOS_PER_SECOND);
EventRead<String> event = reader1.readNextEvent(1000);
if (event.isCheckpoint()) {
checkpointCount++;
} else {
String message = event.getEvent();
assertEquals(testString + numRead, message);
numRead++;
}
}
assertEquals("As there is a second reader that does not pass the checkpoint, only one should occur", 1, checkpointCount);
}
Aggregations