use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.
the class CheckpointTest method testCheckpointAndRestore.
@Test(timeout = 20000)
public void testCheckpointAndRestore() throws ReinitializationRequiredException, InterruptedException, ExecutionException, TimeoutException {
String endpoint = "localhost";
String streamName = "abc";
String readerName = "reader";
String readerGroupName = "group";
int port = TestUtils.getAvailableListenPort();
String testString = "Hello world\n";
String scope = "Scope1";
StreamSegmentStore store = this.serviceBuilder.createStreamSegmentService();
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
server.startListening();
@Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
MockClientFactory clientFactory = streamManager.getClientFactory();
ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(scope, streamName)).build();
streamManager.createScope(scope);
streamManager.createStream(scope, streamName, StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build());
ReaderGroup readerGroup = streamManager.createReaderGroup(readerGroupName, groupConfig);
JavaSerializer<String> serializer = new JavaSerializer<>();
EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
producer.writeEvent(testString);
producer.writeEvent(testString);
producer.writeEvent(testString);
producer.flush();
AtomicLong clock = new AtomicLong();
@Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
EventRead<String> read = reader.readNextEvent(60000);
assertEquals(testString, read.getEvent());
clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
read = reader.readNextEvent(60000);
assertEquals(testString, read.getEvent());
clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
@Cleanup("shutdown") final InlineExecutor backgroundExecutor = new InlineExecutor();
CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("Checkpoint", backgroundExecutor);
assertFalse(checkpoint.isDone());
read = reader.readNextEvent(60000);
assertTrue(read.isCheckpoint());
assertEquals("Checkpoint", read.getCheckpointName());
assertNull(read.getEvent());
clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
read = reader.readNextEvent(60000);
assertEquals(testString, read.getEvent());
Checkpoint cpResult = checkpoint.get(5, TimeUnit.SECONDS);
assertTrue(checkpoint.isDone());
assertEquals("Checkpoint", cpResult.getName());
read = reader.readNextEvent(100);
assertNull(read.getEvent());
assertFalse(read.isCheckpoint());
clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromCheckpoint(cpResult).build());
try {
reader.readNextEvent(60000);
fail();
} catch (ReinitializationRequiredException e) {
// Expected
}
reader.close();
reader = clientFactory.createReader(readerName, readerGroupName, serializer, ReaderConfig.builder().build());
clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
read = reader.readNextEvent(60000);
assertEquals(testString, read.getEvent());
clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
read = reader.readNextEvent(100);
assertNull(read.getEvent());
assertFalse(read.isCheckpoint());
}
use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.
the class EndToEndAutoScaleUpTest method main.
public static void main(String[] args) throws Exception {
try {
@Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
int port = Config.SERVICE_PORT;
@Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port, false);
Controller controller = controllerWrapper.getController();
ClientFactory internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, new ConnectionFactoryImpl(ClientConfig.builder().build()));
ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
@Cleanup SegmentStatsFactory segmentStatsFactory = new SegmentStatsFactory();
SegmentStatsRecorder statsRecorder = segmentStatsFactory.createSegmentStatsRecorder(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).build());
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, "localhost", 12345, store, statsRecorder, null, null, null);
server.startListening();
controllerWrapper.awaitRunning();
controllerWrapper.getControllerService().createScope("test").get();
controller.createStream(CONFIG).get();
@Cleanup MockClientFactory clientFactory = new MockClientFactory("test", controller);
// Mocking pravega service by putting scale up and scale down requests for the stream
EventStreamWriter<String> test = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().build());
// keep writing. Scale should happen
long start = System.currentTimeMillis();
char[] chars = new char[1];
Arrays.fill(chars, 'a');
String str = new String(chars);
CompletableFuture.runAsync(() -> {
while (System.currentTimeMillis() - start < Duration.ofMinutes(3).toMillis()) {
try {
test.writeEvent("0", str).get();
} catch (Throwable e) {
System.err.println("test exception writing events " + e.getMessage());
break;
}
}
});
Retry.withExpBackoff(10, 10, 100, 10000).retryingOn(NotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments("test", "test").thenAccept(streamSegments -> {
if (streamSegments.getSegments().size() > 3) {
System.err.println("Success");
log.info("Success");
System.exit(0);
} else {
throw new NotDoneException();
}
}), Executors.newSingleThreadScheduledExecutor()).exceptionally(e -> {
System.err.println("Failure");
log.error("Failure");
System.exit(1);
return null;
}).get();
} catch (Throwable e) {
System.err.print("Test failed with exception: " + e.getMessage());
System.exit(-1);
}
System.exit(0);
}
use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.
the class AutoCheckpointTest method testCheckpointsOccur.
@Test(timeout = 30000)
public void testCheckpointsOccur() throws ReinitializationRequiredException, DurableDataLogException {
String endpoint = "localhost";
String streamName = "abc";
String readerName = "reader";
String readerGroup = "group";
int port = TestUtils.getAvailableListenPort();
String testString = "Hello world: ";
String scope = "Scope1";
@Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
server.startListening();
@Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
MockClientFactory clientFactory = streamManager.getClientFactory();
ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(10000).stream(Stream.of(scope, streamName)).build();
streamManager.createScope(scope);
streamManager.createStream(scope, streamName, null);
streamManager.createReaderGroup(readerGroup, groupConfig);
JavaSerializer<String> serializer = new JavaSerializer<>();
populateEvents(streamName, testString, clientFactory, serializer);
AtomicLong fakeClock = new AtomicLong(0);
@Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
int numRead = 0;
int checkpointCount = 0;
while (numRead < 100) {
fakeClock.addAndGet(NANOS_PER_SECOND);
EventRead<String> event = reader.readNextEvent(1000);
if (event.isCheckpoint()) {
checkpointCount++;
} else {
String message = event.getEvent();
assertEquals(testString + numRead, message);
numRead++;
}
}
assertTrue("Count was " + checkpointCount, checkpointCount > 5);
assertTrue("Count was " + checkpointCount, checkpointCount < 20);
}
use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.
the class CheckpointTest method testMoreReadersThanSegments.
@Test(timeout = 20000)
public void testMoreReadersThanSegments() throws ReinitializationRequiredException, InterruptedException, ExecutionException, TimeoutException {
String endpoint = "localhost";
String streamName = "abc";
String readerGroupName = "group";
int port = TestUtils.getAvailableListenPort();
String testString = "Hello world\n";
String scope = "Scope1";
StreamSegmentStore store = this.serviceBuilder.createStreamSegmentService();
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
server.startListening();
@Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
MockClientFactory clientFactory = streamManager.getClientFactory();
ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build();
streamManager.createScope(scope);
streamManager.createStream(scope, streamName, StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build());
ReaderGroup readerGroup = streamManager.createReaderGroup(readerGroupName, groupConfig);
JavaSerializer<String> serializer = new JavaSerializer<>();
EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
producer.writeEvent(testString);
producer.writeEvent(testString);
producer.writeEvent(testString);
producer.flush();
AtomicLong clock = new AtomicLong();
@Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("reader1", readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
@Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("reader2", readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
@Cleanup("shutdown") final InlineExecutor backgroundExecutor = new InlineExecutor();
CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("Checkpoint", backgroundExecutor);
assertFalse(checkpoint.isDone());
EventRead<String> read = reader1.readNextEvent(60000);
assertTrue(read.isCheckpoint());
assertEquals("Checkpoint", read.getCheckpointName());
assertNull(read.getEvent());
read = reader2.readNextEvent(60000);
assertTrue(read.isCheckpoint());
assertEquals("Checkpoint", read.getCheckpointName());
assertNull(read.getEvent());
Checkpoint cpResult = checkpoint.get(5, TimeUnit.SECONDS);
assertTrue(checkpoint.isDone());
assertEquals("Checkpoint", cpResult.getName());
}
use of io.pravega.client.stream.impl.JavaSerializer in project pravega by pravega.
the class ReadTest method testEventPointer.
@Test(timeout = 10000)
public void testEventPointer() throws ReinitializationRequiredException {
String endpoint = "localhost";
String streamName = "abc";
String readerName = "reader";
String readerGroup = "group";
int port = TestUtils.getAvailableListenPort();
String testString = "Hello world ";
String scope = "Scope1";
StreamSegmentStore store = this.serviceBuilder.createStreamSegmentService();
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
server.startListening();
@Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
MockClientFactory clientFactory = streamManager.getClientFactory();
ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(scope, streamName)).build();
streamManager.createScope(scope);
streamManager.createStream(scope, streamName, null);
streamManager.createReaderGroup(readerGroup, groupConfig);
JavaSerializer<String> serializer = new JavaSerializer<>();
EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
for (int i = 0; i < 100; i++) {
producer.writeEvent(testString + i);
}
producer.flush();
@Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroup, serializer, ReaderConfig.builder().build());
try {
EventPointer pointer;
String read;
for (int i = 0; i < 100; i++) {
pointer = reader.readNextEvent(5000).getEventPointer();
read = reader.fetchEvent(pointer);
assertEquals(testString + i, read);
}
} catch (NoSuchEventException e) {
fail("Failed to read event using event pointer");
}
}
Aggregations