use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.
the class ScaleTest method main.
public static void main(String[] args) throws Exception {
try {
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
@Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
int port = Config.SERVICE_PORT;
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
server.startListening();
// Create controller object for testing against a separate controller report.
@Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port);
Controller controller = controllerWrapper.getController();
final String scope = "scope";
controllerWrapper.getControllerService().createScope(scope).get();
final String streamName = "stream1";
final StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build();
Stream stream = new StreamImpl(scope, streamName);
log.info("Creating stream {}/{}", scope, streamName);
if (!controller.createStream(config).get()) {
log.error("Stream already existed, exiting");
return;
}
// Test 1: scale stream: split one segment into two
log.info("Scaling stream {}/{}, splitting one segment into two", scope, streamName);
Map<Double, Double> map = new HashMap<>();
map.put(0.0, 0.5);
map.put(0.5, 1.0);
if (!controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get()) {
log.error("Scale stream: splitting segment into two failed, exiting");
return;
}
// Test 2: scale stream: merge two segments into one
log.info("Scaling stream {}/{}, merging two segments into one", scope, streamName);
CompletableFuture<Boolean> scaleResponseFuture = controller.scaleStream(stream, Arrays.asList(1, 2), Collections.singletonMap(0.0, 1.0), executor).getFuture();
if (!scaleResponseFuture.get()) {
log.error("Scale stream: merging two segments into one failed, exiting");
return;
}
// Test 3: create a transaction, and try scale operation, it should fail with precondition check failure
CompletableFuture<TxnSegments> txnFuture = controller.createTransaction(stream, 5000, 29000);
TxnSegments transaction = txnFuture.get();
if (transaction == null) {
log.error("Create transaction failed, exiting");
return;
}
log.info("Scaling stream {}/{}, splitting one segment into two, while transaction is ongoing", scope, streamName);
scaleResponseFuture = controller.scaleStream(stream, Collections.singletonList(3), map, executor).getFuture();
CompletableFuture<Boolean> future = scaleResponseFuture.whenComplete((r, e) -> {
if (e != null) {
log.error("Failed: scale with ongoing transaction.", e);
} else if (getAndHandleExceptions(controller.checkTransactionStatus(stream, transaction.getTxnId()), RuntimeException::new) != Transaction.Status.OPEN) {
log.info("Success: scale with ongoing transaction.");
} else {
log.error("Failed: scale with ongoing transaction.");
}
});
CompletableFuture<Void> statusFuture = controller.abortTransaction(stream, transaction.getTxnId());
statusFuture.get();
future.get();
log.info("All scaling test PASSED");
executor.shutdown();
System.exit(0);
} catch (Throwable t) {
log.error("test failed with {}", t);
System.exit(-1);
}
}
use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.
the class ClientAdapterBase method createStream.
@Override
public CompletableFuture<Void> createStream(String streamName, Duration timeout) {
ensureRunning();
return CompletableFuture.runAsync(() -> {
if (this.streamWriters.containsKey(streamName)) {
throw new CompletionException(new StreamSegmentExistsException(streamName));
}
StreamConfiguration config = StreamConfiguration.builder().streamName(streamName).scalingPolicy(ScalingPolicy.fixed(this.testConfig.getSegmentsPerStream())).scope(SCOPE).build();
if (!getStreamManager().createStream(SCOPE, streamName, config)) {
throw new CompletionException(new StreamingException(String.format("Unable to create Stream '%s'.", streamName)));
}
int writerCount = Math.max(1, this.testConfig.getProducerCount() / this.testConfig.getStreamCount());
List<EventStreamWriter<byte[]>> writers = new ArrayList<>(writerCount);
if (this.streamWriters.putIfAbsent(streamName, writers) == null) {
for (int i = 0; i < writerCount; i++) {
writers.add(getClientFactory().createEventWriter(streamName, SERIALIZER, WRITER_CONFIG));
}
}
}, this.testExecutor);
}
use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.
the class BatchClientTest method testBatchClient.
@Test(timeout = 50000)
public void testBatchClient() throws Exception {
StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(ScalingPolicy.fixed(1)).build();
Controller controller = controllerWrapper.getController();
controllerWrapper.getControllerService().createScope(SCOPE).get();
controller.createStream(config).get();
// create reader and writer.
@Cleanup ClientFactory clientFactory = ClientFactory.withScope(SCOPE, controllerUri);
@Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerUri);
groupManager.createReaderGroup("group", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, STREAM)).build());
@Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(STREAM, serializer, EventWriterConfig.builder().build());
// write events to stream with 1 segment.
writeEvents(writer);
// scale up and write events.
Stream stream = new StreamImpl(SCOPE, STREAM);
Map<Double, Double> map = new HashMap<>();
map.put(0.0, 0.33);
map.put(0.33, 0.66);
map.put(0.66, 1.0);
Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
assertTrue("Scale up operation", result);
writeEvents(writer);
// scale down and write events.
map = new HashMap<>();
map.put(0.0, 0.5);
map.put(0.5, 1.0);
result = controller.scaleStream(stream, Arrays.asList(1, 2, 3), map, executor).getFuture().get();
assertTrue("Scale down operation result", result);
writeEvents(writer);
BatchClient batchClient = clientFactory.createBatchClient();
// List out all the segments in the stream.
ArrayList<SegmentRange> segments = Lists.newArrayList(batchClient.getSegments(stream, null, null).getIterator());
assertEquals("Expected number of segments", 6, segments.size());
// Batch read all events from stream.
List<String> batchEventList = new ArrayList<>();
segments.forEach(segInfo -> {
@Cleanup SegmentIterator<String> segmentIterator = batchClient.readSegment(segInfo, serializer);
batchEventList.addAll(Lists.newArrayList(segmentIterator));
});
assertEquals("Event count", 9, batchEventList.size());
// read from a given offset.
Segment seg0 = new Segment(SCOPE, STREAM, 0);
SegmentRange seg0Info = SegmentRangeImpl.builder().segment(seg0).startOffset(60).endOffset(90).build();
@Cleanup SegmentIterator<String> seg0Iterator = batchClient.readSegment(seg0Info, serializer);
ArrayList<String> dataAtOffset = Lists.newArrayList(seg0Iterator);
assertEquals(1, dataAtOffset.size());
assertEquals(DATA_OF_SIZE_30, dataAtOffset.get(0));
}
use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.
the class ControllerFailoverTest method createStream.
private void createStream(Controller controller, String scope, String stream, ScalingPolicy scalingPolicy) {
StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(stream).scalingPolicy(scalingPolicy).build();
controller.createStream(config).join();
}
use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.
the class EndToEndTransactionTest method main.
@Test
public static void main(String[] args) throws Exception {
@Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
int port = Config.SERVICE_PORT;
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
server.startListening();
Thread.sleep(1000);
@Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port);
Controller controller = controllerWrapper.getController();
controllerWrapper.awaitRunning();
final String testScope = "testScope";
final String testStream = "testStream";
if (!controller.createScope(testScope).get()) {
log.error("FAILURE: Error creating test scope");
return;
}
ScalingPolicy policy = ScalingPolicy.fixed(5);
StreamConfiguration streamConfig = StreamConfiguration.builder().scope(testScope).streamName(testStream).scalingPolicy(policy).build();
if (!controller.createStream(streamConfig).get()) {
log.error("FAILURE: Error creating test stream");
return;
}
final long txnTimeout = 4000;
@Cleanup MockClientFactory clientFactory = new MockClientFactory(testScope, controller);
@Cleanup EventStreamWriter<String> producer = clientFactory.createEventWriter(testStream, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(txnTimeout).transactionTimeoutScaleGracePeriod(txnTimeout).build());
// region Successful commit tests
Transaction<String> transaction = producer.beginTxn();
for (int i = 0; i < 1; i++) {
String event = "\n Transactional Publish \n";
log.info("Producing event: " + event);
transaction.writeEvent("", event);
transaction.flush();
Thread.sleep(500);
}
CompletableFuture<Object> commit = CompletableFuture.supplyAsync(() -> {
try {
transaction.commit();
} catch (Exception e) {
log.warn("Error committing transaction", e);
}
return null;
});
commit.join();
Transaction.Status txnStatus = transaction.checkStatus();
assertTrue(txnStatus == Transaction.Status.COMMITTING || txnStatus == Transaction.Status.COMMITTED);
log.info("SUCCESS: successful in committing transaction. Transaction status=" + txnStatus);
Thread.sleep(2000);
txnStatus = transaction.checkStatus();
assertTrue(txnStatus == Transaction.Status.COMMITTED);
log.info("SUCCESS: successfully committed transaction. Transaction status=" + txnStatus);
// endregion
// region Successful abort tests
Transaction<String> transaction2 = producer.beginTxn();
for (int i = 0; i < 1; i++) {
String event = "\n Transactional Publish \n";
log.info("Producing event: " + event);
transaction2.writeEvent("", event);
transaction2.flush();
Thread.sleep(500);
}
CompletableFuture<Object> drop = CompletableFuture.supplyAsync(() -> {
try {
transaction2.abort();
} catch (Exception e) {
log.warn("Error aborting transaction", e);
}
return null;
});
drop.join();
Transaction.Status txn2Status = transaction2.checkStatus();
assertTrue(txn2Status == Transaction.Status.ABORTING || txn2Status == Transaction.Status.ABORTED);
log.info("SUCCESS: successful in dropping transaction. Transaction status=" + txn2Status);
Thread.sleep(2000);
txn2Status = transaction2.checkStatus();
assertTrue(txn2Status == Transaction.Status.ABORTED);
log.info("SUCCESS: successfully aborted transaction. Transaction status=" + txn2Status);
// endregion
// region Successful timeout tests
Transaction<String> tx1 = producer.beginTxn();
Thread.sleep((long) (1.3 * txnTimeout));
Transaction.Status txStatus = tx1.checkStatus();
Assert.assertTrue(Transaction.Status.ABORTING == txStatus || Transaction.Status.ABORTED == txStatus);
log.info("SUCCESS: successfully aborted transaction after timeout. Transaction status=" + txStatus);
// endregion
// region Ping failure due to controller going into disconnection state
// Fill in these tests once we have controller.stop() implemented.
System.exit(0);
}
Aggregations