Search in sources :

Example 36 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class ScaleTest method main.

public static void main(String[] args) throws Exception {
    try {
        ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
        @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
        ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
        serviceBuilder.initialize();
        StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
        int port = Config.SERVICE_PORT;
        @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
        server.startListening();
        // Create controller object for testing against a separate controller report.
        @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port);
        Controller controller = controllerWrapper.getController();
        final String scope = "scope";
        controllerWrapper.getControllerService().createScope(scope).get();
        final String streamName = "stream1";
        final StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build();
        Stream stream = new StreamImpl(scope, streamName);
        log.info("Creating stream {}/{}", scope, streamName);
        if (!controller.createStream(config).get()) {
            log.error("Stream already existed, exiting");
            return;
        }
        // Test 1: scale stream: split one segment into two
        log.info("Scaling stream {}/{}, splitting one segment into two", scope, streamName);
        Map<Double, Double> map = new HashMap<>();
        map.put(0.0, 0.5);
        map.put(0.5, 1.0);
        if (!controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get()) {
            log.error("Scale stream: splitting segment into two failed, exiting");
            return;
        }
        // Test 2: scale stream: merge two segments into one
        log.info("Scaling stream {}/{}, merging two segments into one", scope, streamName);
        CompletableFuture<Boolean> scaleResponseFuture = controller.scaleStream(stream, Arrays.asList(1, 2), Collections.singletonMap(0.0, 1.0), executor).getFuture();
        if (!scaleResponseFuture.get()) {
            log.error("Scale stream: merging two segments into one failed, exiting");
            return;
        }
        // Test 3: create a transaction, and try scale operation, it should fail with precondition check failure
        CompletableFuture<TxnSegments> txnFuture = controller.createTransaction(stream, 5000, 29000);
        TxnSegments transaction = txnFuture.get();
        if (transaction == null) {
            log.error("Create transaction failed, exiting");
            return;
        }
        log.info("Scaling stream {}/{}, splitting one segment into two, while transaction is ongoing", scope, streamName);
        scaleResponseFuture = controller.scaleStream(stream, Collections.singletonList(3), map, executor).getFuture();
        CompletableFuture<Boolean> future = scaleResponseFuture.whenComplete((r, e) -> {
            if (e != null) {
                log.error("Failed: scale with ongoing transaction.", e);
            } else if (getAndHandleExceptions(controller.checkTransactionStatus(stream, transaction.getTxnId()), RuntimeException::new) != Transaction.Status.OPEN) {
                log.info("Success: scale with ongoing transaction.");
            } else {
                log.error("Failed: scale with ongoing transaction.");
            }
        });
        CompletableFuture<Void> statusFuture = controller.abortTransaction(stream, transaction.getTxnId());
        statusFuture.get();
        future.get();
        log.info("All scaling test PASSED");
        executor.shutdown();
        System.exit(0);
    } catch (Throwable t) {
        log.error("test failed with {}", t);
        System.exit(-1);
    }
}
Also used : HashMap(java.util.HashMap) Cleanup(lombok.Cleanup) PravegaConnectionListener(io.pravega.segmentstore.server.host.handler.PravegaConnectionListener) ServiceBuilder(io.pravega.segmentstore.server.store.ServiceBuilder) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) Stream(io.pravega.client.stream.Stream) TestingServer(org.apache.curator.test.TestingServer) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) TxnSegments(io.pravega.client.stream.impl.TxnSegments) TestingServerStarter(io.pravega.test.common.TestingServerStarter) Controller(io.pravega.client.stream.impl.Controller) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) StreamImpl(io.pravega.client.stream.impl.StreamImpl)

Example 37 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class ClientAdapterBase method createStream.

@Override
public CompletableFuture<Void> createStream(String streamName, Duration timeout) {
    ensureRunning();
    return CompletableFuture.runAsync(() -> {
        if (this.streamWriters.containsKey(streamName)) {
            throw new CompletionException(new StreamSegmentExistsException(streamName));
        }
        StreamConfiguration config = StreamConfiguration.builder().streamName(streamName).scalingPolicy(ScalingPolicy.fixed(this.testConfig.getSegmentsPerStream())).scope(SCOPE).build();
        if (!getStreamManager().createStream(SCOPE, streamName, config)) {
            throw new CompletionException(new StreamingException(String.format("Unable to create Stream '%s'.", streamName)));
        }
        int writerCount = Math.max(1, this.testConfig.getProducerCount() / this.testConfig.getStreamCount());
        List<EventStreamWriter<byte[]>> writers = new ArrayList<>(writerCount);
        if (this.streamWriters.putIfAbsent(streamName, writers) == null) {
            for (int i = 0; i < writerCount; i++) {
                writers.add(getClientFactory().createEventWriter(streamName, SERIALIZER, WRITER_CONFIG));
            }
        }
    }, this.testExecutor);
}
Also used : StreamingException(io.pravega.segmentstore.contracts.StreamingException) StreamSegmentExistsException(io.pravega.segmentstore.contracts.StreamSegmentExistsException) CompletionException(java.util.concurrent.CompletionException) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) ArrayList(java.util.ArrayList) EventStreamWriter(io.pravega.client.stream.EventStreamWriter)

Example 38 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class BatchClientTest method testBatchClient.

@Test(timeout = 50000)
public void testBatchClient() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(ScalingPolicy.fixed(1)).build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope(SCOPE).get();
    controller.createStream(config).get();
    // create reader and writer.
    @Cleanup ClientFactory clientFactory = ClientFactory.withScope(SCOPE, controllerUri);
    @Cleanup ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerUri);
    groupManager.createReaderGroup("group", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(SCOPE, STREAM)).build());
    @Cleanup EventStreamWriter<String> writer = clientFactory.createEventWriter(STREAM, serializer, EventWriterConfig.builder().build());
    // write events to stream with 1 segment.
    writeEvents(writer);
    // scale up and write events.
    Stream stream = new StreamImpl(SCOPE, STREAM);
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 0.33);
    map.put(0.33, 0.66);
    map.put(0.66, 1.0);
    Boolean result = controller.scaleStream(stream, Collections.singletonList(0), map, executor).getFuture().get();
    assertTrue("Scale up operation", result);
    writeEvents(writer);
    // scale down and write events.
    map = new HashMap<>();
    map.put(0.0, 0.5);
    map.put(0.5, 1.0);
    result = controller.scaleStream(stream, Arrays.asList(1, 2, 3), map, executor).getFuture().get();
    assertTrue("Scale down operation result", result);
    writeEvents(writer);
    BatchClient batchClient = clientFactory.createBatchClient();
    // List out all the segments in the stream.
    ArrayList<SegmentRange> segments = Lists.newArrayList(batchClient.getSegments(stream, null, null).getIterator());
    assertEquals("Expected number of segments", 6, segments.size());
    // Batch read all events from stream.
    List<String> batchEventList = new ArrayList<>();
    segments.forEach(segInfo -> {
        @Cleanup SegmentIterator<String> segmentIterator = batchClient.readSegment(segInfo, serializer);
        batchEventList.addAll(Lists.newArrayList(segmentIterator));
    });
    assertEquals("Event count", 9, batchEventList.size());
    // read from a given offset.
    Segment seg0 = new Segment(SCOPE, STREAM, 0);
    SegmentRange seg0Info = SegmentRangeImpl.builder().segment(seg0).startOffset(60).endOffset(90).build();
    @Cleanup SegmentIterator<String> seg0Iterator = batchClient.readSegment(seg0Info, serializer);
    ArrayList<String> dataAtOffset = Lists.newArrayList(seg0Iterator);
    assertEquals(1, dataAtOffset.size());
    assertEquals(DATA_OF_SIZE_30, dataAtOffset.get(0));
}
Also used : SegmentRange(io.pravega.client.batch.SegmentRange) ReaderGroupManager(io.pravega.client.admin.ReaderGroupManager) HashMap(java.util.HashMap) BatchClient(io.pravega.client.batch.BatchClient) ClientFactory(io.pravega.client.ClientFactory) ArrayList(java.util.ArrayList) Controller(io.pravega.client.stream.impl.Controller) Cleanup(lombok.Cleanup) Segment(io.pravega.client.segment.impl.Segment) StreamImpl(io.pravega.client.stream.impl.StreamImpl) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) Stream(io.pravega.client.stream.Stream) Test(org.junit.Test)

Example 39 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class ControllerFailoverTest method createStream.

private void createStream(Controller controller, String scope, String stream, ScalingPolicy scalingPolicy) {
    StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(stream).scalingPolicy(scalingPolicy).build();
    controller.createStream(config).join();
}
Also used : StreamConfiguration(io.pravega.client.stream.StreamConfiguration)

Example 40 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class EndToEndTransactionTest method main.

@Test
public static void main(String[] args) throws Exception {
    @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    int port = Config.SERVICE_PORT;
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    Thread.sleep(1000);
    @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port);
    Controller controller = controllerWrapper.getController();
    controllerWrapper.awaitRunning();
    final String testScope = "testScope";
    final String testStream = "testStream";
    if (!controller.createScope(testScope).get()) {
        log.error("FAILURE: Error creating test scope");
        return;
    }
    ScalingPolicy policy = ScalingPolicy.fixed(5);
    StreamConfiguration streamConfig = StreamConfiguration.builder().scope(testScope).streamName(testStream).scalingPolicy(policy).build();
    if (!controller.createStream(streamConfig).get()) {
        log.error("FAILURE: Error creating test stream");
        return;
    }
    final long txnTimeout = 4000;
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(testScope, controller);
    @Cleanup EventStreamWriter<String> producer = clientFactory.createEventWriter(testStream, new JavaSerializer<>(), EventWriterConfig.builder().transactionTimeoutTime(txnTimeout).transactionTimeoutScaleGracePeriod(txnTimeout).build());
    // region Successful commit tests
    Transaction<String> transaction = producer.beginTxn();
    for (int i = 0; i < 1; i++) {
        String event = "\n Transactional Publish \n";
        log.info("Producing event: " + event);
        transaction.writeEvent("", event);
        transaction.flush();
        Thread.sleep(500);
    }
    CompletableFuture<Object> commit = CompletableFuture.supplyAsync(() -> {
        try {
            transaction.commit();
        } catch (Exception e) {
            log.warn("Error committing transaction", e);
        }
        return null;
    });
    commit.join();
    Transaction.Status txnStatus = transaction.checkStatus();
    assertTrue(txnStatus == Transaction.Status.COMMITTING || txnStatus == Transaction.Status.COMMITTED);
    log.info("SUCCESS: successful in committing transaction. Transaction status=" + txnStatus);
    Thread.sleep(2000);
    txnStatus = transaction.checkStatus();
    assertTrue(txnStatus == Transaction.Status.COMMITTED);
    log.info("SUCCESS: successfully committed transaction. Transaction status=" + txnStatus);
    // endregion
    // region Successful abort tests
    Transaction<String> transaction2 = producer.beginTxn();
    for (int i = 0; i < 1; i++) {
        String event = "\n Transactional Publish \n";
        log.info("Producing event: " + event);
        transaction2.writeEvent("", event);
        transaction2.flush();
        Thread.sleep(500);
    }
    CompletableFuture<Object> drop = CompletableFuture.supplyAsync(() -> {
        try {
            transaction2.abort();
        } catch (Exception e) {
            log.warn("Error aborting transaction", e);
        }
        return null;
    });
    drop.join();
    Transaction.Status txn2Status = transaction2.checkStatus();
    assertTrue(txn2Status == Transaction.Status.ABORTING || txn2Status == Transaction.Status.ABORTED);
    log.info("SUCCESS: successful in dropping transaction. Transaction status=" + txn2Status);
    Thread.sleep(2000);
    txn2Status = transaction2.checkStatus();
    assertTrue(txn2Status == Transaction.Status.ABORTED);
    log.info("SUCCESS: successfully aborted transaction. Transaction status=" + txn2Status);
    // endregion
    // region Successful timeout tests
    Transaction<String> tx1 = producer.beginTxn();
    Thread.sleep((long) (1.3 * txnTimeout));
    Transaction.Status txStatus = tx1.checkStatus();
    Assert.assertTrue(Transaction.Status.ABORTING == txStatus || Transaction.Status.ABORTED == txStatus);
    log.info("SUCCESS: successfully aborted transaction after timeout. Transaction status=" + txStatus);
    // endregion
    // region Ping failure due to controller going into disconnection state
    // Fill in these tests once we have controller.stop() implemented.
    System.exit(0);
}
Also used : TestingServer(org.apache.curator.test.TestingServer) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) TestingServerStarter(io.pravega.test.common.TestingServerStarter) Controller(io.pravega.client.stream.impl.Controller) Cleanup(lombok.Cleanup) PravegaConnectionListener(io.pravega.segmentstore.server.host.handler.PravegaConnectionListener) MockClientFactory(io.pravega.client.stream.mock.MockClientFactory) ServiceBuilder(io.pravega.segmentstore.server.store.ServiceBuilder) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) Transaction(io.pravega.client.stream.Transaction) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) Test(org.junit.Test)

Aggregations

StreamConfiguration (io.pravega.client.stream.StreamConfiguration)80 Test (org.junit.Test)67 ScalingPolicy (io.pravega.client.stream.ScalingPolicy)44 HashMap (java.util.HashMap)21 ArrayList (java.util.ArrayList)20 Controller (io.pravega.client.stream.impl.Controller)19 TestingServerStarter (io.pravega.test.common.TestingServerStarter)17 List (java.util.List)17 Cleanup (lombok.Cleanup)17 ConnectionFactoryImpl (io.pravega.client.netty.impl.ConnectionFactoryImpl)16 CompletableFuture (java.util.concurrent.CompletableFuture)16 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)16 TestingServer (org.apache.curator.test.TestingServer)16 Before (org.junit.Before)16 ClientFactory (io.pravega.client.ClientFactory)15 ClientFactoryImpl (io.pravega.client.stream.impl.ClientFactoryImpl)15 Executors (java.util.concurrent.Executors)15 After (org.junit.After)15 Assert.assertEquals (org.junit.Assert.assertEquals)15 Map (java.util.Map)14