Examples with ServiceBuilder io.pravega.segmentstore.server.store.ServiceBuilder used on opensource projects

Example 11 with ServiceBuilder

use of io.pravega.segmentstore.server.store.ServiceBuilder in project pravega by pravega.

the class AutoCheckpointTest method testOnlyOneOutstanding.

@Test(timeout = 30000)
public void testOnlyOneOutstanding() throws ReinitializationRequiredException, DurableDataLogException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerGroup = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world: ";
    String scope = "Scope1";
    @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(1000).stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, null);
    streamManager.createReaderGroup(readerGroup, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    populateEvents(streamName, testString, clientFactory, serializer);
    AtomicLong fakeClock = new AtomicLong(0);
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("reader1", readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
    @Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("reader2", readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
    int numRead = 0;
    int checkpointCount = 0;
    while (numRead < 100) {
        fakeClock.addAndGet(NANOS_PER_SECOND);
        EventRead<String> event = reader1.readNextEvent(1000);
        if (event.isCheckpoint()) {
            checkpointCount++;
        } else {
            String message = event.getEvent();
            assertEquals(testString + numRead, message);
            numRead++;
        }
    }
    assertEquals("As there is a second reader that does not pass the checkpoint, only one should occur", 1, checkpointCount);
}

Example 12 with ServiceBuilder

use of io.pravega.segmentstore.server.store.ServiceBuilder in project pravega by pravega.

the class ControllerStreamMetadataTest method setUp.

@Before
public void setUp() throws Exception {
    final int controllerPort = TestUtils.getAvailableListenPort();
    final String serviceHost = "localhost";
    final int servicePort = TestUtils.getAvailableListenPort();
    final int containerCount = 4;
    try {
        // 1. Start ZK
        this.zkTestServer = new TestingServerStarter().start();
        // 2. Start Pravega service.
        ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
        serviceBuilder.initialize();
        StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
        this.server = new PravegaConnectionListener(false, servicePort, store);
        this.server.startListening();
        // 3. Start controller
        this.controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), false, controllerPort, serviceHost, servicePort, containerCount);
        this.controllerWrapper.awaitRunning();
        this.controller = controllerWrapper.getController();
        this.streamConfiguration = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(ScalingPolicy.fixed(1)).build();
    } catch (Exception e) {
        log.error("Error during setup", e);
        throw e;
    }
}

Example 13 with ServiceBuilder

use of io.pravega.segmentstore.server.store.ServiceBuilder in project pravega by pravega.

the class PravegaRequestProcessorTest method testTransaction.

@Test(timeout = 20000)
public void testTransaction() throws Exception {
    String streamSegmentName = "testTxn";
    UUID txnid = UUID.randomUUID();
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, connection);
    processor.createSegment(new WireCommands.CreateSegment(0, streamSegmentName, WireCommands.CreateSegment.NO_SCALE, 0, ""));
    order.verify(connection).send(new WireCommands.SegmentCreated(0, streamSegmentName));
    processor.createTransaction(new WireCommands.CreateTransaction(1, streamSegmentName, txnid, ""));
    assertTrue(append(StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid), 1, store));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(2, streamSegmentName, txnid, ""));
    assertTrue(append(StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid), 2, store));
    order.verify(connection).send(new WireCommands.TransactionCreated(1, streamSegmentName, txnid));
    order.verify(connection).send(Mockito.argThat(t -> {
        return t instanceof TransactionInfo && ((TransactionInfo) t).exists();
    }));
    processor.commitTransaction(new WireCommands.CommitTransaction(3, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.TransactionCommitted(3, streamSegmentName, txnid));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(4, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.NoSuchSegment(4, StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid)));
    txnid = UUID.randomUUID();
    processor.createTransaction(new WireCommands.CreateTransaction(1, streamSegmentName, txnid, ""));
    assertTrue(append(StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid), 1, store));
    order.verify(connection).send(new WireCommands.TransactionCreated(1, streamSegmentName, txnid));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(2, streamSegmentName, txnid, ""));
    order.verify(connection).send(Mockito.argThat(t -> {
        return t instanceof TransactionInfo && ((TransactionInfo) t).exists();
    }));
    processor.abortTransaction(new WireCommands.AbortTransaction(3, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.TransactionAborted(3, streamSegmentName, txnid));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(4, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.NoSuchSegment(4, StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid)));
    // Verify the case when the transaction segment is already sealed. This simulates the case when the process
    // crashed after sealing, but before issuing the merge.
    txnid = UUID.randomUUID();
    processor.createTransaction(new WireCommands.CreateTransaction(1, streamSegmentName, txnid, ""));
    assertTrue(append(StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid), 1, store));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(2, streamSegmentName, txnid, ""));
    assertTrue(append(StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid), 2, store));
    // Seal the transaction in the SegmentStore.
    String txnName = StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid);
    store.sealStreamSegment(txnName, Duration.ZERO).join();
    processor.commitTransaction(new WireCommands.CommitTransaction(3, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.TransactionCommitted(3, streamSegmentName, txnid));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(4, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.NoSuchSegment(4, StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid)));
    order.verifyNoMoreInteractions();
}

Example 14 with ServiceBuilder

use of io.pravega.segmentstore.server.store.ServiceBuilder in project pravega by pravega.

the class PravegaRequestProcessorTest method testCreateSegment.

@Test(timeout = 20000)
public void testCreateSegment() throws Exception {
    // Set up PravegaRequestProcessor instance to execute requests against
    String streamSegmentName = "testCreateSegment";
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, connection);
    // Execute and Verify createSegment/getStreamSegmentInfo calling stack is executed as design.
    processor.createSegment(new WireCommands.CreateSegment(1, streamSegmentName, WireCommands.CreateSegment.NO_SCALE, 0, ""));
    assertTrue(append(streamSegmentName, 1, store));
    processor.getStreamSegmentInfo(new WireCommands.GetStreamSegmentInfo(1, streamSegmentName, ""));
    assertTrue(append(streamSegmentName, 2, store));
    order.verify(connection).send(new WireCommands.SegmentCreated(1, streamSegmentName));
    order.verify(connection).send(Mockito.any(WireCommands.StreamSegmentInfo.class));
    // TestCreateSealDelete may executed before this test case,
    // so createSegmentStats may record 1 or 2 createSegment operation here.
    OpStatsData createSegmentStats = processor.getCreateStreamSegment().toOpStatsData();
    assertNotEquals(0, createSegmentStats.getNumSuccessfulEvents());
    assertEquals(0, createSegmentStats.getNumFailedEvents());
}

Example 15 with ServiceBuilder

use of io.pravega.segmentstore.server.store.ServiceBuilder in project pravega by pravega.

the class EndToEndAutoScaleUpTest method main.

public static void main(String[] args) throws Exception {
    try {
        @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
        int port = Config.SERVICE_PORT;
        @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), port, false);
        Controller controller = controllerWrapper.getController();
        ClientFactory internalCF = new ClientFactoryImpl(NameUtils.INTERNAL_SCOPE_NAME, controller, new ConnectionFactoryImpl(ClientConfig.builder().build()));
        ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
        serviceBuilder.initialize();
        StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
        @Cleanup SegmentStatsFactory segmentStatsFactory = new SegmentStatsFactory();
        SegmentStatsRecorder statsRecorder = segmentStatsFactory.createSegmentStatsRecorder(store, internalCF, AutoScalerConfig.builder().with(AutoScalerConfig.MUTE_IN_SECONDS, 0).with(AutoScalerConfig.COOLDOWN_IN_SECONDS, 0).build());
        @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, "localhost", 12345, store, statsRecorder, null, null, null);
        server.startListening();
        controllerWrapper.awaitRunning();
        controllerWrapper.getControllerService().createScope("test").get();
        controller.createStream(CONFIG).get();
        @Cleanup MockClientFactory clientFactory = new MockClientFactory("test", controller);
        // Mocking pravega service by putting scale up and scale down requests for the stream
        EventStreamWriter<String> test = clientFactory.createEventWriter("test", new JavaSerializer<>(), EventWriterConfig.builder().build());
        // keep writing. Scale should happen
        long start = System.currentTimeMillis();
        char[] chars = new char[1];
        Arrays.fill(chars, 'a');
        String str = new String(chars);
        CompletableFuture.runAsync(() -> {
            while (System.currentTimeMillis() - start < Duration.ofMinutes(3).toMillis()) {
                try {
                    test.writeEvent("0", str).get();
                } catch (Throwable e) {
                    System.err.println("test exception writing events " + e.getMessage());
                    break;
                }
            }
        });
        Retry.withExpBackoff(10, 10, 100, 10000).retryingOn(NotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments("test", "test").thenAccept(streamSegments -> {
            if (streamSegments.getSegments().size() > 3) {
                System.err.println("Success");
                log.info("Success");
                System.exit(0);
            } else {
                throw new NotDoneException();
            }
        }), Executors.newSingleThreadScheduledExecutor()).exceptionally(e -> {
            System.err.println("Failure");
            log.error("Failure");
            System.exit(1);
            return null;
        }).get();
    } catch (Throwable e) {
        System.err.print("Test failed with exception: " + e.getMessage());
        System.exit(-1);
    }
    System.exit(0);
}