Examples with StreamSegmentStore io.pravega.segmentstore.contracts.StreamSegmentStore used on opensource projects

Example 61 with StreamSegmentStore

use of io.pravega.segmentstore.contracts.StreamSegmentStore in project pravega by pravega.

the class AutoCheckpointTest method testCheckpointsOccur.

@Test(timeout = 30000)
public void testCheckpointsOccur() throws ReinitializationRequiredException, DurableDataLogException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerName = "reader";
    String readerGroup = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world: ";
    String scope = "Scope1";
    @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(10000).stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, null);
    streamManager.createReaderGroup(readerGroup, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    populateEvents(streamName, testString, clientFactory, serializer);
    AtomicLong fakeClock = new AtomicLong(0);
    @Cleanup EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroup, serializer, ReaderConfig.builder().build(), () -> fakeClock.get(), () -> fakeClock.get() / NANOS_PER_SECOND);
    int numRead = 0;
    int checkpointCount = 0;
    while (numRead < 100) {
        fakeClock.addAndGet(NANOS_PER_SECOND);
        EventRead<String> event = reader.readNextEvent(1000);
        if (event.isCheckpoint()) {
            checkpointCount++;
        } else {
            String message = event.getEvent();
            assertEquals(testString + numRead, message);
            numRead++;
        }
    }
    assertTrue("Count was " + checkpointCount, checkpointCount > 5);
    assertTrue("Count was " + checkpointCount, checkpointCount < 20);
}

Example 62 with StreamSegmentStore

use of io.pravega.segmentstore.contracts.StreamSegmentStore in project pravega by pravega.

the class CheckpointTest method testMoreReadersThanSegments.

@Test(timeout = 20000)
public void testMoreReadersThanSegments() throws ReinitializationRequiredException, InterruptedException, ExecutionException, TimeoutException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerGroupName = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world\n";
    String scope = "Scope1";
    StreamSegmentStore store = this.serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build());
    ReaderGroup readerGroup = streamManager.createReaderGroup(readerGroupName, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    EventStreamWriter<String> producer = clientFactory.createEventWriter(streamName, serializer, EventWriterConfig.builder().build());
    producer.writeEvent(testString);
    producer.writeEvent(testString);
    producer.writeEvent(testString);
    producer.flush();
    AtomicLong clock = new AtomicLong();
    @Cleanup EventStreamReader<String> reader1 = clientFactory.createReader("reader1", readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    @Cleanup EventStreamReader<String> reader2 = clientFactory.createReader("reader2", readerGroupName, serializer, ReaderConfig.builder().build(), clock::get, clock::get);
    clock.addAndGet(CLOCK_ADVANCE_INTERVAL);
    @Cleanup("shutdown") final InlineExecutor backgroundExecutor = new InlineExecutor();
    CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("Checkpoint", backgroundExecutor);
    assertFalse(checkpoint.isDone());
    EventRead<String> read = reader1.readNextEvent(60000);
    assertTrue(read.isCheckpoint());
    assertEquals("Checkpoint", read.getCheckpointName());
    assertNull(read.getEvent());
    read = reader2.readNextEvent(60000);
    assertTrue(read.isCheckpoint());
    assertEquals("Checkpoint", read.getCheckpointName());
    assertNull(read.getEvent());
    Checkpoint cpResult = checkpoint.get(5, TimeUnit.SECONDS);
    assertTrue(checkpoint.isDone());
    assertEquals("Checkpoint", cpResult.getName());
}

Example 63 with StreamSegmentStore

use of io.pravega.segmentstore.contracts.StreamSegmentStore in project pravega by pravega.

the class ControllerBootstrapTest method bootstrapTest.

@Test(timeout = 20000)
public void bootstrapTest() throws Exception {
    Controller controller = controllerWrapper.getController();
    // Create test scope. This operation should succeed.
    Boolean scopeStatus = controller.createScope(SCOPE).join();
    Assert.assertEquals(true, scopeStatus);
    // Try creating a stream. It should not complete until Pravega host has started.
    // After Pravega host starts, stream should be successfully created.
    StreamConfiguration streamConfiguration = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(ScalingPolicy.fixed(1)).build();
    CompletableFuture<Boolean> streamStatus = controller.createStream(streamConfiguration);
    Assert.assertTrue(!streamStatus.isDone());
    // Create transaction should fail.
    CompletableFuture<TxnSegments> txIdFuture = controller.createTransaction(new StreamImpl(SCOPE, STREAM), 10000, 30000);
    try {
        txIdFuture.join();
        Assert.fail();
    } catch (CompletionException ce) {
        Assert.assertEquals(IllegalStateException.class, ce.getCause().getClass());
        Assert.assertTrue("Expected failure", true);
    }
    // Now start Pravega service.
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    server = new PravegaConnectionListener(false, servicePort, store);
    server.startListening();
    // Ensure that create stream succeeds.
    try {
        Boolean status = streamStatus.join();
        Assert.assertEquals(true, status);
    } catch (CompletionException ce) {
        Assert.fail();
    }
    // Sleep for a while for initialize to complete
    boolean initialized = controllerWrapper.awaitTasksModuleInitialization(5000, TimeUnit.MILLISECONDS);
    Assert.assertTrue(initialized);
    // Now create transaction should succeed.
    txIdFuture = controller.createTransaction(new StreamImpl(SCOPE, STREAM), 10000, 30000);
    try {
        TxnSegments id = txIdFuture.join();
        Assert.assertNotNull(id);
    } catch (CompletionException ce) {
        Assert.fail();
    }
    controllerWrapper.awaitRunning();
}

Example 64 with StreamSegmentStore

use of io.pravega.segmentstore.contracts.StreamSegmentStore in project pravega by pravega.

the class ControllerFailoverTest method setup.

@Before
public void setup() throws Exception {
    // 1. Start ZK
    zkTestServer = new TestingServerStarter().start();
    // 2. Start Pravega SSS
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    server = new PravegaConnectionListener(false, servicePort, store);
    server.startListening();
}

Example 65 with StreamSegmentStore

use of io.pravega.segmentstore.contracts.StreamSegmentStore in project pravega by pravega.

the class MultiReadersEndToEndTest method runTestUsingMock.

private void runTestUsingMock(final Set<String> streamNames, final int numParallelReaders, final int numSegments) throws Exception {
    int servicePort = TestUtils.getAvailableListenPort();
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, servicePort, store);
    server.startListening();
    @Cleanup MockStreamManager streamManager = new MockStreamManager("scope", "localhost", servicePort);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    streamManager.createScope("scope");
    streamNames.stream().forEach(stream -> {
        streamManager.createStream("scope", stream, StreamConfiguration.builder().scope("scope").streamName(stream).scalingPolicy(ScalingPolicy.fixed(numSegments)).build());
        EventStreamWriter<Integer> eventWriter = clientFactory.createEventWriter(stream, new IntegerSerializer(), EventWriterConfig.builder().build());
        for (Integer i = 0; i < NUM_TEST_EVENTS; i++) {
            eventWriter.writeEvent(String.valueOf(i), i);
        }
        eventWriter.flush();
        log.info("Wrote {} events", NUM_TEST_EVENTS);
    });
    final String readerGroupName = "testReaderGroup";
    ReaderGroupConfig.ReaderGroupConfigBuilder builder = ReaderGroupConfig.builder();
    streamNames.forEach(s -> builder.stream(Stream.of("scope", s)));
    streamManager.createReaderGroup(readerGroupName, builder.build());
    Collection<Integer> read = readAllEvents(numParallelReaders, clientFactory, readerGroupName, numSegments);
    Assert.assertEquals(NUM_TEST_EVENTS * streamNames.size(), read.size());
    // Check unique events.
    Assert.assertEquals(NUM_TEST_EVENTS, new TreeSet<>(read).size());
    streamManager.deleteReaderGroup(readerGroupName);
}