Search in sources :

Example 91 with Controller

use of io.pravega.client.control.impl.Controller in project pravega by pravega.

the class SegmentSelectorTest method testSameRoutingKey.

@Test
public void testSameRoutingKey() {
    Controller controller = Mockito.mock(Controller.class);
    SegmentOutputStreamFactory factory = Mockito.mock(SegmentOutputStreamFactory.class);
    SegmentSelector selector = new SegmentSelector(new StreamImpl(scope, streamName), controller, factory, config, DelegationTokenProviderFactory.createWithEmptyToken());
    TreeMap<Double, SegmentWithRange> segments = new TreeMap<>();
    addNewSegment(segments, 0, 0.0, 0.25);
    addNewSegment(segments, 1, 0.25, 0.5);
    addNewSegment(segments, 2, 0.5, 0.75);
    addNewSegment(segments, 3, 0.75, 1.0);
    StreamSegments streamSegments = new StreamSegments(segments);
    when(controller.getCurrentSegments(scope, streamName)).thenReturn(CompletableFuture.completedFuture(streamSegments));
    selector.refreshSegmentEventWriters(segmentSealedCallback);
    int[] counts = new int[4];
    Arrays.fill(counts, 0);
    for (int i = 0; i < 20; i++) {
        Segment segment = selector.getSegmentForEvent("Foo");
        assertNotNull(segment);
        counts[NameUtils.getSegmentNumber(segment.getSegmentId())]++;
    }
    assertArrayEquals(new int[] { 20, 0, 0, 0 }, counts);
}
Also used : SegmentOutputStreamFactory(io.pravega.client.segment.impl.SegmentOutputStreamFactory) Controller(io.pravega.client.control.impl.Controller) TreeMap(java.util.TreeMap) Segment(io.pravega.client.segment.impl.Segment) Test(org.junit.Test)

Example 92 with Controller

use of io.pravega.client.control.impl.Controller in project pravega by pravega.

the class EventStreamReaderTest method testTimeWindow.

@Test
public void testTimeWindow() throws SegmentSealedException {
    String scope = "scope";
    String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    String groupName = "readerGroup";
    String readerGroupStream = NameUtils.getStreamForReaderGroup(groupName);
    String markStream = NameUtils.getMarkStreamForStream(streamName);
    // Create factories
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    @Cleanup MockClientFactory clientFactory = new MockClientFactory(scope, segmentStreamFactory);
    MockController controller = (MockController) clientFactory.getController();
    @Cleanup InlineExecutor executor = new InlineExecutor();
    // Create streams
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).build());
    controller.createStream(scope, readerGroupStream, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build());
    // Reader group state synchronizer
    ReaderGroupConfig config = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(stream).build();
    StateSynchronizer<ReaderGroupState> sync = clientFactory.createStateSynchronizer(readerGroupStream, new ReaderGroupStateUpdatesSerializer(), new ReaderGroupStateInitSerializer(), SynchronizerConfig.builder().build());
    // Watermark reader/writer
    @Cleanup RevisionedStreamClient<Watermark> markWriter = clientFactory.createRevisionedStreamClient(markStream, new WatermarkSerializer(), SynchronizerConfig.builder().build());
    @Cleanup WatermarkReaderImpl markReader = new WatermarkReaderImpl(stream, markWriter, executor);
    // Initialize reader group state
    Map<SegmentWithRange, Long> segments = ReaderGroupImpl.getSegmentsForStreams(controller, config);
    sync.initialize(new ReaderGroupState.ReaderGroupStateInit(config, segments, getEndSegmentsForStreams(config), false));
    // Data segment writers
    Segment segment1 = new Segment(scope, streamName, 0);
    Segment segment2 = new Segment(scope, streamName, 1);
    @Cleanup SegmentOutputStream stream1 = segmentStreamFactory.createOutputStreamForSegment(segment1, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    @Cleanup SegmentOutputStream stream2 = segmentStreamFactory.createOutputStreamForSegment(segment2, segmentSealedCallback, writerConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    // Write stream data
    writeInt(stream1, 1);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    writeInt(stream2, 2);
    // Write mark data
    val r1 = new SegmentWithRange(segment1, 0, 0.5).convert();
    val r2 = new SegmentWithRange(segment2, 0.5, 1).convert();
    markWriter.writeUnconditionally(new Watermark(0L, 99L, ImmutableMap.of(r1, 0L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(100L, 199L, ImmutableMap.of(r1, 12L, r2, 0L)));
    markWriter.writeUnconditionally(new Watermark(200L, 299L, ImmutableMap.of(r1, 12L, r2, 12L)));
    markWriter.writeUnconditionally(new Watermark(300L, 399L, ImmutableMap.of(r1, 12L, r2, 24L)));
    markWriter.writeUnconditionally(new Watermark(400L, 499L, ImmutableMap.of(r1, 12L, r2, 36L)));
    // Create reader
    AtomicLong clock = new AtomicLong();
    ReaderGroupStateManager groupState = new ReaderGroupStateManager(scope, groupName, "reader1", sync, controller, clock::get);
    groupState.initializeReader(0);
    @Cleanup EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory, new ByteArraySerializer(), groupState, new Orderer(), clock::get, ReaderConfig.builder().build(), ImmutableMap.of(stream, markReader), Mockito.mock(Controller.class));
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    EventRead<byte[]> event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    TimeWindow timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(199, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(1, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(0, timeWindow.getLowerTimeBound().longValue());
    assertEquals(299, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(200, timeWindow.getLowerTimeBound().longValue());
    assertEquals(399, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(2, readInt(event));
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(300, timeWindow.getLowerTimeBound().longValue());
    assertEquals(499, timeWindow.getUpperTimeBound().longValue());
    clock.addAndGet(ReaderGroupStateManager.UPDATE_WINDOW.toNanos());
    event = reader.readNextEvent(100);
    assertEquals(null, event.getEvent());
    timeWindow = reader.getCurrentTimeWindow(Stream.of(scope, streamName));
    assertEquals(400, timeWindow.getLowerTimeBound().longValue());
    assertEquals(null, timeWindow.getUpperTimeBound());
}
Also used : ReaderGroupStateInitSerializer(io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateInitSerializer) Cleanup(lombok.Cleanup) MockClientFactory(io.pravega.client.stream.mock.MockClientFactory) Segment(io.pravega.client.segment.impl.Segment) InlineExecutor(io.pravega.test.common.InlineExecutor) SegmentOutputStream(io.pravega.client.segment.impl.SegmentOutputStream) Stream(io.pravega.client.stream.Stream) SegmentOutputStream(io.pravega.client.segment.impl.SegmentOutputStream) ReaderGroupConfig(io.pravega.client.stream.ReaderGroupConfig) lombok.val(lombok.val) MockSegmentStreamFactory(io.pravega.client.stream.mock.MockSegmentStreamFactory) WatermarkSerializer(io.pravega.client.watermark.WatermarkSerializer) MockController(io.pravega.client.stream.mock.MockController) Controller(io.pravega.client.control.impl.Controller) TimeWindow(io.pravega.client.stream.TimeWindow) ReaderGroupStateUpdatesSerializer(io.pravega.client.admin.impl.ReaderGroupManagerImpl.ReaderGroupStateUpdatesSerializer) AtomicLong(java.util.concurrent.atomic.AtomicLong) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) AtomicLong(java.util.concurrent.atomic.AtomicLong) MockController(io.pravega.client.stream.mock.MockController) Watermark(io.pravega.shared.watermarks.Watermark) Test(org.junit.Test)

Example 93 with Controller

use of io.pravega.client.control.impl.Controller in project pravega by pravega.

the class MetadataScalabilityTest method scale.

List<List<Segment>> scale(ControllerImpl controller) {
    int numSegments = getStreamConfig().getScalingPolicy().getMinNumSegments();
    int scalesToPerform = getScalesToPerform();
    // manually scale the stream SCALES_TO_PERFORM times
    Stream stream = new StreamImpl(SCOPE, getStreamName());
    AtomicInteger counter = new AtomicInteger(0);
    List<List<Segment>> listOfEpochs = new LinkedList<>();
    CompletableFuture<Void> scaleFuture = Futures.loop(() -> counter.incrementAndGet() <= scalesToPerform, () -> controller.getCurrentSegments(SCOPE, streamName).thenCompose(segments -> {
        ArrayList<Segment> sorted = Lists.newArrayList(segments.getSegments().stream().sorted(Comparator.comparingInt(x -> NameUtils.getSegmentNumber(x.getSegmentId()) % numSegments)).collect(Collectors.toList()));
        listOfEpochs.add(sorted);
        // note: with SCALES_TO_PERFORM < numSegments, we can use the segment number as the index
        // into the range map
        Pair<List<Long>, Map<Double, Double>> scaleInput = getScaleInput(sorted);
        List<Long> segmentsToSeal = scaleInput.getKey();
        Map<Double, Double> newRanges = scaleInput.getValue();
        return controller.scaleStream(stream, segmentsToSeal, newRanges, executorService).getFuture().thenAccept(scaleStatus -> {
            log.info("scale stream for epoch {} completed with status {}", counter.get(), scaleStatus);
            assert scaleStatus;
        });
    }), executorService);
    scaleFuture.join();
    return listOfEpochs;
}
Also used : Segment(io.pravega.client.segment.impl.Segment) StreamCut(io.pravega.client.stream.StreamCut) StreamImpl(io.pravega.client.stream.impl.StreamImpl) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) Random(java.util.Random) CompletableFuture(java.util.concurrent.CompletableFuture) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) Pair(org.apache.commons.lang3.tuple.Pair) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Stream(io.pravega.client.stream.Stream) StreamCutImpl(io.pravega.client.stream.impl.StreamCutImpl) Map(java.util.Map) After(org.junit.After) Timeout(org.junit.rules.Timeout) URI(java.net.URI) LinkedList(java.util.LinkedList) Before(org.junit.Before) Environment(io.pravega.test.system.framework.Environment) NameUtils(io.pravega.shared.NameUtils) Assert.assertTrue(org.junit.Assert.assertTrue) Collectors(java.util.stream.Collectors) ExecutionException(java.util.concurrent.ExecutionException) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Rule(org.junit.Rule) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) ExecutorServiceHelpers(io.pravega.common.concurrent.ExecutorServiceHelpers) Comparator(java.util.Comparator) Controller(io.pravega.client.control.impl.Controller) Futures(io.pravega.common.concurrent.Futures) SystemTestRunner(io.pravega.test.system.framework.SystemTestRunner) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) StreamImpl(io.pravega.client.stream.impl.StreamImpl) Stream(io.pravega.client.stream.Stream) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) List(java.util.List) HashMap(java.util.HashMap) Map(java.util.Map) Pair(org.apache.commons.lang3.tuple.Pair)

Example 94 with Controller

use of io.pravega.client.control.impl.Controller in project pravega by pravega.

the class MetadataScalabilityTest method setup.

/**
 * Invoke the createStream method, ensure we are able to create stream.
 *
 * @throws InterruptedException if interrupted
 * @throws ExecutionException   if error in create stream
 */
@Before
public void setup() throws InterruptedException, ExecutionException {
    // create a scope
    Controller controller = getController();
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(5, "Scalability-main");
    Boolean createScopeStatus = controller.createScope(SCOPE).get();
    log.debug("create scope status {}", createScopeStatus);
    // create a stream
    Boolean createStreamStatus = controller.createStream(SCOPE, getStreamName(), getStreamConfig()).get();
    log.debug("create stream status for scale up stream {}", createStreamStatus);
}
Also used : Controller(io.pravega.client.control.impl.Controller) Before(org.junit.Before)

Example 95 with Controller

use of io.pravega.client.control.impl.Controller in project pravega by pravega.

the class AutoScaleTest method scaleUpTxnTest.

/**
 * Invoke the scale up Test with transactional writes. Produce traffic from multiple writers in parallel. Each
 * writer writes using transactions. The test will periodically check if a scale event has occurred by talking to
 * controller via controller client.
 *
 * @throws InterruptedException if interrupted
 * @throws URISyntaxException   If URI is invalid
 */
private CompletableFuture<Void> scaleUpTxnTest() {
    ControllerImpl controller = getController();
    final AtomicBoolean exit = new AtomicBoolean(false);
    ClientFactoryImpl clientFactory = getClientFactory();
    startWritingIntoTxn(clientFactory.createTransactionalEventWriter("writer", SCALE_UP_TXN_STREAM_NAME, new JavaSerializer<>(), EventWriterConfig.builder().build()), exit);
    // overall wait for test to complete in 260 seconds (4.2 minutes) or scale up, whichever happens first.
    return Retry.withExpBackoff(10, 10, 30, Duration.ofSeconds(10).toMillis()).retryingOn(ScaleOperationNotDoneException.class).throwingOn(RuntimeException.class).runAsync(() -> controller.getCurrentSegments(SCOPE, SCALE_UP_TXN_STREAM_NAME).thenAccept(x -> {
        if (x.getSegments().size() == 1) {
            throw new ScaleOperationNotDoneException();
        } else {
            log.info("txn test scale up done successfully");
            exit.set(true);
        }
    }), scaleExecutorService);
}
Also used : StreamImpl(io.pravega.client.stream.impl.StreamImpl) Retry(io.pravega.common.util.Retry) URISyntaxException(java.net.URISyntaxException) RunWith(org.junit.runner.RunWith) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) Duration(java.time.Duration) Map(java.util.Map) After(org.junit.After) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Timeout(org.junit.rules.Timeout) URI(java.net.URI) Before(org.junit.Before) EventWriterConfig(io.pravega.client.stream.EventWriterConfig) Environment(io.pravega.test.system.framework.Environment) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) ExecutionException(java.util.concurrent.ExecutionException) Slf4j(lombok.extern.slf4j.Slf4j) Rule(org.junit.Rule) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) ExecutorServiceHelpers(io.pravega.common.concurrent.ExecutorServiceHelpers) Collections(java.util.Collections) Controller(io.pravega.client.control.impl.Controller) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) Futures(io.pravega.common.concurrent.Futures) SystemTestRunner(io.pravega.test.system.framework.SystemTestRunner) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) ControllerImpl(io.pravega.client.control.impl.ControllerImpl) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer)

Aggregations

Controller (io.pravega.client.control.impl.Controller)120 Test (org.junit.Test)95 Cleanup (lombok.Cleanup)81 Segment (io.pravega.client.segment.impl.Segment)53 EventWriterConfig (io.pravega.client.stream.EventWriterConfig)50 StreamConfiguration (io.pravega.client.stream.StreamConfiguration)47 Stream (io.pravega.client.stream.Stream)37 CompletableFuture (java.util.concurrent.CompletableFuture)35 SegmentOutputStreamFactory (io.pravega.client.segment.impl.SegmentOutputStreamFactory)34 SocketConnectionFactoryImpl (io.pravega.client.connection.impl.SocketConnectionFactoryImpl)33 HashMap (java.util.HashMap)29 ClientConfig (io.pravega.client.ClientConfig)28 ClientFactoryImpl (io.pravega.client.stream.impl.ClientFactoryImpl)28 StreamImpl (io.pravega.client.stream.impl.StreamImpl)25 ReaderGroupManager (io.pravega.client.admin.ReaderGroupManager)24 Slf4j (lombok.extern.slf4j.Slf4j)24 Before (org.junit.Before)23 ConnectionFactory (io.pravega.client.connection.impl.ConnectionFactory)22 ConnectionPoolImpl (io.pravega.client.connection.impl.ConnectionPoolImpl)21 ScalingPolicy (io.pravega.client.stream.ScalingPolicy)21