Examples with StreamImpl io.pravega.client.stream.impl.StreamImpl used on opensource projects

Example 16 with StreamImpl

use of io.pravega.client.stream.impl.StreamImpl in project pravega by pravega.

the class WatermarkWorkflowTest method testWatermarkClient.

@Test(timeout = 10000L)
public void testWatermarkClient() {
    Stream stream = new StreamImpl("scope", "stream");
    SynchronizerClientFactory clientFactory = spy(SynchronizerClientFactory.class);
    @Cleanup MockRevisionedStreamClient revisionedClient = new MockRevisionedStreamClient();
    doAnswer(x -> revisionedClient).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    @Cleanup PeriodicWatermarking.WatermarkClient client = new PeriodicWatermarking.WatermarkClient(stream, clientFactory);
    // iteration 1 ==> null -> w1
    client.reinitialize();
    // There is no watermark in the stream. All values should be null and all writers active and participating.
    assertEquals(revisionedClient.getMark(), MockRevision.EMPTY);
    assertTrue(revisionedClient.watermarks.isEmpty());
    assertEquals(client.getPreviousWatermark(), Watermark.EMPTY);
    Map.Entry<String, WriterMark> entry0 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(0L, ImmutableMap.of()));
    Map.Entry<String, WriterMark> entry1 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(1L, ImmutableMap.of()));
    Map.Entry<String, WriterMark> entry2 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(2L, ImmutableMap.of()));
    Map.Entry<String, WriterMark> entry3 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(3L, ImmutableMap.of()));
    Map.Entry<String, WriterMark> entry4 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(4L, ImmutableMap.of()));
    Map.Entry<String, WriterMark> entry5 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(5L, ImmutableMap.of()));
    assertTrue(client.isWriterActive(entry0, 0L));
    assertTrue(client.isWriterParticipating(0L));
    Watermark first = new Watermark(1L, 2L, ImmutableMap.of());
    client.completeIteration(first);
    // iteration 2 : do not emit ==> w1 -> w1
    client.reinitialize();
    // There is one watermark. All writers should be active and writers greater than last watermark should be participating
    assertEquals(revisionedClient.getMark(), MockRevision.EMPTY);
    assertEquals(revisionedClient.watermarks.size(), 1);
    assertEquals(client.getPreviousWatermark(), first);
    assertTrue(client.isWriterActive(entry2, 0L));
    assertFalse(client.isWriterActive(entry1, 0L));
    assertTrue(client.isWriterTracked(entry1.getKey()));
    assertFalse(client.isWriterParticipating(1L));
    assertTrue(client.isWriterParticipating(2L));
    // dont emit a watermark. Everything stays same as before.
    client.completeIteration(null);
    // iteration 3 : emit ==> w1 -> w1 w2
    client.reinitialize();
    // There is one watermark. All writers should be active and writers greater than last watermark should be participating
    assertEquals(revisionedClient.getMark(), MockRevision.EMPTY);
    assertEquals(revisionedClient.watermarks.size(), 1);
    assertEquals(client.getPreviousWatermark(), first);
    assertTrue(client.isWriterActive(entry2, 0L));
    assertFalse(client.isWriterParticipating(1L));
    assertTrue(client.isWriterParticipating(2L));
    // emit second watermark
    Watermark second = new Watermark(2L, 3L, ImmutableMap.of());
    client.completeIteration(second);
    // iteration 4: do not emit ==> w1 w2 -> w1 w2
    client.reinitialize();
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(0).getKey());
    assertEquals(2, revisionedClient.watermarks.size());
    assertEquals(client.getPreviousWatermark(), second);
    assertFalse(client.isWriterActive(entry2, 0L));
    assertTrue(client.isWriterTracked(entry2.getKey()));
    assertTrue(client.isWriterActive(entry3, 0L));
    assertFalse(client.isWriterParticipating(2L));
    assertTrue(client.isWriterParticipating(3L));
    assertTrue(client.isWriterActive(entry0, 1000L));
    assertTrue(client.isWriterTracked(entry0.getKey()));
    // dont emit a watermark but complete this iteration.
    client.completeIteration(null);
    // iteration 6: emit ==> w1 w2 -> w1 w2 w3
    client.reinitialize();
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(0).getKey());
    assertEquals(2, revisionedClient.watermarks.size());
    assertEquals(client.getPreviousWatermark(), second);
    assertTrue(client.isWriterActive(entry3, 0L));
    assertFalse(client.isWriterTracked(entry3.getKey()));
    assertFalse(client.isWriterParticipating(2L));
    assertTrue(client.isWriterParticipating(3L));
    // emit third watermark
    Watermark third = new Watermark(3L, 4L, ImmutableMap.of());
    client.completeIteration(third);
    // iteration 7: do not emit ==> w1 w2 w3 -> w1 w2 w3
    client.reinitialize();
    // active writers should be ahead of first watermark. participating writers should be ahead of second watermark
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(1).getKey());
    assertEquals(3, revisionedClient.watermarks.size());
    assertEquals(client.getPreviousWatermark(), third);
    assertFalse(client.isWriterActive(entry3, 0L));
    assertTrue(client.isWriterActive(entry4, 0L));
    assertFalse(client.isWriterParticipating(3L));
    assertTrue(client.isWriterParticipating(4L));
    client.completeIteration(null);
    // iteration 8 : emit ==> w2 w3 -> w2 w3 w4
    client.reinitialize();
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(1).getKey());
    // window = w2 w3
    assertEquals(revisionedClient.watermarks.size(), 3);
    assertEquals(client.getPreviousWatermark(), third);
    assertFalse(client.isWriterActive(entry3, 0L));
    assertTrue(client.isWriterActive(entry4, 0L));
    assertFalse(client.isWriterParticipating(3L));
    assertTrue(client.isWriterParticipating(4L));
    // emit fourth watermark
    Watermark fourth = new Watermark(4L, 5L, ImmutableMap.of());
    client.completeIteration(fourth);
    // iteration 9: do not emit ==> w1 w2 w3 w4 -> w1 w2 w3 w4.. check writer timeout
    client.reinitialize();
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(2).getKey());
    assertEquals(revisionedClient.watermarks.size(), 4);
    assertEquals(client.getPreviousWatermark(), fourth);
    assertFalse(client.isWriterActive(entry3, 0L));
    assertTrue(client.isWriterTracked(entry4.getKey()));
    assertFalse(client.isWriterParticipating(4L));
    assertTrue(client.isWriterParticipating(5L));
    // verify that writer is active if we specify a higher timeout
    assertTrue(client.isWriterActive(entry1, 1000L));
    assertTrue(client.isWriterTracked(entry1.getKey()));
    // now that the writer is being tracked
    assertFalse(Futures.delayedTask(() -> client.isWriterActive(entry1, 1L), Duration.ofSeconds(1), executor).join());
    assertTrue(client.isWriterTracked(entry1.getKey()));
    // dont emit a watermark but complete this iteration. This should shrink the window again.
    client.completeIteration(null);
    // iteration 10
    client.reinitialize();
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(2).getKey());
    assertEquals(revisionedClient.watermarks.size(), 4);
    assertEquals(client.getPreviousWatermark(), fourth);
    assertFalse(client.isWriterActive(entry4, 0L));
    assertTrue(client.isWriterActive(entry5, 0L));
    assertFalse(client.isWriterParticipating(4L));
    assertTrue(client.isWriterParticipating(5L));
}

Example 17 with StreamImpl

use of io.pravega.client.stream.impl.StreamImpl in project pravega by pravega.

the class WatermarkWorkflowTest method testWatermarkingWorkflow.

@Test(timeout = 30000L)
public void testWatermarkingWorkflow() {
    SynchronizerClientFactory clientFactory = spy(SynchronizerClientFactory.class);
    ConcurrentHashMap<String, MockRevisionedStreamClient> revisionedStreamClientMap = new ConcurrentHashMap<>();
    doAnswer(x -> {
        String streamName = x.getArgument(0);
        return revisionedStreamClientMap.compute(streamName, (s, rsc) -> {
            if (rsc != null) {
                return rsc;
            } else {
                return new MockRevisionedStreamClient();
            }
        });
    }).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    @Cleanup PeriodicWatermarking periodicWatermarking = new PeriodicWatermarking(streamMetadataStore, bucketStore, sp -> clientFactory, executor, new RequestTracker(false));
    String streamName = "stream";
    String scope = "scope";
    streamMetadataStore.createScope(scope, null, executor).join();
    streamMetadataStore.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(3)).timestampAggregationTimeout(10000L).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, streamName, State.ACTIVE, null, executor).join();
    // set minimum number of segments to 1
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).timestampAggregationTimeout(10000L).build();
    streamMetadataStore.startUpdateConfiguration(scope, streamName, config, null, executor).join();
    VersionedMetadata<StreamConfigurationRecord> configRecord = streamMetadataStore.getConfigurationRecord(scope, streamName, null, executor).join();
    streamMetadataStore.completeUpdateConfiguration(scope, streamName, configRecord, null, executor).join();
    // 2. note writer1, writer2, writer3 marks
    // writer 1 reports segments 0, 1.
    // writer 2 reports segments 1, 2,
    // writer 3 reports segment 0, 2
    String writer1 = "writer1";
    Map<Long, Long> map1 = ImmutableMap.of(0L, 100L, 1L, 200L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 100L, map1, null, executor).join();
    String writer2 = "writer2";
    Map<Long, Long> map2 = ImmutableMap.of(1L, 100L, 2L, 200L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer2, 101L, map2, null, executor).join();
    String writer3 = "writer3";
    Map<Long, Long> map3 = ImmutableMap.of(2L, 100L, 0L, 200L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer3, 102L, map3, null, executor).join();
    // 3. run watermarking workflow.
    StreamImpl stream = new StreamImpl(scope, streamName);
    periodicWatermarking.watermark(stream).join();
    // verify that a watermark has been emitted.
    // this should emit a watermark that contains all three segments with offsets = 200L
    // and timestamp = 100L
    MockRevisionedStreamClient revisionedClient = revisionedStreamClientMap.get(NameUtils.getMarkStreamForStream(streamName));
    assertEquals(revisionedClient.watermarks.size(), 1);
    Watermark watermark = revisionedClient.watermarks.get(0).getValue();
    assertEquals(watermark.getLowerTimeBound(), 100L);
    assertEquals(watermark.getStreamCut().size(), 3);
    assertEquals(getSegmentOffset(watermark, 0L), 200L);
    assertEquals(getSegmentOffset(watermark, 1L), 200L);
    assertEquals(getSegmentOffset(watermark, 2L), 200L);
    // send positions only on segment 1 and segment 2. nothing on segment 0.
    map1 = ImmutableMap.of(1L, 300L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 200L, map1, null, executor).join();
    map2 = ImmutableMap.of(1L, 100L, 2L, 300L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer2, 201L, map2, null, executor).join();
    map3 = ImmutableMap.of(2L, 300L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer3, 202L, map3, null, executor).join();
    // run watermark workflow. this will emit a watermark with time = 200L and streamcut = 0 -> 200L, 1 -> 300L, 2 -> 300L
    periodicWatermarking.watermark(stream).join();
    assertEquals(revisionedClient.watermarks.size(), 2);
    watermark = revisionedClient.watermarks.get(1).getValue();
    assertEquals(watermark.getLowerTimeBound(), 200L);
    assertEquals(watermark.getStreamCut().size(), 3);
    assertEquals(getSegmentOffset(watermark, 0L), 200L);
    assertEquals(getSegmentOffset(watermark, 1L), 300L);
    assertEquals(getSegmentOffset(watermark, 2L), 300L);
    // scale stream 0, 1, 2 -> 3, 4
    scaleStream(streamName, scope);
    // writer 1 reports segments 0, 1.
    // writer 2 reports segments 1, 2
    // writer 3 reports segment 3
    map1 = ImmutableMap.of(0L, 300L, 1L, 400L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 302L, map1, null, executor).join();
    map2 = ImmutableMap.of(1L, 100L, 2L, 400L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer2, 301L, map2, null, executor).join();
    long segment3 = NameUtils.computeSegmentId(3, 1);
    long segment4 = NameUtils.computeSegmentId(4, 1);
    map3 = ImmutableMap.of(segment3, 100L);
    // writer 3 has lowest reported time.
    streamMetadataStore.noteWriterMark(scope, streamName, writer3, 300L, map3, null, executor).join();
    // run watermark workflow. this will emit a watermark with time = 300L and streamcut = 3 -> 100L, 4 -> 0L
    periodicWatermarking.watermark(stream).join();
    assertEquals(revisionedClient.watermarks.size(), 3);
    watermark = revisionedClient.watermarks.get(2).getValue();
    assertEquals(watermark.getLowerTimeBound(), 300L);
    assertEquals(watermark.getStreamCut().size(), 2);
    assertEquals(getSegmentOffset(watermark, segment3), 100L);
    assertEquals(getSegmentOffset(watermark, segment4), 0L);
    // report complete positions from writers.
    // writer 1 reports 0, 1, 2
    // writer 2 reports 0, 1, 2
    // writer 3 doesnt report.
    map1 = ImmutableMap.of(0L, 400L, 1L, 400L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 400L, map1, null, executor).join();
    map2 = ImmutableMap.of(1L, 100L, 2L, 400L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer2, 401L, map2, null, executor).join();
    // run watermark workflow. there shouldn't be a watermark emitted because writer 3 is active and has not reported a time.
    periodicWatermarking.watermark(stream).join();
    assertEquals(revisionedClient.watermarks.size(), 3);
    // even though writer3 is excluded from computation, its mark is still not removed because it is still active
    WriterMark writer3Mark = streamMetadataStore.getWriterMark(scope, streamName, writer3, null, executor).join();
    assertTrue(writer3Mark.isAlive());
    assertEquals(writer3Mark.getTimestamp(), 300L);
    // report shutdown of writer 3
    streamMetadataStore.shutdownWriter(scope, streamName, writer3, null, executor).join();
    writer3Mark = streamMetadataStore.getWriterMark(scope, streamName, writer3, null, executor).join();
    assertFalse(writer3Mark.isAlive());
    assertEquals(writer3Mark.getTimestamp(), 300L);
    // now a watermark should be generated. Time should be advanced. But watermark's stream cut is already ahead of writer's
    // positions so stream cut should not advance.
    // Also writer 3 being inactive and shutdown, should be removed.
    periodicWatermarking.watermark(stream).join();
    assertEquals(revisionedClient.watermarks.size(), 4);
    watermark = revisionedClient.watermarks.get(3).getValue();
    assertEquals(watermark.getLowerTimeBound(), 400L);
    assertEquals(watermark.getStreamCut().size(), 2);
    assertEquals(getSegmentOffset(watermark, segment3), 100L);
    assertEquals(getSegmentOffset(watermark, segment4), 0L);
    AssertExtensions.assertFutureThrows("Writer 3 should have been removed from store", streamMetadataStore.getWriterMark(scope, streamName, writer3, null, executor), e -> Exceptions.unwrap(e) instanceof StoreException.DataNotFoundException);
    // writer 1, 2 and 3 report marks. With writer 3 reporting mark on segment 4. Writer3 will get added again
    map1 = ImmutableMap.of(0L, 500L, 1L, 500L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 500L, map1, null, executor).join();
    map2 = ImmutableMap.of(1L, 100L, 2L, 500L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer2, 501L, map2, null, executor).join();
    map3 = ImmutableMap.of(segment4, 500L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer3, 502L, map3, null, executor).join();
    // run watermarking workflow. It should generate watermark that includes segments 3 -> 100L and 4 -> 500L with time 500L
    periodicWatermarking.watermark(stream).join();
    assertEquals(revisionedClient.watermarks.size(), 5);
    watermark = revisionedClient.watermarks.get(4).getValue();
    assertEquals(watermark.getLowerTimeBound(), 500L);
    assertEquals(watermark.getStreamCut().size(), 2);
    assertEquals(getSegmentOffset(watermark, segment3), 100L);
    assertEquals(getSegmentOffset(watermark, segment4), 500L);
}

Example 18 with StreamImpl

use of io.pravega.client.stream.impl.StreamImpl in project pravega by pravega.

the class ControllerServiceTest method getSegmentsAtTime.

private static void getSegmentsAtTime(Controller controller, final String scope, final String streamName) throws InterruptedException, ExecutionException {
    CompletableFuture<Map<Segment, Long>> segments = controller.getSegmentsAtTime(new StreamImpl(scope, streamName), System.currentTimeMillis());
    assertFalse("FAILURE: Fetching positions at given time stamp failed", segments.get().isEmpty());
}

Example 19 with StreamImpl

use of io.pravega.client.stream.impl.StreamImpl in project pravega by pravega.

the class ControllerServiceTest method getSegmentsBeforeCreation.

private static void getSegmentsBeforeCreation(Controller controller, final String scope, final String streamName) throws InterruptedException, ExecutionException {
    CompletableFuture<Map<Segment, Long>> segments = controller.getSegmentsAtTime(new StreamImpl(scope, streamName), System.currentTimeMillis() - 36000);
    assertFalse("FAILURE: Fetching positions at given time before stream creation failed", segments.get().size() == 1);
}

Example 20 with StreamImpl

use of io.pravega.client.stream.impl.StreamImpl in project pravega by pravega.

the class LargeEventTest method testReadWriteScaleStreamSeal.

/**
 * Tests that if a Stream is sealed no data will continue to be produced by the {@link io.pravega.client.stream.impl.LargeEventWriter}
 */
@Test(timeout = 60000)
public void testReadWriteScaleStreamSeal() throws ExecutionException, InterruptedException, TimeoutException {
    String readerGroupName = "testLargeEventScaleStreamSealReaderGroup";
    String streamName = "ScaleStreamSeal";
    StreamConfiguration config = StreamConfiguration.builder().retentionPolicy(RetentionPolicy.bySizeBytes(Long.MAX_VALUE)).build();
    createScopeStream(SCOPE_NAME, streamName, config);
    AtomicInteger generation = new AtomicInteger(0);
    // Creates some data to write.
    int events = 1;
    Map<Integer, List<ByteBuffer>> data = generateEventData(NUM_WRITERS, events * generation.getAndIncrement(), events, LARGE_EVENT_SIZE);
    merge(eventsWrittenToPravega, data);
    // Perform a basic read-write cycle.
    Queue<ByteBuffer> reads = readWriteCycle(streamName, readerGroupName, eventsWrittenToPravega);
    validateEventReads(reads, eventsWrittenToPravega);
    // Define the scale operation.
    Runnable scale = () -> {
        Stream stream = new StreamImpl(SCOPE_NAME, streamName);
        try {
            StreamSegments segments = controller.getCurrentSegments(SCOPE_NAME, streamName).get();
            List<Long> ids = segments.getSegments().stream().map(Segment::getSegmentId).collect(Collectors.toList());
            controller.startScale(stream, ids, Map.of(0.0, 0.5, 0.5, 1.0)).get();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            throw new RuntimeException(e);
        }
    };
    // Reset reads.
    eventReadCount.set(0);
    // Try to scale the segment after two `send` calls.
    AtomicInteger sendCount = new AtomicInteger(0);
    Supplier<Boolean> predicate = () -> sendCount.getAndIncrement() == CLOSE_WRITE_COUNT;
    AtomicReference<Boolean> latch = new AtomicReference<>(true);
    try (ConnectionExporter connectionFactory = new ConnectionExporter(ClientConfig.builder().build(), latch, scale, predicate);
        ClientFactoryImpl clientFactory = new ClientFactoryImpl(SCOPE_NAME, controller, connectionFactory);
        ReaderGroupManager readerGroupManager = new ReaderGroupManagerImpl(SCOPE_NAME, controller, clientFactory)) {
        // Next set of writes.
        data = generateEventData(NUM_WRITERS, events * generation.getAndIncrement(), events, LARGE_EVENT_SIZE);
        merge(eventsWrittenToPravega, data);
        // Start writing events to the stream.
        val writers = createEventWriters(streamName, NUM_WRITERS, clientFactory, data);
        Futures.allOf(writers).get();
        // Wait for the scale event.
        TestUtils.await(() -> !latch.get(), 200, 2000);
        // Create a ReaderGroup.
        createReaderGroup(readerGroupName, readerGroupManager, streamName);
        // Create Readers.
        val readers = createEventReaders(NUM_READERS, clientFactory, readerGroupName, eventsReadFromPravega);
        stopReadFlag.set(true);
        Futures.allOf(readers).get();
        log.info("Deleting ReaderGroup: {}", readerGroupName);
        readerGroupManager.deleteReaderGroup(readerGroupName);
    }
    StreamSegments segments = controller.getCurrentSegments(SCOPE_NAME, streamName).get();
    // Make sure that the scale event has happened.
    Assert.assertEquals("Expected 2 StreamSegments.", 2, segments.getSegments().size());
    // This time there are successor segments, so the data should have been accepted.
    validateEventReads(eventsReadFromPravega, eventsWrittenToPravega);
}