Examples with WriterMark io.pravega.controller.store.stream.records.WriterMark used on opensource projects

Example 1 with WriterMark

use of io.pravega.controller.store.stream.records.WriterMark in project pravega by pravega.

the class ControllerMetadataJsonSerializerTest method testWriterMark.

@Test
public void testWriterMark() {
    Map<Long, Long> map = new HashMap<>();
    map.put(0L, 0L);
    map.put(1L, 0L);
    map.put(2L, 0L);
    WriterMark record = new WriterMark(100L, ImmutableMap.copyOf(map), true);
    testRecordSerialization(record, WriterMark.class);
}

Example 2 with WriterMark

use of io.pravega.controller.store.stream.records.WriterMark in project pravega by pravega.

the class PeriodicWatermarking method filterWritersAndComputeWatermark.

private CompletionStage<Void> filterWritersAndComputeWatermark(String scope, String streamName, OperationContext context, WatermarkClient watermarkClient, Map<String, WriterMark> writers, StreamConfiguration config) {
    // 1. filter writers that are active.
    List<Entry<String, WriterMark>> activeWriters = new ArrayList<>();
    List<Entry<String, WriterMark>> inactiveWriters = new ArrayList<>();
    AtomicBoolean allActiveAreParticipating = new AtomicBoolean(true);
    writers.entrySet().forEach(x -> {
        if (watermarkClient.isWriterActive(x, config.getTimestampAggregationTimeout())) {
            activeWriters.add(x);
            if (!watermarkClient.isWriterParticipating(x.getValue().getTimestamp())) {
                allActiveAreParticipating.set(false);
            }
        } else {
            inactiveWriters.add(x);
        }
    });
    // Stop all inactive writers that have been shutdown.
    CompletableFuture<List<Void>> removeInactiveWriters = Futures.allOfWithResults(inactiveWriters.stream().map(x -> Futures.exceptionallyExpecting(streamMetadataStore.removeWriter(scope, streamName, x.getKey(), x.getValue(), context, executor).thenAccept(v -> watermarkClient.untrackWriterInactivity(x.getKey())), e -> Exceptions.unwrap(e) instanceof StoreException.WriteConflictException, null)).collect(Collectors.toList()));
    if (activeWriters.isEmpty()) {
        // periodic processing will resume.
        return removeInactiveWriters.thenCompose(v -> bucketStore.removeStreamFromBucketStore(BucketStore.ServiceType.WatermarkingService, scope, streamName, executor));
    }
    CompletableFuture<Watermark> watermarkFuture;
    if (!allActiveAreParticipating.get()) {
        // there are active writers that have not reported their marks. We should wait
        // until they either report or become inactive. So we will complete this iteration without
        // emitting any watermark (null) and in subsequent iterations if these writers have made progress
        // we will emit watermark or evict writers from watermark computation.
        watermarkFuture = CompletableFuture.completedFuture(null);
    } else {
        // compute new mark
        watermarkFuture = computeWatermark(scope, streamName, context, activeWriters, watermarkClient.getPreviousWatermark());
    }
    // we will compute watermark and remove inactive writers concurrently
    return CompletableFuture.allOf(removeInactiveWriters, watermarkFuture.thenAccept(watermarkClient::completeIteration));
}

Example 3 with WriterMark

use of io.pravega.controller.store.stream.records.WriterMark in project pravega by pravega.

the class PravegaTablesStream method updateWriterMarkRecord.

@Override
CompletableFuture<Void> updateWriterMarkRecord(String writer, long timestamp, ImmutableMap<Long, Long> position, boolean isAlive, Version version, OperationContext context) {
    Preconditions.checkNotNull(context, "operation context cannot be null");
    WriterMark mark = new WriterMark(timestamp, position, isAlive);
    return Futures.toVoid(getWritersTable(context).thenCompose(table -> storeHelper.updateEntry(table, writer, mark, WriterMark::toBytes, version, context.getRequestId())));
}

Example 4 with WriterMark

use of io.pravega.controller.store.stream.records.WriterMark in project pravega by pravega.

the class PravegaTablesStream method createWriterMarkRecord.

@Override
CompletableFuture<Void> createWriterMarkRecord(String writer, long timestamp, ImmutableMap<Long, Long> position, OperationContext context) {
    Preconditions.checkNotNull(context, "operation context cannot be null");
    WriterMark mark = new WriterMark(timestamp, position);
    return Futures.toVoid(getWritersTable(context).thenCompose(table -> storeHelper.addNewEntry(table, writer, mark, WriterMark::toBytes, context.getRequestId())));
}

Example 5 with WriterMark

use of io.pravega.controller.store.stream.records.WriterMark in project pravega by pravega.

the class WatermarkWorkflowTest method testWatermarkClient.

@Test(timeout = 10000L)
public void testWatermarkClient() {
    Stream stream = new StreamImpl("scope", "stream");
    SynchronizerClientFactory clientFactory = spy(SynchronizerClientFactory.class);
    @Cleanup MockRevisionedStreamClient revisionedClient = new MockRevisionedStreamClient();
    doAnswer(x -> revisionedClient).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    @Cleanup PeriodicWatermarking.WatermarkClient client = new PeriodicWatermarking.WatermarkClient(stream, clientFactory);
    // iteration 1 ==> null -> w1
    client.reinitialize();
    // There is no watermark in the stream. All values should be null and all writers active and participating.
    assertEquals(revisionedClient.getMark(), MockRevision.EMPTY);
    assertTrue(revisionedClient.watermarks.isEmpty());
    assertEquals(client.getPreviousWatermark(), Watermark.EMPTY);
    Map.Entry<String, WriterMark> entry0 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(0L, ImmutableMap.of()));
    Map.Entry<String, WriterMark> entry1 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(1L, ImmutableMap.of()));
    Map.Entry<String, WriterMark> entry2 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(2L, ImmutableMap.of()));
    Map.Entry<String, WriterMark> entry3 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(3L, ImmutableMap.of()));
    Map.Entry<String, WriterMark> entry4 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(4L, ImmutableMap.of()));
    Map.Entry<String, WriterMark> entry5 = new AbstractMap.SimpleEntry<>("writerId", new WriterMark(5L, ImmutableMap.of()));
    assertTrue(client.isWriterActive(entry0, 0L));
    assertTrue(client.isWriterParticipating(0L));
    Watermark first = new Watermark(1L, 2L, ImmutableMap.of());
    client.completeIteration(first);
    // iteration 2 : do not emit ==> w1 -> w1
    client.reinitialize();
    // There is one watermark. All writers should be active and writers greater than last watermark should be participating
    assertEquals(revisionedClient.getMark(), MockRevision.EMPTY);
    assertEquals(revisionedClient.watermarks.size(), 1);
    assertEquals(client.getPreviousWatermark(), first);
    assertTrue(client.isWriterActive(entry2, 0L));
    assertFalse(client.isWriterActive(entry1, 0L));
    assertTrue(client.isWriterTracked(entry1.getKey()));
    assertFalse(client.isWriterParticipating(1L));
    assertTrue(client.isWriterParticipating(2L));
    // dont emit a watermark. Everything stays same as before.
    client.completeIteration(null);
    // iteration 3 : emit ==> w1 -> w1 w2
    client.reinitialize();
    // There is one watermark. All writers should be active and writers greater than last watermark should be participating
    assertEquals(revisionedClient.getMark(), MockRevision.EMPTY);
    assertEquals(revisionedClient.watermarks.size(), 1);
    assertEquals(client.getPreviousWatermark(), first);
    assertTrue(client.isWriterActive(entry2, 0L));
    assertFalse(client.isWriterParticipating(1L));
    assertTrue(client.isWriterParticipating(2L));
    // emit second watermark
    Watermark second = new Watermark(2L, 3L, ImmutableMap.of());
    client.completeIteration(second);
    // iteration 4: do not emit ==> w1 w2 -> w1 w2
    client.reinitialize();
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(0).getKey());
    assertEquals(2, revisionedClient.watermarks.size());
    assertEquals(client.getPreviousWatermark(), second);
    assertFalse(client.isWriterActive(entry2, 0L));
    assertTrue(client.isWriterTracked(entry2.getKey()));
    assertTrue(client.isWriterActive(entry3, 0L));
    assertFalse(client.isWriterParticipating(2L));
    assertTrue(client.isWriterParticipating(3L));
    assertTrue(client.isWriterActive(entry0, 1000L));
    assertTrue(client.isWriterTracked(entry0.getKey()));
    // dont emit a watermark but complete this iteration.
    client.completeIteration(null);
    // iteration 6: emit ==> w1 w2 -> w1 w2 w3
    client.reinitialize();
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(0).getKey());
    assertEquals(2, revisionedClient.watermarks.size());
    assertEquals(client.getPreviousWatermark(), second);
    assertTrue(client.isWriterActive(entry3, 0L));
    assertFalse(client.isWriterTracked(entry3.getKey()));
    assertFalse(client.isWriterParticipating(2L));
    assertTrue(client.isWriterParticipating(3L));
    // emit third watermark
    Watermark third = new Watermark(3L, 4L, ImmutableMap.of());
    client.completeIteration(third);
    // iteration 7: do not emit ==> w1 w2 w3 -> w1 w2 w3
    client.reinitialize();
    // active writers should be ahead of first watermark. participating writers should be ahead of second watermark
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(1).getKey());
    assertEquals(3, revisionedClient.watermarks.size());
    assertEquals(client.getPreviousWatermark(), third);
    assertFalse(client.isWriterActive(entry3, 0L));
    assertTrue(client.isWriterActive(entry4, 0L));
    assertFalse(client.isWriterParticipating(3L));
    assertTrue(client.isWriterParticipating(4L));
    client.completeIteration(null);
    // iteration 8 : emit ==> w2 w3 -> w2 w3 w4
    client.reinitialize();
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(1).getKey());
    // window = w2 w3
    assertEquals(revisionedClient.watermarks.size(), 3);
    assertEquals(client.getPreviousWatermark(), third);
    assertFalse(client.isWriterActive(entry3, 0L));
    assertTrue(client.isWriterActive(entry4, 0L));
    assertFalse(client.isWriterParticipating(3L));
    assertTrue(client.isWriterParticipating(4L));
    // emit fourth watermark
    Watermark fourth = new Watermark(4L, 5L, ImmutableMap.of());
    client.completeIteration(fourth);
    // iteration 9: do not emit ==> w1 w2 w3 w4 -> w1 w2 w3 w4.. check writer timeout
    client.reinitialize();
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(2).getKey());
    assertEquals(revisionedClient.watermarks.size(), 4);
    assertEquals(client.getPreviousWatermark(), fourth);
    assertFalse(client.isWriterActive(entry3, 0L));
    assertTrue(client.isWriterTracked(entry4.getKey()));
    assertFalse(client.isWriterParticipating(4L));
    assertTrue(client.isWriterParticipating(5L));
    // verify that writer is active if we specify a higher timeout
    assertTrue(client.isWriterActive(entry1, 1000L));
    assertTrue(client.isWriterTracked(entry1.getKey()));
    // now that the writer is being tracked
    assertFalse(Futures.delayedTask(() -> client.isWriterActive(entry1, 1L), Duration.ofSeconds(1), executor).join());
    assertTrue(client.isWriterTracked(entry1.getKey()));
    // dont emit a watermark but complete this iteration. This should shrink the window again.
    client.completeIteration(null);
    // iteration 10
    client.reinitialize();
    assertEquals(revisionedClient.getMark(), revisionedClient.watermarks.get(2).getKey());
    assertEquals(revisionedClient.watermarks.size(), 4);
    assertEquals(client.getPreviousWatermark(), fourth);
    assertFalse(client.isWriterActive(entry4, 0L));
    assertTrue(client.isWriterActive(entry5, 0L));
    assertFalse(client.isWriterParticipating(4L));
    assertTrue(client.isWriterParticipating(5L));
}