Examples with SynchronizerClientFactory io.pravega.client.SynchronizerClientFactory used on opensource projects

Example 21 with SynchronizerClientFactory

use of io.pravega.client.SynchronizerClientFactory in project pravega by pravega.

the class RevisionedStreamClientTest method testSegmentSealedFromSegmentOutputStreamError.

@Test
public void testSegmentSealedFromSegmentOutputStreamError() {
    String scope = "scope";
    String stream = "stream";
    // Setup Environment
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    @Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    @Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    // Setup mock
    SegmentOutputStreamFactory outFactory = mock(SegmentOutputStreamFactory.class);
    SegmentOutputStream out = mock(SegmentOutputStream.class);
    when(outFactory.createOutputStreamForSegment(eq(new Segment(scope, stream, 0)), any(), any(), any(DelegationTokenProvider.class))).thenReturn(out);
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, outFactory, streamFactory, streamFactory);
    CompletableFuture<Void> writeFuture = new CompletableFuture<>();
    PendingEvent event1 = PendingEvent.withoutHeader("key", ByteBufferUtils.EMPTY, writeFuture);
    PendingEvent event2 = PendingEvent.withoutHeader("key", ByteBufferUtils.EMPTY, null);
    // Two events are returned when the callback invokes getUnackedEventsOnSeal
    when(out.getUnackedEventsOnSeal()).thenReturn(Arrays.asList(event1, event2));
    @Cleanup RevisionedStreamClient<String> client = clientFactory.createRevisionedStreamClient(stream, new JavaSerializer<>(), SynchronizerConfig.builder().build());
    // simulate invocation of handleSegmentSealed by Segment writer.
    ((RevisionedStreamClientImpl) client).handleSegmentSealed();
    // Verify SegmentOutputStream#getUnackedEventsOnSeal is invoked.
    verify(out, times(1)).getUnackedEventsOnSeal();
    assertTrue(writeFuture.isCompletedExceptionally());
    assertThrows(SegmentSealedException.class, writeFuture::get);
}

Example 22 with SynchronizerClientFactory

use of io.pravega.client.SynchronizerClientFactory in project pravega by pravega.

the class RevisionedStreamClientTest method testConditionalWrite.

@Test
public void testConditionalWrite() {
    String scope = "scope";
    String stream = "stream";
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    @Cleanup MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    @Cleanup MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory, false);
    createScopeAndStream(scope, stream, controller);
    MockSegmentStreamFactory streamFactory = new MockSegmentStreamFactory();
    @Cleanup SynchronizerClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory, streamFactory, streamFactory, streamFactory, streamFactory);
    SynchronizerConfig config = SynchronizerConfig.builder().build();
    @Cleanup RevisionedStreamClient<String> client = clientFactory.createRevisionedStreamClient(stream, new JavaSerializer<>(), config);
    client.writeUnconditionally("a");
    Revision revision = client.fetchLatestRevision();
    Revision newRevision = client.writeConditionally(revision, "b");
    assertNotNull(newRevision);
    assertTrue(newRevision.compareTo(revision) > 0);
    assertEquals(newRevision, client.fetchLatestRevision());
    Revision failed = client.writeConditionally(revision, "fail");
    assertNull(failed);
    assertEquals(newRevision, client.fetchLatestRevision());
    Iterator<Entry<Revision, String>> iter = client.readFrom(revision);
    assertTrue(iter.hasNext());
    Entry<Revision, String> entry = iter.next();
    assertEquals(newRevision, entry.getKey());
    assertEquals("b", entry.getValue());
    assertFalse(iter.hasNext());
}

Example 23 with SynchronizerClientFactory

use of io.pravega.client.SynchronizerClientFactory in project pravega by pravega.

the class WatermarkWorkflowTest method testRevisionedClientThrowsNoSuchSegmentException.

@Test(timeout = 30000L)
public void testRevisionedClientThrowsNoSuchSegmentException() {
    String scope = "scope";
    String streamName = "stream";
    StreamImpl stream = new StreamImpl(scope, streamName);
    String markStreamName = NameUtils.getMarkStreamForStream(streamName);
    SynchronizerClientFactory clientFactory = spy(SynchronizerClientFactory.class);
    ConcurrentHashMap<String, MockRevisionedStreamClient> revisionedStreamClientMap = new ConcurrentHashMap<>();
    doAnswer(x -> {
        String name = x.getArgument(0);
        return revisionedStreamClientMap.compute(name, (s, rsc) -> new MockRevisionedStreamClient(() -> streamMetadataStore.getActiveSegments(scope, name, null, executor).join().get(0).segmentId()));
    }).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    @Cleanup PeriodicWatermarking periodicWatermarking = new PeriodicWatermarking(streamMetadataStore, bucketStore, sp -> clientFactory, executor, new RequestTracker(false));
    streamMetadataStore.createScope(scope, null, executor).join();
    streamMetadataStore.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).timestampAggregationTimeout(10000L).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.createStream(scope, markStreamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, markStreamName, State.ACTIVE, null, executor).join();
    streamMetadataStore.setState(scope, streamName, State.ACTIVE, null, executor).join();
    // 1. note writer1 marks
    // writer 1 reports segments 0, 1.
    String writer1 = "writer1";
    Map<Long, Long> map1 = ImmutableMap.of(0L, 100L, 1L, 100L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 100L, map1, null, executor).join();
    // 2. run watermarking workflow.
    periodicWatermarking.watermark(stream).join();
    assertTrue(periodicWatermarking.checkExistsInCache(stream));
    // verify that a watermark has been emitted.
    MockRevisionedStreamClient revisionedClient = revisionedStreamClientMap.get(markStreamName);
    assertEquals(revisionedClient.watermarks.size(), 1);
    Watermark watermark = revisionedClient.watermarks.get(0).getValue();
    assertEquals(watermark.getLowerTimeBound(), 100L);
    // delete and recreate stream and its mark stream
    streamMetadataStore.deleteStream(scope, markStreamName, null, executor).join();
    streamMetadataStore.deleteStream(scope, streamName, null, executor).join();
    streamMetadataStore.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).timestampAggregationTimeout(10000L).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, streamName, State.ACTIVE, null, executor).join();
    streamMetadataStore.createStream(scope, markStreamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, markStreamName, State.ACTIVE, null, executor).join();
    // 1. note writer1 marks
    // writer 1 reports segments 0, 1.
    map1 = ImmutableMap.of(2L, 10L, 3L, 10L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 10L, map1, null, executor).join();
    // 2. run watermarking workflow. this should fail and revisioned stream client should be invalidated in the cache.
    periodicWatermarking.watermark(stream).join();
    assertFalse(periodicWatermarking.checkExistsInCache(stream));
    // 3. run watermarking workflow again.
    periodicWatermarking.watermark(stream).join();
    assertTrue(periodicWatermarking.checkExistsInCache(stream));
    // verify that a watermark has been emitted.
    revisionedClient = revisionedStreamClientMap.get(markStreamName);
    assertEquals(revisionedClient.segment, 1L);
    assertEquals(revisionedClient.watermarks.size(), 1);
    watermark = revisionedClient.watermarks.get(0).getValue();
    assertEquals(watermark.getLowerTimeBound(), 10L);
}

Example 24 with SynchronizerClientFactory

use of io.pravega.client.SynchronizerClientFactory in project pravega by pravega.

the class WatermarkWorkflowTest method testWatermarkClientClose.

@Test(timeout = 10000L)
public void testWatermarkClientClose() {
    String scope = "scope1";
    String streamName = "stream1";
    StreamImpl stream = new StreamImpl(scope, streamName);
    SynchronizerClientFactory clientFactory = spy(SynchronizerClientFactory.class);
    String markStreamName = NameUtils.getMarkStreamForStream(streamName);
    @Cleanup MockRevisionedStreamClient revisionedClient = new MockRevisionedStreamClient();
    doAnswer(x -> revisionedClient).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    doNothing().when(clientFactory).close();
    PeriodicWatermarking.WatermarkClient client = new PeriodicWatermarking.WatermarkClient(stream, clientFactory);
    client.close();
    verify(clientFactory, never()).close();
    client = new PeriodicWatermarking.WatermarkClient(stream, clientFactory);
    client.close();
    verify(clientFactory, never()).close();
    String s = "failing creation";
    doThrow(new RuntimeException(s)).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    AssertExtensions.assertThrows("constructor should throw", () -> new PeriodicWatermarking.WatermarkClient(stream, clientFactory), e -> e instanceof RuntimeException && s.equals(e.getMessage()));
    @Cleanup PeriodicWatermarking periodicWatermarking = new PeriodicWatermarking(streamMetadataStore, bucketStore, sp -> clientFactory, executor, new RequestTracker(false));
    streamMetadataStore.createScope(scope, null, executor).join();
    streamMetadataStore.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).timestampAggregationTimeout(10000L).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.createStream(scope, markStreamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, markStreamName, State.ACTIVE, null, executor).join();
    streamMetadataStore.setState(scope, streamName, State.ACTIVE, null, executor).join();
    String writer1 = "writer1";
    Map<Long, Long> map1 = ImmutableMap.of(0L, 100L, 1L, 100L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 100L, map1, null, executor).join();
    // 2. run watermarking workflow.
    periodicWatermarking.watermark(stream).join();
    assertTrue(periodicWatermarking.checkExistsInCache(scope));
    periodicWatermarking.evictFromCache(scope);
    // verify that the syncfactory was closed
    verify(clientFactory, times(1)).close();
}

Example 25 with SynchronizerClientFactory

use of io.pravega.client.SynchronizerClientFactory in project pravega by pravega.

the class WatermarkWorkflowTest method testWriterTimeout.

@Test(timeout = 30000L)
public void testWriterTimeout() {
    SynchronizerClientFactory clientFactory = spy(SynchronizerClientFactory.class);
    ConcurrentHashMap<String, MockRevisionedStreamClient> revisionedStreamClientMap = new ConcurrentHashMap<>();
    doAnswer(x -> {
        String streamName = x.getArgument(0);
        return revisionedStreamClientMap.compute(streamName, (s, rsc) -> {
            if (rsc != null) {
                return rsc;
            } else {
                return new MockRevisionedStreamClient();
            }
        });
    }).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    StreamMetadataStore streamMetadataStoreSpied = spy(this.streamMetadataStore);
    BucketStore bucketStoreSpied = spy(this.bucketStore);
    @Cleanup PeriodicWatermarking periodicWatermarking = new PeriodicWatermarking(streamMetadataStoreSpied, bucketStoreSpied, sp -> clientFactory, executor, new RequestTracker(false));
    String streamName = "stream";
    String scope = "scope";
    streamMetadataStoreSpied.createScope(scope, null, executor).join();
    streamMetadataStoreSpied.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(3)).timestampAggregationTimeout(3000L).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStoreSpied.setState(scope, streamName, State.ACTIVE, null, executor).join();
    // 2. note writer1, writer2, writer3 marks
    // writer 1 reports segments 0, 1.
    // writer 2 reports segments 1, 2,
    // writer 3 reports segment 0, 2
    String writer1 = "writer1";
    streamMetadataStoreSpied.noteWriterMark(scope, streamName, writer1, 102L, ImmutableMap.of(0L, 100L, 1L, 0L, 2L, 0L), null, executor).join();
    String writer2 = "writer2";
    streamMetadataStoreSpied.noteWriterMark(scope, streamName, writer2, 101L, ImmutableMap.of(0L, 0L, 1L, 100L, 2L, 0L), null, executor).join();
    String writer3 = "writer3";
    streamMetadataStoreSpied.noteWriterMark(scope, streamName, writer3, 100L, ImmutableMap.of(0L, 0L, 1L, 0L, 2L, 100L), null, executor).join();
    // 3. run watermarking workflow.
    StreamImpl stream = new StreamImpl(scope, streamName);
    periodicWatermarking.watermark(stream).join();
    // verify that a watermark has been emitted.
    MockRevisionedStreamClient revisionedClient = revisionedStreamClientMap.get(NameUtils.getMarkStreamForStream(streamName));
    assertEquals(revisionedClient.watermarks.size(), 1);
    // Don't report time from writer3
    streamMetadataStoreSpied.noteWriterMark(scope, streamName, writer1, 200L, ImmutableMap.of(0L, 200L, 1L, 0L, 2L, 0L), null, executor).join();
    streamMetadataStoreSpied.noteWriterMark(scope, streamName, writer2, 200L, ImmutableMap.of(0L, 0L, 1L, 200L, 2L, 0L), null, executor).join();
    // no new watermark should be emitted, writers should be tracked for inactivity
    periodicWatermarking.watermark(stream).join();
    assertEquals(revisionedClient.watermarks.size(), 1);
    verify(streamMetadataStoreSpied, never()).removeWriter(anyString(), anyString(), anyString(), any(), any(), any());
    verify(bucketStoreSpied, never()).removeStreamFromBucketStore(any(), anyString(), anyString(), any());
    // call again. Still no new watermark should be emitted as writers have not timed out
    periodicWatermarking.watermark(stream).join();
    assertEquals(revisionedClient.watermarks.size(), 1);
    verify(streamMetadataStoreSpied, never()).removeWriter(anyString(), anyString(), anyString(), any(), any(), any());
    verify(bucketStoreSpied, never()).removeStreamFromBucketStore(any(), anyString(), anyString(), any());
    // call watermark after a delay of 5 more seconds. The writer3 should timeout because it has a timeout of 3 seconds.
    Futures.delayedFuture(() -> periodicWatermarking.watermark(stream), 5000L, executor).join();
    verify(streamMetadataStoreSpied, times(1)).removeWriter(anyString(), anyString(), anyString(), any(), any(), any());
    verify(bucketStoreSpied, never()).removeStreamFromBucketStore(any(), anyString(), anyString(), any());
    // watermark should be emitted. without considering writer3
    assertEquals(revisionedClient.watermarks.size(), 2);
    Watermark watermark = revisionedClient.watermarks.get(1).getValue();
    assertEquals(watermark.getLowerTimeBound(), 200L);
    assertEquals(watermark.getStreamCut().size(), 3);
    assertEquals(getSegmentOffset(watermark, 0L), 200L);
    assertEquals(getSegmentOffset(watermark, 1L), 200L);
    assertEquals(getSegmentOffset(watermark, 2L), 100L);
    // call watermark workflow again so that both writers are tracked for inactivity
    periodicWatermarking.watermark(stream).join();
    assertEquals(revisionedClient.watermarks.size(), 2);
    verify(streamMetadataStoreSpied, times(1)).removeWriter(anyString(), anyString(), anyString(), any(), any(), any());
    verify(bucketStoreSpied, never()).removeStreamFromBucketStore(any(), anyString(), anyString(), any());
    // now introduce more delays and see all writers are removed and stream is discontinued from watermarking computation.
    Futures.delayedFuture(() -> periodicWatermarking.watermark(stream), 5000L, executor).join();
    // verify that stream is discontinued from tracking for watermarking
    verify(streamMetadataStoreSpied, times(3)).removeWriter(anyString(), anyString(), anyString(), any(), any(), any());
    verify(bucketStoreSpied, times(1)).removeStreamFromBucketStore(any(), anyString(), anyString(), any());
    // call note time for writer3 and verify that watermark is emitted.
    streamMetadataStoreSpied.noteWriterMark(scope, streamName, writer3, 300L, ImmutableMap.of(0L, 300L, 1L, 0L, 2L, 0L), null, executor).join();
    periodicWatermarking.watermark(stream).join();
    assertEquals(revisionedClient.watermarks.size(), 3);
    watermark = revisionedClient.watermarks.get(2).getValue();
    assertEquals(watermark.getLowerTimeBound(), 300L);
    assertEquals(watermark.getStreamCut().size(), 3);
    assertEquals(getSegmentOffset(watermark, 0L), 300L);
    assertEquals(getSegmentOffset(watermark, 1L), 200L);
    assertEquals(getSegmentOffset(watermark, 2L), 100L);
}