Examples with StreamImpl io.pravega.client.stream.impl.StreamImpl used on opensource projects

Example 51 with StreamImpl

use of io.pravega.client.stream.impl.StreamImpl in project pravega by pravega.

the class ControllerImplTest method testCreateTransaction.

@Test
public void testCreateTransaction() throws Exception {
    CompletableFuture<TxnSegments> transaction;
    transaction = controllerClient.createTransaction(new StreamImpl("scope1", "stream1"), 0);
    assertEquals(new UUID(11L, 22L), transaction.get().getTxnId());
    assertEquals(2, transaction.get().getStreamSegments().getSegments().size());
    assertEquals(new Segment("scope1", "stream1", 0), transaction.get().getStreamSegments().getSegmentForKey(.2));
    assertEquals(new Segment("scope1", "stream1", 1), transaction.get().getStreamSegments().getSegmentForKey(.8));
    transaction = controllerClient.createTransaction(new StreamImpl("scope1", "stream2"), 0);
    assertEquals(new UUID(33L, 44L), transaction.get().getTxnId());
    assertEquals(1, transaction.get().getStreamSegments().getSegments().size());
    assertEquals(new Segment("scope1", "stream2", 0), transaction.get().getStreamSegments().getSegmentForKey(.2));
    assertEquals(new Segment("scope1", "stream2", 0), transaction.get().getStreamSegments().getSegmentForKey(.8));
    transaction = controllerClient.createTransaction(new StreamImpl("scope1", "stream3"), 0);
    AssertExtensions.assertFutureThrows("Should throw Exception", transaction, throwable -> true);
}

Example 52 with StreamImpl

use of io.pravega.client.stream.impl.StreamImpl in project pravega by pravega.

the class LocalControllerTest method testGetSegmentsBetween.

@Test(timeout = 10000)
public void testGetSegmentsBetween() throws ExecutionException, InterruptedException {
    List<StreamSegmentRecord> list = new ArrayList<>();
    when(this.mockControllerService.getSegmentsBetweenStreamCuts(any(), anyLong())).thenReturn(CompletableFuture.completedFuture(list));
    Assert.assertTrue(Futures.await(this.testController.getSegments(new StreamCutImpl(new StreamImpl("scope", "stream"), Collections.emptyMap()), new StreamCutImpl(new StreamImpl("scope", "stream"), Collections.emptyMap()))));
}

Example 53 with StreamImpl

use of io.pravega.client.stream.impl.StreamImpl in project pravega by pravega.

the class BucketServiceTest method testWatermarkingService.

@Test(timeout = 10000)
public void testWatermarkingService() {
    Map<Integer, BucketService> bucketServices = watermarkingService.getBucketServices();
    assertNotNull(bucketServices);
    assertEquals(3, bucketServices.size());
    assertTrue(watermarkingService.takeBucketOwnership(0, hostId, executor).join());
    assertTrue(watermarkingService.takeBucketOwnership(1, hostId, executor).join());
    assertTrue(watermarkingService.takeBucketOwnership(2, hostId, executor).join());
    AssertExtensions.assertThrows("", () -> watermarkingService.takeBucketOwnership(3, hostId, executor).join(), e -> e instanceof IllegalArgumentException);
    watermarkingService.tryTakeOwnership(0).join();
    String scope = "scope";
    String streamName = "stream";
    Stream stream = new StreamImpl(scope, streamName);
    bucketStore.addStreamToBucketStore(BucketStore.ServiceType.WatermarkingService, scope, streamName, executor).join();
    // verify that at least one of the buckets got the notification
    int bucketId = BucketStore.getBucket(scope, streamName, 3);
    Set<String> streams = bucketStore.getStreamsForBucket(BucketStore.ServiceType.WatermarkingService, bucketId, executor).join();
    BucketService bucketService = bucketServices.get(bucketId);
    AtomicBoolean added = new AtomicBoolean(false);
    RetryHelper.loopWithDelay(() -> !added.get(), () -> CompletableFuture.completedFuture(null).thenAccept(x -> added.set(bucketService.getKnownStreams().size() > 0)), Duration.ofSeconds(1).toMillis(), executor).join();
    assertTrue(bucketService.getKnownStreams().contains(stream));
    bucketStore.removeStreamFromBucketStore(BucketStore.ServiceType.WatermarkingService, scope, streamName, executor).join();
    AtomicBoolean removed = new AtomicBoolean(false);
    RetryHelper.loopWithDelay(() -> !removed.get(), () -> CompletableFuture.completedFuture(null).thenAccept(x -> removed.set(bucketService.getKnownStreams().size() == 0)), Duration.ofSeconds(1).toMillis(), executor).join();
    assertEquals(0, bucketService.getKnownStreams().size());
}

Example 54 with StreamImpl

use of io.pravega.client.stream.impl.StreamImpl in project pravega by pravega.

the class WatermarkWorkflowTest method testRevisionedClientThrowsNoSuchSegmentException.

@Test(timeout = 30000L)
public void testRevisionedClientThrowsNoSuchSegmentException() {
    String scope = "scope";
    String streamName = "stream";
    StreamImpl stream = new StreamImpl(scope, streamName);
    String markStreamName = NameUtils.getMarkStreamForStream(streamName);
    SynchronizerClientFactory clientFactory = spy(SynchronizerClientFactory.class);
    ConcurrentHashMap<String, MockRevisionedStreamClient> revisionedStreamClientMap = new ConcurrentHashMap<>();
    doAnswer(x -> {
        String name = x.getArgument(0);
        return revisionedStreamClientMap.compute(name, (s, rsc) -> new MockRevisionedStreamClient(() -> streamMetadataStore.getActiveSegments(scope, name, null, executor).join().get(0).segmentId()));
    }).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    @Cleanup PeriodicWatermarking periodicWatermarking = new PeriodicWatermarking(streamMetadataStore, bucketStore, sp -> clientFactory, executor, new RequestTracker(false));
    streamMetadataStore.createScope(scope, null, executor).join();
    streamMetadataStore.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).timestampAggregationTimeout(10000L).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.createStream(scope, markStreamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, markStreamName, State.ACTIVE, null, executor).join();
    streamMetadataStore.setState(scope, streamName, State.ACTIVE, null, executor).join();
    // 1. note writer1 marks
    // writer 1 reports segments 0, 1.
    String writer1 = "writer1";
    Map<Long, Long> map1 = ImmutableMap.of(0L, 100L, 1L, 100L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 100L, map1, null, executor).join();
    // 2. run watermarking workflow.
    periodicWatermarking.watermark(stream).join();
    assertTrue(periodicWatermarking.checkExistsInCache(stream));
    // verify that a watermark has been emitted.
    MockRevisionedStreamClient revisionedClient = revisionedStreamClientMap.get(markStreamName);
    assertEquals(revisionedClient.watermarks.size(), 1);
    Watermark watermark = revisionedClient.watermarks.get(0).getValue();
    assertEquals(watermark.getLowerTimeBound(), 100L);
    // delete and recreate stream and its mark stream
    streamMetadataStore.deleteStream(scope, markStreamName, null, executor).join();
    streamMetadataStore.deleteStream(scope, streamName, null, executor).join();
    streamMetadataStore.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).timestampAggregationTimeout(10000L).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, streamName, State.ACTIVE, null, executor).join();
    streamMetadataStore.createStream(scope, markStreamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, markStreamName, State.ACTIVE, null, executor).join();
    // 1. note writer1 marks
    // writer 1 reports segments 0, 1.
    map1 = ImmutableMap.of(2L, 10L, 3L, 10L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 10L, map1, null, executor).join();
    // 2. run watermarking workflow. this should fail and revisioned stream client should be invalidated in the cache.
    periodicWatermarking.watermark(stream).join();
    assertFalse(periodicWatermarking.checkExistsInCache(stream));
    // 3. run watermarking workflow again.
    periodicWatermarking.watermark(stream).join();
    assertTrue(periodicWatermarking.checkExistsInCache(stream));
    // verify that a watermark has been emitted.
    revisionedClient = revisionedStreamClientMap.get(markStreamName);
    assertEquals(revisionedClient.segment, 1L);
    assertEquals(revisionedClient.watermarks.size(), 1);
    watermark = revisionedClient.watermarks.get(0).getValue();
    assertEquals(watermark.getLowerTimeBound(), 10L);
}

Example 55 with StreamImpl

use of io.pravega.client.stream.impl.StreamImpl in project pravega by pravega.

the class WatermarkWorkflowTest method testWatermarkClientClose.

@Test(timeout = 10000L)
public void testWatermarkClientClose() {
    String scope = "scope1";
    String streamName = "stream1";
    StreamImpl stream = new StreamImpl(scope, streamName);
    SynchronizerClientFactory clientFactory = spy(SynchronizerClientFactory.class);
    String markStreamName = NameUtils.getMarkStreamForStream(streamName);
    @Cleanup MockRevisionedStreamClient revisionedClient = new MockRevisionedStreamClient();
    doAnswer(x -> revisionedClient).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    doNothing().when(clientFactory).close();
    PeriodicWatermarking.WatermarkClient client = new PeriodicWatermarking.WatermarkClient(stream, clientFactory);
    client.close();
    verify(clientFactory, never()).close();
    client = new PeriodicWatermarking.WatermarkClient(stream, clientFactory);
    client.close();
    verify(clientFactory, never()).close();
    String s = "failing creation";
    doThrow(new RuntimeException(s)).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    AssertExtensions.assertThrows("constructor should throw", () -> new PeriodicWatermarking.WatermarkClient(stream, clientFactory), e -> e instanceof RuntimeException && s.equals(e.getMessage()));
    @Cleanup PeriodicWatermarking periodicWatermarking = new PeriodicWatermarking(streamMetadataStore, bucketStore, sp -> clientFactory, executor, new RequestTracker(false));
    streamMetadataStore.createScope(scope, null, executor).join();
    streamMetadataStore.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).timestampAggregationTimeout(10000L).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.createStream(scope, markStreamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, markStreamName, State.ACTIVE, null, executor).join();
    streamMetadataStore.setState(scope, streamName, State.ACTIVE, null, executor).join();
    String writer1 = "writer1";
    Map<Long, Long> map1 = ImmutableMap.of(0L, 100L, 1L, 100L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 100L, map1, null, executor).join();
    // 2. run watermarking workflow.
    periodicWatermarking.watermark(stream).join();
    assertTrue(periodicWatermarking.checkExistsInCache(scope));
    periodicWatermarking.evictFromCache(scope);
    // verify that the syncfactory was closed
    verify(clientFactory, times(1)).close();
}