Examples with StreamCutImpl io.pravega.client.stream.impl.StreamCutImpl used on opensource projects

Example 6 with StreamCutImpl

use of io.pravega.client.stream.impl.StreamCutImpl in project pravega by pravega.

the class StreamCutTest method testStreamCutSerialization.

@Test
public void testStreamCutSerialization() throws Exception {
    ImmutableMap<Segment, Long> segmentOffsetMap = ImmutableMap.<Segment, Long>builder().put(new Segment("scope", "stream", computeSegmentId(1, 1)), 10L).put(new Segment("scope", "stream", computeSegmentId(2, 1)), 20L).put(new Segment("scope", "stream", computeSegmentId(3, 1)), 30L).put(new Segment("scope", "stream", computeSegmentId(4, 1)), 20L).put(new Segment("scope", "stream", computeSegmentId(5, 2)), 50L).put(new Segment("scope", "stream", computeSegmentId(8, 2)), 50L).put(new Segment("scope", "stream", computeSegmentId(9, 2)), 60L).put(new Segment("scope", "stream", computeSegmentId(10, 2)), -1L).build();
    StreamCut sc = new StreamCutImpl(Stream.of("scope", "stream"), segmentOffsetMap);
    byte[] buf = serialize(sc);
    String base64 = sc.asText();
    assertEquals(sc, deSerializeStreamCut(buf));
    assertEquals(sc, StreamCut.fromBytes(sc.toBytes()));
    assertEquals(sc, StreamCut.from(base64));
}

Example 7 with StreamCutImpl

use of io.pravega.client.stream.impl.StreamCutImpl in project pravega by pravega.

the class StreamCutTest method testStreamCutSerializationCompatabilityV0.

@Test
public void testStreamCutSerializationCompatabilityV0() throws Exception {
    ImmutableMap<Segment, Long> segmentOffsetMap = ImmutableMap.<Segment, Long>builder().put(new Segment("scope", "stream", computeSegmentId(1, 1)), 10L).put(new Segment("scope", "stream", computeSegmentId(2, 1)), 20L).build();
    StreamCut sc = new StreamCutImpl(Stream.of("scope", "stream"), segmentOffsetMap);
    // Obtain version 0 serialized data
    final byte[] bufV0 = new StreamCutSerializerV0().serialize(sc.asImpl()).array();
    // deserialize it using current version 1 serialization and ensure compatibility.
    assertEquals(sc, new StreamCutImpl.StreamCutSerializer().deserialize(bufV0));
}

Example 8 with StreamCutImpl

use of io.pravega.client.stream.impl.StreamCutImpl in project pravega by pravega.

the class ControllerServiceTest method readerGroupsTest.

private static void readerGroupsTest(Controller controller, final String scope, final String stream1, final String stream2, final String stream3) throws InterruptedException, ExecutionException {
    final String scopedStreamName1 = NameUtils.getScopedStreamName(scope, stream1);
    final String scopedStreamName2 = NameUtils.getScopedStreamName(scope, stream2);
    final String scopedStreamName3 = NameUtils.getScopedStreamName(scope, stream3);
    final Segment seg0 = new Segment(scope, stream1, 0L);
    final Segment seg1 = new Segment(scope, stream1, 1L);
    ImmutableMap<Segment, Long> startStreamCut = ImmutableMap.of(seg0, 10L, seg1, 10L);
    Map<Stream, StreamCut> startSC = ImmutableMap.of(Stream.of(scope, stream1), new StreamCutImpl(Stream.of(scope, stream1), startStreamCut), Stream.of(scope, stream2), new StreamCutImpl(Stream.of(scope, stream2), startStreamCut));
    ImmutableMap<Segment, Long> endStreamCut = ImmutableMap.of(seg0, 200L, seg1, 300L);
    Map<Stream, StreamCut> endSC = ImmutableMap.of(Stream.of(scope, stream1), new StreamCutImpl(Stream.of(scope, stream1), endStreamCut), Stream.of(scope, stream2), new StreamCutImpl(Stream.of(scope, stream2), endStreamCut));
    ReaderGroupConfig rgConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(30000L).groupRefreshTimeMillis(20000L).maxOutstandingCheckpointRequest(2).retentionType(ReaderGroupConfig.StreamDataRetention.AUTOMATIC_RELEASE_AT_LAST_CHECKPOINT).startingStreamCuts(startSC).endingStreamCuts(endSC).build();
    final ReaderGroupConfig rgConfig1 = ReaderGroupConfig.cloneConfig(rgConfig, UUID.randomUUID(), 0L);
    // Create Reader Group rg1
    ReaderGroupConfig createRGResult = controller.createReaderGroup(scope, "rg1", rgConfig1).get();
    assertEquals(rgConfig1.getReaderGroupId(), createRGResult.getReaderGroupId());
    assertThrows(IllegalArgumentException.class, () -> controller.createReaderGroup(scope, "bad_rg_name", rgConfig1).get());
    assertThrows(IllegalArgumentException.class, () -> controller.createReaderGroup("badscope", "rg3", rgConfig1).get());
    ReaderGroupConfig rgConfig2 = ReaderGroupConfig.builder().retentionType(ReaderGroupConfig.StreamDataRetention.MANUAL_RELEASE_AT_USER_STREAMCUT).stream(scopedStreamName1).stream(scopedStreamName2).automaticCheckpointIntervalMillis(7000L).build();
    rgConfig2 = ReaderGroupConfig.cloneConfig(rgConfig2, UUID.randomUUID(), 0L);
    // Create Reader Group rg2
    createRGResult = controller.createReaderGroup(scope, "rg2", rgConfig2).get();
    assertEquals(rgConfig2.getReaderGroupId(), createRGResult.getReaderGroupId());
    List<String> subscribers = controller.listSubscribers(scope, stream1).get();
    assertTrue(subscribers.size() == 2);
    assertTrue(controller.deleteReaderGroup(scope, "rg2", rgConfig2.getReaderGroupId()).get());
    assertThrows(IllegalArgumentException.class, () -> controller.getReaderGroupConfig(scope, "rg2").get());
    subscribers = controller.listSubscribers(scope, stream1).get();
    assertTrue(subscribers.size() == 1);
    ReaderGroupConfig config = controller.getReaderGroupConfig(scope, "rg1").get();
    assertEquals(rgConfig1.getGroupRefreshTimeMillis(), config.getGroupRefreshTimeMillis());
    assertEquals(rgConfig1.getGeneration(), config.getGeneration());
    assertEquals(rgConfig1.getMaxOutstandingCheckpointRequest(), config.getMaxOutstandingCheckpointRequest());
    assertEquals(rgConfig1.getRetentionType(), config.getRetentionType());
    assertEquals(rgConfig1.getReaderGroupId(), config.getReaderGroupId());
    assertEquals(rgConfig1.getStartingStreamCuts().keySet().size(), config.getStartingStreamCuts().keySet().size());
    assertEquals(rgConfig1.getEndingStreamCuts().keySet().size(), config.getEndingStreamCuts().keySet().size());
    assertTrue(config.getStartingStreamCuts().keySet().contains(Stream.of(scope, stream1)));
    assertTrue(config.getStartingStreamCuts().keySet().contains(Stream.of(scope, stream2)));
    Map<Stream, StreamCut> startSCNew = ImmutableMap.of(Stream.of(scope, stream2), new StreamCutImpl(Stream.of(scope, stream2), startStreamCut), Stream.of(scope, stream3), new StreamCutImpl(Stream.of(scope, stream3), startStreamCut));
    Map<Stream, StreamCut> endSCNew = ImmutableMap.of(Stream.of(scope, stream2), new StreamCutImpl(Stream.of(scope, stream2), endStreamCut), Stream.of(scope, stream3), new StreamCutImpl(Stream.of(scope, stream3), endStreamCut));
    ReaderGroupConfig newRGConfig = ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(1000L).groupRefreshTimeMillis(5000L).maxOutstandingCheckpointRequest(7).retentionType(ReaderGroupConfig.StreamDataRetention.AUTOMATIC_RELEASE_AT_LAST_CHECKPOINT).startingStreamCuts(startSCNew).endingStreamCuts(endSCNew).build();
    newRGConfig = ReaderGroupConfig.cloneConfig(newRGConfig, rgConfig1.getReaderGroupId(), rgConfig1.getGeneration());
    // Update Reader Group rg1
    assertNotNull(controller.updateReaderGroup(scope, "rg1", newRGConfig).get());
    ReaderGroupConfig updatedConfig = controller.getReaderGroupConfig(scope, "rg1").get();
    assertEquals(newRGConfig.getGroupRefreshTimeMillis(), updatedConfig.getGroupRefreshTimeMillis());
    assertEquals(newRGConfig.getGeneration() + 1, updatedConfig.getGeneration());
    assertEquals(newRGConfig.getMaxOutstandingCheckpointRequest(), updatedConfig.getMaxOutstandingCheckpointRequest());
    assertEquals(newRGConfig.getRetentionType(), updatedConfig.getRetentionType());
    assertEquals(newRGConfig.getReaderGroupId(), updatedConfig.getReaderGroupId());
    assertEquals(newRGConfig.getStartingStreamCuts().keySet().size(), updatedConfig.getStartingStreamCuts().keySet().size());
    assertEquals(newRGConfig.getEndingStreamCuts().keySet().size(), updatedConfig.getEndingStreamCuts().keySet().size());
    assertTrue(updatedConfig.getStartingStreamCuts().keySet().contains(Stream.of(scope, stream3)));
    assertTrue(updatedConfig.getStartingStreamCuts().keySet().contains(Stream.of(scope, stream2)));
    // re-create ReaderGroup "rg2"
    ReaderGroupConfig rgConfigRecreate = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(scopedStreamName3).retentionType(ReaderGroupConfig.StreamDataRetention.AUTOMATIC_RELEASE_AT_LAST_CHECKPOINT).build();
    rgConfigRecreate = ReaderGroupConfig.cloneConfig(rgConfigRecreate, UUID.randomUUID(), 0L);
    ReaderGroupConfig recreateRGResponse = controller.createReaderGroup(scope, "rg2", rgConfigRecreate).get();
    assertEquals(rgConfigRecreate.getReaderGroupId(), recreateRGResponse.getReaderGroupId());
    assertEquals(rgConfigRecreate.getRetentionType(), recreateRGResponse.getRetentionType());
    // Update a ReaderGroup from Subscriber to Non-subscriber
    final String readerGroupName = "rg3";
    ReaderGroupConfig rgConfigSubscriber = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(scopedStreamName1).retentionType(ReaderGroupConfig.StreamDataRetention.MANUAL_RELEASE_AT_USER_STREAMCUT).build();
    rgConfigSubscriber = ReaderGroupConfig.cloneConfig(rgConfigSubscriber, UUID.randomUUID(), 0L);
    ReaderGroupConfig subscriberRG = controller.createReaderGroup(scope, readerGroupName, rgConfigSubscriber).join();
    assertEquals(rgConfigSubscriber.getReaderGroupId(), subscriberRG.getReaderGroupId());
    subscribers = controller.listSubscribers(scope, stream1).get();
    assertEquals(1, subscribers.size());
    ReaderGroupConfig rgConfigNonSubscriber = ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(scopedStreamName1).build();
    rgConfigNonSubscriber = ReaderGroupConfig.cloneConfig(rgConfigNonSubscriber, rgConfigSubscriber.getReaderGroupId(), rgConfigSubscriber.getGeneration());
    long updatedGen = controller.updateReaderGroup(scope, readerGroupName, rgConfigNonSubscriber).join();
    assertEquals(rgConfigNonSubscriber.getGeneration() + 1, updatedGen);
    updatedConfig = controller.getReaderGroupConfig(scope, readerGroupName).join();
    assertEquals(rgConfigNonSubscriber.getReaderGroupId(), updatedConfig.getReaderGroupId());
    assertEquals(rgConfigNonSubscriber.getRetentionType(), updatedConfig.getRetentionType());
    assertEquals(rgConfigNonSubscriber.getGeneration() + 1, updatedConfig.getGeneration());
    subscribers = controller.listSubscribers(scope, stream1).get();
    assertEquals(0, subscribers.size());
    // Update ReaderGroup from Non-Subscriber to Subscriber
    ReaderGroupConfig subscriberConfig = ReaderGroupConfig.builder().stream(scopedStreamName1).retentionType(ReaderGroupConfig.StreamDataRetention.AUTOMATIC_RELEASE_AT_LAST_CHECKPOINT).build();
    subscriberConfig = ReaderGroupConfig.cloneConfig(subscriberConfig, updatedConfig.getReaderGroupId(), updatedConfig.getGeneration());
    long gen = controller.updateReaderGroup(scope, readerGroupName, subscriberConfig).join();
    assertEquals(subscriberConfig.getGeneration() + 1, gen);
    ReaderGroupConfig newUpdatedConfig = controller.getReaderGroupConfig(scope, readerGroupName).join();
    assertEquals(subscriberConfig.getReaderGroupId(), newUpdatedConfig.getReaderGroupId());
    assertEquals(subscriberConfig.getRetentionType(), newUpdatedConfig.getRetentionType());
    assertEquals(gen, newUpdatedConfig.getGeneration());
    subscribers = controller.listSubscribers(scope, stream1).get();
    assertEquals(1, subscribers.size());
}

Example 9 with StreamCutImpl

use of io.pravega.client.stream.impl.StreamCutImpl in project pravega by pravega.

the class RestoreBackUpDataRecoveryTest method getStreamCutsFromWaterMarks.

private List<Map<Stream, StreamCut>> getStreamCutsFromWaterMarks(Stream streamObj, LinkedBlockingQueue<Watermark> watermarks) throws InterruptedException {
    Watermark watermark0 = watermarks.take();
    Watermark watermark1 = watermarks.take();
    assertTrue(watermark0.getLowerTimeBound() <= watermark0.getUpperTimeBound());
    assertTrue(watermark1.getLowerTimeBound() <= watermark1.getUpperTimeBound());
    assertTrue(watermark0.getLowerTimeBound() < watermark1.getLowerTimeBound());
    Map<Segment, Long> positionMap0 = watermark0.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(SCOPE, STREAM1, x.getKey().getSegmentId()), Map.Entry::getValue));
    Map<Segment, Long> positionMap1 = watermark1.getStreamCut().entrySet().stream().collect(Collectors.toMap(x -> new Segment(SCOPE, STREAM1, x.getKey().getSegmentId()), Map.Entry::getValue));
    StreamCut streamCutFirst = new StreamCutImpl(streamObj, positionMap0);
    StreamCut streamCutSecond = new StreamCutImpl(streamObj, positionMap1);
    Map<Stream, StreamCut> firstMarkStreamCut = Collections.singletonMap(streamObj, streamCutFirst);
    Map<Stream, StreamCut> secondMarkStreamCut = Collections.singletonMap(streamObj, streamCutSecond);
    return Arrays.asList(firstMarkStreamCut, secondMarkStreamCut);
}

Example 10 with StreamCutImpl

use of io.pravega.client.stream.impl.StreamCutImpl in project pravega by pravega.

the class BoundedStreamReaderTest method getStreamCut.

/*
     * Test method to create StreamCuts. In the real world StreamCuts are obtained via the Pravega client apis.
     */
private StreamCut getStreamCut(String streamName, long offset, int... segmentNumbers) {
    ImmutableMap.Builder<Segment, Long> builder = ImmutableMap.<Segment, Long>builder();
    Arrays.stream(segmentNumbers).forEach(seg -> {
        builder.put(new Segment(SCOPE, streamName, seg), offset);
    });
    return new StreamCutImpl(Stream.of(SCOPE, streamName), builder.build());
}