Examples with SegmentWithRange io.pravega.client.stream.impl.SegmentWithRange used on opensource projects

Example 6 with SegmentWithRange

use of io.pravega.client.stream.impl.SegmentWithRange in project pravega by pravega.

the class MockController method getCurrentSegments.

private StreamSegments getCurrentSegments(Stream stream) {
    if (isStreamSealed(stream)) {
        return new StreamSegments(new TreeMap<>());
    } else {
        List<Segment> segmentsInStream = getSegmentsForStream(stream);
        TreeMap<Double, SegmentWithRange> segments = new TreeMap<>();
        for (int i = 0; i < segmentsInStream.size(); i++) {
            SegmentWithRange s = createRange(stream.getScope(), stream.getStreamName(), segmentsInStream.size(), i);
            segments.put(s.getRange().getHigh(), s);
        }
        return new StreamSegments(segments);
    }
}

Example 7 with SegmentWithRange

use of io.pravega.client.stream.impl.SegmentWithRange in project pravega by pravega.

the class MockController method getCurrentSegments.

private KeyValueTableSegments getCurrentSegments(KeyValueTableInfo kvt) {
    List<Segment> segmentsInStream = getSegmentsForKeyValueTable(kvt);
    TreeMap<Double, SegmentWithRange> segments = new TreeMap<>();
    for (int i = 0; i < segmentsInStream.size(); i++) {
        SegmentWithRange s = createRange(kvt.getScope(), kvt.getKeyValueTableName(), segmentsInStream.size(), i);
        segments.put(s.getRange().getHigh(), s);
    }
    return new KeyValueTableSegments(segments);
}

Example 8 with SegmentWithRange

use of io.pravega.client.stream.impl.SegmentWithRange in project pravega by pravega.

the class ControllerEventProcessorsTest method testTruncate.

@Test(timeout = 10000L)
public void testTruncate() throws CheckpointStoreException, InterruptedException {
    LocalController controller = mock(LocalController.class);
    CheckpointStore checkpointStore = mock(CheckpointStore.class);
    StreamMetadataStore streamStore = mock(StreamMetadataStore.class);
    BucketStore bucketStore = mock(BucketStore.class);
    ConnectionPool connectionPool = mock(ConnectionPool.class);
    StreamMetadataTasks streamMetadataTasks = mock(StreamMetadataTasks.class);
    StreamTransactionMetadataTasks streamTransactionMetadataTasks = mock(StreamTransactionMetadataTasks.class);
    KVTableMetadataStore kvtStore = mock(KVTableMetadataStore.class);
    TableMetadataTasks kvtTasks = mock(TableMetadataTasks.class);
    ControllerEventProcessorConfig config = ControllerEventProcessorConfigImpl.withDefault();
    EventProcessorSystem system = mock(EventProcessorSystem.class);
    Map<SegmentWithRange, Long> map1 = new HashMap<>();
    map1.put(new SegmentWithRange(new Segment("scope", "stream", 0L), 0.0, 0.33), 10L);
    map1.put(new SegmentWithRange(new Segment("scope", "stream", 1L), 0.33, 0.66), 10L);
    map1.put(new SegmentWithRange(new Segment("scope", "stream", 2L), 0.66, 1.0), 20L);
    Map<SegmentWithRange, Long> map2 = new HashMap<>();
    map2.put(new SegmentWithRange(new Segment("scope", "stream", 0L), 0.0, 0.33), 20L);
    map2.put(new SegmentWithRange(new Segment("scope", "stream", 2L), 0.66, 1.0), 10L);
    Map<SegmentWithRange, Long> map3 = new HashMap<>();
    map3.put(new SegmentWithRange(new Segment("scope", "stream", 3L), 0.0, 0.33), 0L);
    map3.put(new SegmentWithRange(new Segment("scope", "stream", 4L), 0.33, 0.66), 10L);
    map3.put(new SegmentWithRange(new Segment("scope", "stream", 5L), 0.66, 1.0), 20L);
    PositionImpl position1 = new PositionImpl(map1);
    PositionImpl position2 = new PositionImpl(map2);
    PositionImpl position3 = new PositionImpl(map3);
    doReturn(getProcessor()).when(system).createEventProcessorGroup(any(), any(), any());
    doReturn(CompletableFuture.completedFuture(null)).when(controller).createScope(anyString());
    doReturn(CompletableFuture.completedFuture(null)).when(controller).createInternalStream(anyString(), anyString(), any());
    doNothing().when(streamMetadataTasks).initializeStreamWriters(any(), anyString());
    doNothing().when(streamTransactionMetadataTasks).initializeStreamWriters(any(EventStreamClientFactory.class), any(ControllerEventProcessorConfig.class));
    AtomicBoolean requestCalled = new AtomicBoolean(false);
    AtomicBoolean commitCalled = new AtomicBoolean(false);
    CompletableFuture<Void> requestStreamTruncationFuture = new CompletableFuture<>();
    CompletableFuture<Void> kvtStreamTruncationFuture = new CompletableFuture<>();
    CompletableFuture<Void> abortStreamTruncationFuture = new CompletableFuture<>();
    CompletableFuture<Void> commitStreamTruncationFuture = new CompletableFuture<>();
    doAnswer(x -> {
        String argument = x.getArgument(1);
        if (argument.equals(config.getRequestStreamName())) {
            // let one of the processors throw the exception. this should still be retried in the next cycle.
            if (!requestCalled.get()) {
                requestCalled.set(true);
                throw new RuntimeException("inducing sporadic failure");
            } else {
                requestStreamTruncationFuture.complete(null);
            }
        } else if (argument.equals(config.getCommitStreamName())) {
            // let one of the processors throw the exception. this should still be retried in the next cycle.
            if (commitCalled.get()) {
                commitStreamTruncationFuture.complete(null);
            } else {
                commitCalled.set(true);
                return CompletableFuture.completedFuture(false);
            }
        } else if (argument.equals(config.getAbortStreamName())) {
            abortStreamTruncationFuture.complete(null);
        } else if (argument.equals(config.getKvtStreamName())) {
            kvtStreamTruncationFuture.complete(null);
        }
        return CompletableFuture.completedFuture(true);
    }).when(streamMetadataTasks).startTruncation(anyString(), anyString(), any(), any());
    Set<String> processes = Sets.newHashSet("p1", "p2", "p3");
    // first throw checkpoint store exception
    AtomicBoolean signal = new AtomicBoolean(false);
    CountDownLatch cd = new CountDownLatch(4);
    doAnswer(x -> {
        // this ensures that the call to truncate has been invoked for all 4 internal streams.
        cd.countDown();
        cd.await();
        if (!signal.get()) {
            throw new CheckpointStoreException("CheckpointStoreException");
        } else {
            return processes;
        }
    }).when(checkpointStore).getProcesses();
    Map<String, PositionImpl> r1 = Collections.singletonMap("r1", position1);
    doReturn(r1).when(checkpointStore).getPositions(eq("p1"), anyString());
    Map<String, PositionImpl> r2 = Collections.singletonMap("r2", position1);
    doReturn(r2).when(checkpointStore).getPositions(eq("p2"), anyString());
    Map<String, PositionImpl> r3 = Collections.singletonMap("r3", position1);
    doReturn(r3).when(checkpointStore).getPositions(eq("p3"), anyString());
    @Cleanup ControllerEventProcessors processors = new ControllerEventProcessors("host1", config, controller, checkpointStore, streamStore, bucketStore, connectionPool, streamMetadataTasks, streamTransactionMetadataTasks, kvtStore, kvtTasks, system, executorService());
    // set truncation interval
    processors.setTruncationInterval(100L);
    processors.startAsync();
    processors.awaitRunning();
    ControllerEventProcessors processorsSpied = spy(processors);
    processorsSpied.bootstrap(streamTransactionMetadataTasks, streamMetadataTasks, kvtTasks);
    // wait for all 4 countdown exceptions to have been thrown.
    cd.await();
    verify(processorsSpied, atLeast(4)).truncate(any(), any(), any());
    verify(checkpointStore, atLeast(4)).getProcesses();
    verify(checkpointStore, never()).getPositions(anyString(), anyString());
    verify(streamMetadataTasks, never()).startTruncation(anyString(), anyString(), any(), any());
    signal.set(true);
    CompletableFuture.allOf(requestStreamTruncationFuture, commitStreamTruncationFuture, abortStreamTruncationFuture, kvtStreamTruncationFuture).join();
    // verify that truncate method is being called periodically.
    verify(processorsSpied, atLeastOnce()).truncate(config.getRequestStreamName(), config.getRequestReaderGroupName(), streamMetadataTasks);
    verify(processorsSpied, atLeastOnce()).truncate(config.getCommitStreamName(), config.getCommitReaderGroupName(), streamMetadataTasks);
    verify(processorsSpied, atLeastOnce()).truncate(config.getAbortStreamName(), config.getAbortReaderGroupName(), streamMetadataTasks);
    verify(processorsSpied, atLeastOnce()).truncate(config.getKvtStreamName(), config.getKvtReaderGroupName(), streamMetadataTasks);
    for (int i = 1; i <= 3; i++) {
        verify(checkpointStore, atLeastOnce()).getPositions("p" + i, config.getRequestReaderGroupName());
        verify(checkpointStore, atLeastOnce()).getPositions("p" + i, config.getCommitReaderGroupName());
        verify(checkpointStore, atLeastOnce()).getPositions("p" + i, config.getAbortReaderGroupName());
        verify(checkpointStore, atLeastOnce()).getPositions("p" + i, config.getKvtReaderGroupName());
    }
}

Example 9 with SegmentWithRange

use of io.pravega.client.stream.impl.SegmentWithRange in project pravega by pravega.

the class ControllerImplTest method testGetSegmentsImmediatlyFollowing.

@Test
public void testGetSegmentsImmediatlyFollowing() throws Exception {
    CompletableFuture<Map<SegmentWithRange, List<Long>>> successors;
    successors = controllerClient.getSuccessors(new Segment("scope1", "stream1", 0L)).thenApply(StreamSegmentsWithPredecessors::getSegmentToPredecessor);
    assertEquals(2, successors.get().size());
    assertEquals(20, successors.get().get(new SegmentWithRange(new Segment("scope1", "stream1", 2L), 0.0, 0.25)).get(0).longValue());
    assertEquals(30, successors.get().get(new SegmentWithRange(new Segment("scope1", "stream1", 3L), 0.25, 0.5)).get(0).longValue());
    successors = controllerClient.getSuccessors(new Segment("scope1", "stream2", 0L)).thenApply(StreamSegmentsWithPredecessors::getSegmentToPredecessor);
    AssertExtensions.assertFutureThrows("Should throw Exception", successors, throwable -> true);
}

Example 10 with SegmentWithRange

use of io.pravega.client.stream.impl.SegmentWithRange in project pravega by pravega.

the class ModelHelperTest method encodeSegmentRange.

@Test
public void encodeSegmentRange() {
    Controller.SegmentRange range = createSegmentRange(0.1, 0.5);
    SegmentWithRange result = ModelHelper.encode(range);
    assertEquals(0, result.getSegment().getSegmentId());
    assertEquals("testScope", result.getSegment().getScope());
    assertEquals("testStream", result.getSegment().getStreamName());
    final Controller.SegmentRange invalidMinSegrange = createSegmentRange(-0.1, 0.5);
    AssertExtensions.assertThrows("Unexpected behaviour of invalid minkey", () -> ModelHelper.encode(invalidMinSegrange), ex -> ex instanceof IllegalArgumentException);
    final Controller.SegmentRange invalidMinSegrange1 = createSegmentRange(1.5, 0.5);
    AssertExtensions.assertThrows("Unexpected behaviour of invalid minkey", () -> ModelHelper.encode(invalidMinSegrange1), ex -> ex instanceof IllegalArgumentException);
    final Controller.SegmentRange invalidMaxSegrange = createSegmentRange(0.1, 1.5);
    AssertExtensions.assertThrows("Unexpected behaviour of invalid minkey", () -> ModelHelper.encode(invalidMaxSegrange), ex -> ex instanceof IllegalArgumentException);
    final Controller.SegmentRange invalidMaxSegrange1 = createSegmentRange(0.1, -0.5);
    AssertExtensions.assertThrows("Unexpected behaviour of invalid minkey", () -> ModelHelper.encode(invalidMaxSegrange1), ex -> ex instanceof IllegalArgumentException);
}