Examples with StreamMetadataTasks io.pravega.controller.task.Stream.StreamMetadataTasks used on opensource projects

Example 11 with StreamMetadataTasks

use of io.pravega.controller.task.Stream.StreamMetadataTasks in project pravega by pravega.

the class ControllerEventProcessorTest method setUp.

@Before
public void setUp() throws Exception {
    executor = ExecutorServiceHelpers.newScheduledThreadPool(10, "test");
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new RetryOneTime(2000));
    zkClient.start();
    streamStore = createStore();
    bucketStore = StreamStoreFactory.createZKBucketStore(PRAVEGA_ZK_CURATOR_RESOURCE.client, executor);
    hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    eventHelperMock = EventHelperMock.getEventHelperMock(executor, "1", ((AbstractStreamMetadataStore) this.streamStore).getHostTaskIndex());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, bucketStore, TaskStoreFactory.createInMemoryStore(executor), segmentHelperMock, executor, "1", GrpcAuthHelper.getDisabledAuthHelper(), eventHelperMock);
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, segmentHelperMock, executor, "host", GrpcAuthHelper.getDisabledAuthHelper());
    streamTransactionMetadataTasks.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    // region createStream
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    streamStore.createScope(SCOPE, null, executor).join();
    long start = System.currentTimeMillis();
    streamStore.createStream(SCOPE, STREAM, configuration1, start, null, executor).join();
    streamStore.setState(SCOPE, STREAM, State.ACTIVE, null, executor).join();
// endregion
}

Example 12 with StreamMetadataTasks

use of io.pravega.controller.task.Stream.StreamMetadataTasks in project pravega by pravega.

the class ControllerServiceWithKVTableTest method setup.

@Before
public void setup() {
    segmentHelperMock = SegmentHelperMock.getSegmentHelperMockForTables(executor);
    streamStore = spy(getStore());
    kvtStore = spy(getKVTStore());
    BucketStore bucketStore = StreamStoreFactory.createZKBucketStore(PRAVEGA_ZK_CURATOR_RESOURCE.client, executor);
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(PRAVEGA_ZK_CURATOR_RESOURCE.client, executor);
    connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    GrpcAuthHelper disabledAuthHelper = GrpcAuthHelper.getDisabledAuthHelper();
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    EventHelper helperMock = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, bucketStore, taskMetadataStore, segmentHelperMock, executor, "host", disabledAuthHelper, helperMock);
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, segmentHelperMock, executor, "host", disabledAuthHelper);
    kvtMetadataTasks = spy(new TableMetadataTasks(kvtStore, segmentHelperMock, executor, executor, "host", GrpcAuthHelper.getDisabledAuthHelper(), helperMock));
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), new CreateReaderGroupTask(streamMetadataTasks, streamStore, executor), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, executor), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, executor), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, executor), executor);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executor));
    consumer = new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null, requestTracker);
}

Example 13 with StreamMetadataTasks

use of io.pravega.controller.task.Stream.StreamMetadataTasks in project pravega by pravega.

the class RequestHandlersTest method testDeleteBucketReferences.

@Test
public void testDeleteBucketReferences() {
    String stream = "deleteReferences";
    createStreamInStore(stream, scope);
    String scopedStreamName = NameUtils.getScopedStreamName(scope, stream);
    int watermarkingBuckets = bucketStore.getBucketCount(BucketStore.ServiceType.WatermarkingService);
    int retentionBuckets = bucketStore.getBucketCount(BucketStore.ServiceType.RetentionService);
    bucketStore.addStreamToBucketStore(BucketStore.ServiceType.RetentionService, scope, stream, executor).join();
    bucketStore.addStreamToBucketStore(BucketStore.ServiceType.WatermarkingService, scope, stream, executor).join();
    // seal stream.
    // set state to sealing and send event to the processor
    streamStore.setState(scope, stream, State.SEALING, null, executor).join();
    streamStore.setState(scope, stream, State.SEALED, null, executor).join();
    List<String> retentionStreams = IntStream.range(0, retentionBuckets).boxed().map(x -> bucketStore.getStreamsForBucket(BucketStore.ServiceType.RetentionService, x, executor).join()).flatMap(Collection::stream).collect(Collectors.toList());
    assertTrue(retentionStreams.contains(scopedStreamName));
    List<String> watermarkStreams = IntStream.range(0, watermarkingBuckets).boxed().map(x -> bucketStore.getStreamsForBucket(BucketStore.ServiceType.WatermarkingService, x, executor).join()).flatMap(Collection::stream).collect(Collectors.toList());
    assertTrue(watermarkStreams.contains(scopedStreamName));
    // delete the stream
    DeleteStreamTask deleteStreamTask = new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, executor);
    long creationTime = streamStore.getCreationTime(scope, stream, null, executor).join();
    DeleteStreamEvent firstDeleteEvent = new DeleteStreamEvent(scope, stream, 0L, creationTime);
    deleteStreamTask.execute(firstDeleteEvent).join();
    watermarkStreams = IntStream.range(0, watermarkingBuckets).boxed().map(x -> bucketStore.getStreamsForBucket(BucketStore.ServiceType.WatermarkingService, x, executor).join()).flatMap(Collection::stream).collect(Collectors.toList());
    assertFalse(watermarkStreams.contains(scopedStreamName));
    retentionStreams = IntStream.range(0, retentionBuckets).boxed().map(x -> bucketStore.getStreamsForBucket(BucketStore.ServiceType.RetentionService, x, executor).join()).flatMap(Collection::stream).collect(Collectors.toList());
    assertFalse(retentionStreams.contains(scopedStreamName));
}

Example 14 with StreamMetadataTasks

use of io.pravega.controller.task.Stream.StreamMetadataTasks in project pravega by pravega.

the class ControllerGrpcAuthFocusedTest method setup.

@Before
public void setup() throws IOException {
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createInMemoryStore(EXECUTOR);
    StreamMetadataStore streamStore = StreamStoreFactory.createInMemoryStore();
    this.kvtStore = spy(KVTableStoreFactory.createInMemoryStore(streamStore, EXECUTOR));
    BucketStore bucketStore = StreamStoreFactory.createInMemoryBucketStore();
    SegmentHelper segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    RequestTracker requestTracker = new RequestTracker(true);
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    GrpcAuthHelper authHelper = new GrpcAuthHelper(true, "secret", 300);
    EventHelper helper = EventHelperMock.getEventHelperMock(EXECUTOR, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, bucketStore, taskMetadataStore, segmentHelper, EXECUTOR, "host", authHelper, helper);
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, segmentHelper, EXECUTOR, "host", authHelper);
    kvtMetadataTasks = new TableMetadataTasks(kvtStore, segmentHelper, EXECUTOR, EXECUTOR, "host", authHelper, helper);
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, EXECUTOR), new ScaleOperationTask(streamMetadataTasks, streamStore, EXECUTOR), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, EXECUTOR), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, EXECUTOR), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, EXECUTOR), new TruncateStreamTask(streamMetadataTasks, streamStore, EXECUTOR), new CreateReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, EXECUTOR), EXECUTOR);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, EXECUTOR));
    streamTransactionMetadataTasks.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    Cluster mockCluster = mock(Cluster.class);
    when(mockCluster.getClusterMembers()).thenReturn(Collections.singleton(new Host("localhost", 9090, null)));
    ControllerServiceGrpc.ControllerServiceImplBase controllerServiceImplBase = new ControllerServiceImpl(new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelper, EXECUTOR, mockCluster, requestTracker), authHelper, requestTracker, true, true, 2);
    ControllerServiceGrpc.ControllerServiceImplBase controllerServiceImplBaseStrict = new ControllerServiceImpl(new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelper, EXECUTOR, mockCluster, requestTracker), authHelper, requestTracker, true, false, 2);
    PasswordAuthHandler authHandler = new PasswordAuthHandler();
    authHandler.initialize(AUTH_FILE.getAbsolutePath());
    String uniqueServerName = String.format("Test server name: %s", getClass());
    String uniqueServerNameStrict = String.format("Test server name: %sStrict", getClass());
    // Using a builder that creates a server for servicing in-process requests.
    // Also, using a direct executor which executes app code directly in transport thread. See
    // https://grpc.io/grpc-java/javadoc/io/grpc/inprocess/InProcessServerBuilder.html for more information.
    grpcServer = InProcessServerBuilder.forName(uniqueServerName).addService(ServerInterceptors.intercept(controllerServiceImplBase, new AuthInterceptor(authHandler))).directExecutor().build().start();
    grpcServerStrict = InProcessServerBuilder.forName(uniqueServerNameStrict).addService(ServerInterceptors.intercept(controllerServiceImplBaseStrict, new AuthInterceptor(authHandler))).directExecutor().build().start();
    inProcessChannel = InProcessChannelBuilder.forName(uniqueServerName).directExecutor().build();
    inProcessChannelStrict = InProcessChannelBuilder.forName(uniqueServerNameStrict).directExecutor().build();
}

Example 15 with StreamMetadataTasks

use of io.pravega.controller.task.Stream.StreamMetadataTasks in project pravega by pravega.

the class ScaleRequestHandlerTest method testScaleRequest.

@SuppressWarnings("unchecked")
@Test(timeout = 30000)
public void testScaleRequest() throws ExecutionException, InterruptedException {
    AutoScaleTask requestHandler = new AutoScaleTask(streamMetadataTasks, streamStore, executor);
    ScaleOperationTask scaleRequestHandler = new ScaleOperationTask(streamMetadataTasks, streamStore, executor);
    StreamRequestHandler multiplexer = new StreamRequestHandler(requestHandler, scaleRequestHandler, null, null, null, null, null, null, null, streamStore, null, executor);
    // Send number of splits = 1
    EventWriterMock writer = new EventWriterMock();
    streamMetadataTasks.setRequestEventWriter(writer);
    AutoScaleEvent scaleUpEvent = new AutoScaleEvent(scope, stream, 2, AutoScaleEvent.UP, System.currentTimeMillis(), 1, false, System.currentTimeMillis());
    assertTrue(Futures.await(multiplexer.process(scaleUpEvent, () -> false)));
    // verify that one scaleOp event is written into the stream
    assertEquals(1, writer.queue.size());
    ControllerEvent event = writer.queue.take();
    assertTrue(event instanceof ScaleOpEvent);
    ScaleOpEvent scaleOpEvent = (ScaleOpEvent) event;
    double start = 2.0 / 3.0;
    double end = 1.0;
    double middle = (start + end) / 2;
    assertEquals(2, scaleOpEvent.getNewRanges().size());
    double delta = 0.0000000000001;
    assertEquals(start, scaleOpEvent.getNewRanges().get(0).getKey(), delta);
    assertEquals(middle, scaleOpEvent.getNewRanges().get(0).getValue(), delta);
    assertEquals(middle, scaleOpEvent.getNewRanges().get(1).getKey(), delta);
    assertEquals(end, scaleOpEvent.getNewRanges().get(1).getValue(), delta);
    assertEquals(1, scaleOpEvent.getSegmentsToSeal().size());
    assertTrue(scaleOpEvent.getSegmentsToSeal().contains(2L));
    assertTrue(Futures.await(multiplexer.process(scaleOpEvent, () -> false)));
    // verify that the event is processed successfully
    List<StreamSegmentRecord> activeSegments = streamStore.getActiveSegments(scope, stream, null, executor).get();
    assertTrue(activeSegments.stream().noneMatch(z -> z.segmentId() == 2L));
    // verify that two splits are created even when we sent 1 as numOfSplits in AutoScaleEvent.
    long three = computeSegmentId(3, 1);
    long four = computeSegmentId(4, 1);
    assertTrue(activeSegments.stream().anyMatch(z -> z.segmentId() == three));
    assertTrue(activeSegments.stream().anyMatch(z -> z.segmentId() == four));
    assertTrue(activeSegments.size() == 4);
    // process first scale down event. it should only mark the segment as cold
    AutoScaleEvent scaleDownEvent = new AutoScaleEvent(scope, stream, four, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis());
    assertTrue(Futures.await(multiplexer.process(scaleDownEvent, () -> false)));
    assertTrue(writer.queue.isEmpty());
    activeSegments = streamStore.getActiveSegments(scope, stream, null, executor).get();
    assertTrue(activeSegments.stream().anyMatch(z -> z.segmentId() == four));
    assertTrue(activeSegments.size() == 4);
    assertTrue(streamStore.isCold(scope, stream, four, null, executor).join());
    AutoScaleEvent scaleDownEvent2 = new AutoScaleEvent(scope, stream, three, AutoScaleEvent.DOWN, System.currentTimeMillis(), 0, false, System.currentTimeMillis());
    assertTrue(Futures.await(multiplexer.process(scaleDownEvent2, () -> false)));
    assertTrue(streamStore.isCold(scope, stream, three, null, executor).join());
    // verify that a new event has been posted
    assertEquals(1, writer.queue.size());
    event = writer.queue.take();
    assertTrue(event instanceof ScaleOpEvent);
    scaleOpEvent = (ScaleOpEvent) event;
    assertEquals(1, scaleOpEvent.getNewRanges().size());
    assertEquals(start, scaleOpEvent.getNewRanges().get(0).getKey(), delta);
    assertEquals(end, scaleOpEvent.getNewRanges().get(0).getValue(), delta);
    assertEquals(2, scaleOpEvent.getSegmentsToSeal().size());
    assertTrue(scaleOpEvent.getSegmentsToSeal().contains(three));
    assertTrue(scaleOpEvent.getSegmentsToSeal().contains(four));
    // process scale down event
    assertTrue(Futures.await(multiplexer.process(scaleOpEvent, () -> false)));
    long five = computeSegmentId(5, 2);
    activeSegments = streamStore.getActiveSegments(scope, stream, null, executor).get();
    assertTrue(activeSegments.stream().noneMatch(z -> z.segmentId() == three));
    assertTrue(activeSegments.stream().noneMatch(z -> z.segmentId() == four));
    assertTrue(activeSegments.stream().anyMatch(z -> z.segmentId() == five));
    assertTrue(activeSegments.size() == 3);
    // make it throw a non retryable failure so that test does not wait for number of retries.
    // This will bring down the test duration drastically because a retryable failure can keep retrying for few seconds.
    // And if someone changes retry durations and number of attempts in retry helper, it will impact this test's running time.
    // hence sending incorrect segmentsToSeal list which will result in a non retryable failure and this will fail immediately
    assertFalse(Futures.await(multiplexer.process(new ScaleOpEvent(scope, stream, Lists.newArrayList(five), Lists.newArrayList(new AbstractMap.SimpleEntry<>(0.5, 1.0)), false, System.currentTimeMillis(), System.currentTimeMillis()), () -> false)));
    activeSegments = streamStore.getActiveSegments(scope, stream, null, executor).get();
    assertTrue(activeSegments.stream().noneMatch(z -> z.segmentId() == three));
    assertTrue(activeSegments.stream().noneMatch(z -> z.segmentId() == four));
    assertTrue(activeSegments.stream().anyMatch(z -> z.segmentId() == five));
    assertTrue(activeSegments.size() == 3);
    assertFalse(Futures.await(multiplexer.process(new AbortEvent(scope, stream, 0, UUID.randomUUID(), 11L), () -> false)));
}