Examples with UpdateStreamTask io.pravega.controller.server.eventProcessor.requesthandlers.UpdateStreamTask used on opensource projects

Example 1 with UpdateStreamTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.UpdateStreamTask in project pravega by pravega.

the class StreamMetadataTasksTest method updateStreamTest.

@Test(timeout = 30000)
public void updateStreamTest() throws Exception {
    assertNotEquals(0, consumer.getCurrentSegments(SCOPE, stream1).get().size());
    WriterMock requestEventWriter = new WriterMock(streamMetadataTasks, executor);
    streamMetadataTasks.setRequestEventWriter(requestEventWriter);
    StreamConfiguration streamConfiguration = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(ScalingPolicy.fixed(5)).build();
    StreamProperty<StreamConfiguration> configProp = streamStorePartialMock.getConfigurationProperty(SCOPE, stream1, true, null, executor).join();
    assertFalse(configProp.isUpdating());
    // 1. happy day test
    // update.. should succeed
    CompletableFuture<UpdateStreamStatus.Status> updateOperationFuture = streamMetadataTasks.updateStream(SCOPE, stream1, streamConfiguration, null);
    assertTrue(Futures.await(processEvent(requestEventWriter)));
    assertEquals(UpdateStreamStatus.Status.SUCCESS, updateOperationFuture.join());
    configProp = streamStorePartialMock.getConfigurationProperty(SCOPE, stream1, true, null, executor).join();
    assertTrue(configProp.getProperty().equals(streamConfiguration));
    streamConfiguration = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(ScalingPolicy.fixed(6)).build();
    // 2. change state to scaling
    streamStorePartialMock.setState(SCOPE, stream1, State.SCALING, null, executor).get();
    // call update should fail without posting the event
    streamMetadataTasks.updateStream(SCOPE, stream1, streamConfiguration, null);
    AtomicBoolean loop = new AtomicBoolean(false);
    Futures.loop(() -> !loop.get(), () -> streamStorePartialMock.getConfigurationProperty(SCOPE, stream1, true, null, executor).thenApply(StreamProperty::isUpdating).thenAccept(loop::set), executor).join();
    // event posted, first step performed. now pick the event for processing
    UpdateStreamTask updateStreamTask = new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, executor);
    UpdateStreamEvent taken = (UpdateStreamEvent) requestEventWriter.eventQueue.take();
    AssertExtensions.assertThrows("", updateStreamTask.execute(taken), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    // now with state = active, process the same event. it should succeed now.
    assertTrue(Futures.await(updateStreamTask.execute(taken)));
    // 3. multiple back to back updates.
    StreamConfiguration streamConfiguration1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(ScalingPolicy.byEventRate(1, 1, 2)).build();
    CompletableFuture<UpdateStreamStatus.Status> updateOperationFuture1 = streamMetadataTasks.updateStream(SCOPE, stream1, streamConfiguration1, null);
    // ensure that previous updatestream has posted the event and set status to updating,
    // only then call second updateStream
    AtomicBoolean loop2 = new AtomicBoolean(false);
    Futures.loop(() -> !loop2.get(), () -> streamStorePartialMock.getConfigurationProperty(SCOPE, stream1, true, null, executor).thenApply(StreamProperty::isUpdating).thenAccept(loop2::set), executor).join();
    configProp = streamStorePartialMock.getConfigurationProperty(SCOPE, stream1, true, null, executor).join();
    assertTrue(configProp.getProperty().equals(streamConfiguration1) && configProp.isUpdating());
    StreamConfiguration streamConfiguration2 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(ScalingPolicy.fixed(7)).build();
    // post the second update request. This should fail here itself as previous one has started.
    CompletableFuture<UpdateStreamStatus.Status> updateOperationFuture2 = streamMetadataTasks.updateStream(SCOPE, stream1, streamConfiguration2, null);
    assertEquals(UpdateStreamStatus.Status.FAILURE, updateOperationFuture2.join());
    // process event
    assertTrue(Futures.await(processEvent(requestEventWriter)));
    // verify that first request for update also completes with success.
    assertEquals(UpdateStreamStatus.Status.SUCCESS, updateOperationFuture1.join());
    configProp = streamStorePartialMock.getConfigurationProperty(SCOPE, stream1, true, null, executor).join();
    assertTrue(configProp.getProperty().equals(streamConfiguration1) && !configProp.isUpdating());
}

Example 2 with UpdateStreamTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.UpdateStreamTask in project pravega by pravega.

the class StreamMetadataTasksTest method setup.

@Before
public void setup() throws Exception {
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetadataStore streamStore = StreamStoreFactory.createInMemoryStore(1, executor);
    // create a partial mock.
    streamStorePartialMock = spy(streamStore);
    doReturn(CompletableFuture.completedFuture(false)).when(streamStorePartialMock).isTransactionOngoing(anyString(), anyString(), any(), // mock only isTransactionOngoing call.
    any());
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    streamMetadataTasks = spy(new StreamMetadataTasks(streamStorePartialMock, hostStore, taskMetadataStore, segmentHelperMock, executor, "host", connectionFactory, authEnabled, "key"));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStorePartialMock, hostStore, segmentHelperMock, executor, "host", connectionFactory, authEnabled, "key");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStorePartialMock, executor), new ScaleOperationTask(streamMetadataTasks, streamStorePartialMock, executor), new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new SealStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new DeleteStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new TruncateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), executor);
    consumer = new ControllerService(streamStorePartialMock, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    streamStorePartialMock.createScope(SCOPE).join();
    long start = System.currentTimeMillis();
    streamStorePartialMock.createStream(SCOPE, stream1, configuration1, start, null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    AbstractMap.SimpleEntry<Double, Double> segment1 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment2 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Integer> sealedSegments = Collections.singletonList(1);
    StartScaleResponse response = streamStorePartialMock.startScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), start + 20, false, null, executor).get();
    List<Segment> segmentsCreated = response.getSegmentsCreated();
    streamStorePartialMock.setState(SCOPE, stream1, State.SCALING, null, executor).get();
    streamStorePartialMock.scaleNewSegmentsCreated(SCOPE, stream1, sealedSegments, segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
    streamStorePartialMock.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
}

Example 3 with UpdateStreamTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.UpdateStreamTask in project pravega by pravega.

the class StreamMetadataTasksTest method retentionPolicyUpdateTest.

@Test(timeout = 30000)
public void retentionPolicyUpdateTest() throws Exception {
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    String stream = "test";
    final StreamConfiguration noRetentionConfig = StreamConfiguration.builder().scope(SCOPE).streamName(stream).scalingPolicy(policy).build();
    // add stream without retention policy
    streamMetadataTasks.createStreamBody(SCOPE, stream, noRetentionConfig, System.currentTimeMillis()).join();
    String scopedStreamName = String.format("%s/%s", SCOPE, stream);
    // verify that stream is not added to bucket
    assertTrue(!streamStorePartialMock.getStreamsForBucket(0, executor).join().contains(scopedStreamName));
    UpdateStreamTask task = new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, executor);
    final RetentionPolicy retentionPolicy = RetentionPolicy.builder().retentionType(RetentionPolicy.RetentionType.TIME).retentionParam(Duration.ofMinutes(60).toMillis()).build();
    final StreamConfiguration withRetentionConfig = StreamConfiguration.builder().scope(SCOPE).streamName(stream).scalingPolicy(policy).retentionPolicy(retentionPolicy).build();
    // now update stream with a retention policy
    streamStorePartialMock.startUpdateConfiguration(SCOPE, stream, withRetentionConfig, null, executor).join();
    UpdateStreamEvent update = new UpdateStreamEvent(SCOPE, stream);
    task.execute(update).join();
    // verify that bucket has the stream.
    assertTrue(streamStorePartialMock.getStreamsForBucket(0, executor).join().contains(scopedStreamName));
    // update stream such that stream is updated with null retention policy
    streamStorePartialMock.startUpdateConfiguration(SCOPE, stream, noRetentionConfig, null, executor).join();
    task.execute(update).join();
    // verify that the stream is no longer present in the bucket
    assertTrue(!streamStorePartialMock.getStreamsForBucket(0, executor).join().contains(scopedStreamName));
}

Example 4 with UpdateStreamTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.UpdateStreamTask in project pravega by pravega.

the class ControllerServiceWithZKStreamTest method setup.

@Before
public void setup() {
    try {
        zkServer = new TestingServerStarter().start();
    } catch (Exception e) {
        log.error("Error starting ZK server", e);
    }
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetadataStore streamStore = StreamStoreFactory.createZKStore(zkClient, executor);
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, executor), new SealStreamTask(streamMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), executor);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executor));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null);
}

Example 5 with UpdateStreamTask

use of io.pravega.controller.server.eventProcessor.requesthandlers.UpdateStreamTask in project pravega by pravega.

the class InMemoryControllerServiceImplTest method setup.

@Override
public void setup() throws Exception {
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(20, "testpool");
    taskMetadataStore = TaskStoreFactory.createInMemoryStore(executorService);
    hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    streamStore = StreamStoreFactory.createInMemoryStore(executorService);
    segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executorService), new ScaleOperationTask(streamMetadataTasks, streamStore, executorService), new UpdateStreamTask(streamMetadataTasks, streamStore, executorService), new SealStreamTask(streamMetadataTasks, streamStore, executorService), new DeleteStreamTask(streamMetadataTasks, streamStore, executorService), new TruncateStreamTask(streamMetadataTasks, streamStore, executorService), executorService);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executorService));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    streamTransactionMetadataTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
    Cluster mockCluster = mock(Cluster.class);
    when(mockCluster.getClusterMembers()).thenReturn(Collections.singleton(new Host("localhost", 9090, null)));
    controllerService = new ControllerServiceImpl(new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, new SegmentHelper(), executorService, mockCluster), "secret", false);
}