Examples with StreamMetadataTasks io.pravega.controller.task.Stream.StreamMetadataTasks used on opensource projects

Example 16 with StreamMetadataTasks

use of io.pravega.controller.task.Stream.StreamMetadataTasks in project pravega by pravega.

the class ControllerServiceWithZKStreamTest method setup.

@Before
public void setup() {
    try {
        zkServer = new TestingServerStarter().start();
    } catch (Exception e) {
        log.error("Error starting ZK server", e);
    }
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetadataStore streamStore = StreamStoreFactory.createZKStore(zkClient, executor);
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executor), new ScaleOperationTask(streamMetadataTasks, streamStore, executor), new UpdateStreamTask(streamMetadataTasks, streamStore, executor), new SealStreamTask(streamMetadataTasks, streamStore, executor), new DeleteStreamTask(streamMetadataTasks, streamStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStore, executor), executor);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executor));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null);
}

Example 17 with StreamMetadataTasks

use of io.pravega.controller.task.Stream.StreamMetadataTasks in project pravega by pravega.

the class ControllerEventProcessorTest method testCommitEventProcessorFailedWrite.

@Test(timeout = 10000)
public void testCommitEventProcessorFailedWrite() {
    UUID txnId = UUID.randomUUID();
    VersionedTransactionData txnData = streamStore.createTransaction(SCOPE, STREAM, txnId, 10000, 10000, 10000, null, executor).join();
    CommitEventProcessor commitEventProcessor = spy(new CommitEventProcessor(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, null));
    EventProcessor.Writer<CommitEvent> successWriter = event -> CompletableFuture.completedFuture(null);
    EventProcessor.Writer<CommitEvent> failedWriter = event -> {
        CompletableFuture<Void> future = new CompletableFuture<>();
        future.completeExceptionally(new RuntimeException("Error"));
        return future;
    };
    // Simulate a failed write
    when(commitEventProcessor.getSelfWriter()).thenReturn(failedWriter).thenReturn(successWriter);
    // invoke process with epoch > txnData.
    commitEventProcessor.process(new CommitEvent(SCOPE, STREAM, txnData.getEpoch() + 1, txnData.getId()), null);
    verify(commitEventProcessor, times(2)).getSelfWriter();
}

Example 18 with StreamMetadataTasks

use of io.pravega.controller.task.Stream.StreamMetadataTasks in project pravega by pravega.

the class InMemoryControllerServiceImplTest method setup.

@Override
public void setup() throws Exception {
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(20, "testpool");
    taskMetadataStore = TaskStoreFactory.createInMemoryStore(executorService);
    hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    streamStore = StreamStoreFactory.createInMemoryStore(executorService);
    segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executorService), new ScaleOperationTask(streamMetadataTasks, streamStore, executorService), new UpdateStreamTask(streamMetadataTasks, streamStore, executorService), new SealStreamTask(streamMetadataTasks, streamStore, executorService), new DeleteStreamTask(streamMetadataTasks, streamStore, executorService), new TruncateStreamTask(streamMetadataTasks, streamStore, executorService), executorService);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executorService));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    streamTransactionMetadataTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
    Cluster mockCluster = mock(Cluster.class);
    when(mockCluster.getClusterMembers()).thenReturn(Collections.singleton(new Host("localhost", 9090, null)));
    controllerService = new ControllerServiceImpl(new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, new SegmentHelper(), executorService, mockCluster), "secret", false);
}

Example 19 with StreamMetadataTasks

use of io.pravega.controller.task.Stream.StreamMetadataTasks in project pravega by pravega.

the class ZKControllerServiceImplTest method setup.

@Override
public void setup() throws Exception {
    final StreamMetadataStore streamStore;
    final HostControllerStore hostStore;
    final TaskMetadataStore taskMetadataStore;
    final SegmentHelper segmentHelper;
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    storeClient = StoreClientFactory.createZKStoreClient(zkClient);
    executorService = ExecutorServiceHelpers.newScheduledThreadPool(20, "testpool");
    taskMetadataStore = TaskStoreFactory.createStore(storeClient, executorService);
    hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    streamStore = StreamStoreFactory.createZKStore(zkClient, executorService);
    segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, executorService), new ScaleOperationTask(streamMetadataTasks, streamStore, executorService), new UpdateStreamTask(streamMetadataTasks, streamStore, executorService), new SealStreamTask(streamMetadataTasks, streamStore, executorService), new DeleteStreamTask(streamMetadataTasks, streamStore, executorService), new TruncateStreamTask(streamMetadataTasks, streamStore, executorService), executorService);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executorService));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelper, executorService, "host", connectionFactory, false, "");
    streamTransactionMetadataTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
    cluster = new ClusterZKImpl(zkClient, ClusterType.CONTROLLER);
    final CountDownLatch latch = new CountDownLatch(1);
    cluster.addListener((type, host) -> latch.countDown());
    cluster.registerHost(new Host("localhost", 9090, null));
    latch.await();
    ControllerService controller = new ControllerService(streamStore, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, new SegmentHelper(), executorService, cluster);
    controllerService = new ControllerServiceImpl(controller, "", false);
}

Example 20 with StreamMetadataTasks

use of io.pravega.controller.task.Stream.StreamMetadataTasks in project pravega by pravega.

the class TaskTest method testStreamTaskSweeping.

@Test(timeout = 10000)
public void testStreamTaskSweeping() throws Exception {
    final String stream = "testPartialCreationStream";
    final String deadHost = "deadHost";
    final int initialSegments = 2;
    final ScalingPolicy policy1 = ScalingPolicy.fixed(initialSegments);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    final ArrayList<Long> sealSegments = new ArrayList<>();
    sealSegments.add(0L);
    final ArrayList<AbstractMap.SimpleEntry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.0, 0.25));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.25, 0.5));
    // Create objects.
    @Cleanup StreamMetadataTasks mockStreamTasks = new StreamMetadataTasks(streamStore, StreamStoreFactory.createInMemoryBucketStore(), taskMetadataStore, segmentHelperMock, executor, deadHost, GrpcAuthHelper.getDisabledAuthHelper());
    mockStreamTasks.setCreateIndexOnlyMode();
    TaskSweeper sweeper = new TaskSweeper(taskMetadataStore, HOSTNAME, executor, streamMetadataTasks);
    // Create stream test.
    // this will create 2 new streams -> stream + mark stream
    completePartialTask(mockStreamTasks.createStream(SCOPE, stream, configuration1, System.currentTimeMillis(), 0L), deadHost, sweeper);
    Assert.assertEquals(initialSegments, streamStore.getActiveSegments(SCOPE, stream, null, executor).join().size());
    Map<String, StreamConfiguration> streams = streamStore.listStreamsInScope(SCOPE, null, executor).join();
    assertEquals(4, streams.size());
    assertEquals(configuration1, streams.get(stream));
}