Examples with TableMetadataTasks io.pravega.controller.task.KeyValueTable.TableMetadataTasks used on opensource projects

Example 6 with TableMetadataTasks

use of io.pravega.controller.task.KeyValueTable.TableMetadataTasks in project pravega by pravega.

the class ControllerEventProcessorsTest method testIsReady.

@Test(timeout = 30000L)
public void testIsReady() throws Exception {
    LocalController controller = mock(LocalController.class);
    StreamMetadataStore streamStore = mock(StreamMetadataStore.class);
    BucketStore bucketStore = mock(BucketStore.class);
    ConnectionPool connectionPool = mock(ConnectionPool.class);
    StreamMetadataTasks streamMetadataTasks = mock(StreamMetadataTasks.class);
    StreamTransactionMetadataTasks streamTransactionMetadataTasks = mock(StreamTransactionMetadataTasks.class);
    KVTableMetadataStore kvtStore = mock(KVTableMetadataStore.class);
    TableMetadataTasks kvtTasks = mock(TableMetadataTasks.class);
    ControllerEventProcessorConfig config = ControllerEventProcessorConfigImpl.withDefault();
    EventProcessorSystem system = mock(EventProcessorSystem.class);
    CuratorZookeeperClient curatorZKClientMock = mock(CuratorZookeeperClient.class);
    CuratorFramework client = mock(CuratorFramework.class);
    Listenable listen = mock(Listenable.class);
    doNothing().when(listen).addListener(any(ConnectionStateListener.class));
    doReturn(listen).when(client).getConnectionStateListenable();
    doReturn(curatorZKClientMock).when(client).getZookeeperClient();
    doReturn(true).when(curatorZKClientMock).isConnected();
    ZKCheckpointStore checkpointStore = (ZKCheckpointStore) CheckpointStoreFactory.createZKStore(client);
    doAnswer(x -> null).when(streamMetadataTasks).initializeStreamWriters(any(), any());
    doAnswer(x -> null).when(streamTransactionMetadataTasks).initializeStreamWriters(any(EventStreamClientFactory.class), any(ControllerEventProcessorConfig.class));
    CompletableFuture<Boolean> createScopeResponseFuture = new CompletableFuture<>();
    CompletableFuture<Void> createScopeSignalFuture = new CompletableFuture<>();
    doAnswer(x -> {
        createScopeSignalFuture.complete(null);
        return createScopeResponseFuture;
    }).when(controller).createScope(anyString());
    LinkedBlockingQueue<CompletableFuture<Boolean>> createStreamResponses = new LinkedBlockingQueue<>();
    LinkedBlockingQueue<CompletableFuture<Void>> createStreamSignals = new LinkedBlockingQueue<>();
    List<CompletableFuture<Boolean>> createStreamResponsesList = new LinkedList<>();
    List<CompletableFuture<Void>> createStreamSignalsList = new LinkedList<>();
    for (int i = 0; i < 4; i++) {
        CompletableFuture<Boolean> responseFuture = new CompletableFuture<>();
        CompletableFuture<Void> signalFuture = new CompletableFuture<>();
        createStreamResponsesList.add(responseFuture);
        createStreamResponses.add(responseFuture);
        createStreamSignalsList.add(signalFuture);
        createStreamSignals.add(signalFuture);
    }
    // return a future from latches queue
    doAnswer(x -> {
        createStreamSignals.take().complete(null);
        return createStreamResponses.take();
    }).when(controller).createInternalStream(anyString(), anyString(), any());
    @Cleanup ControllerEventProcessors processors = spy(new ControllerEventProcessors("host1", config, controller, checkpointStore, streamStore, bucketStore, connectionPool, streamMetadataTasks, streamTransactionMetadataTasks, kvtStore, kvtTasks, system, executorService()));
    // Check isReady() method before invoking bootstrap
    Assert.assertFalse(processors.getBootstrapCompleted().get());
    Assert.assertTrue(processors.isMetadataServiceConnected());
    Assert.assertFalse(processors.isRunning());
    Assert.assertFalse(processors.isReady());
    // Call bootstrap on ControllerEventProcessors
    processors.bootstrap(streamTransactionMetadataTasks, streamMetadataTasks, kvtTasks);
    // Wait on create scope being called.
    createScopeSignalFuture.join();
    createScopeResponseFuture.complete(true);
    createStreamSignalsList.get(0).join();
    createStreamSignalsList.get(1).join();
    createStreamSignalsList.get(2).join();
    createStreamSignalsList.get(3).join();
    createStreamResponsesList.get(0).complete(true);
    createStreamResponsesList.get(1).complete(true);
    createStreamResponsesList.get(2).complete(true);
    createStreamResponsesList.get(3).complete(true);
    AssertExtensions.assertEventuallyEquals(true, () -> processors.getBootstrapCompleted().get(), 10000);
    Assert.assertTrue(processors.isMetadataServiceConnected());
    Assert.assertFalse(processors.isRunning());
    Assert.assertFalse(processors.isReady());
    EventProcessorGroup mockEventProcessorGroup = mock(EventProcessorGroup.class);
    doNothing().when(mockEventProcessorGroup).awaitRunning();
    doReturn(mockEventProcessorGroup).when(system).createEventProcessorGroup(any(EventProcessorConfig.class), any(CheckpointStore.class), any(ScheduledExecutorService.class));
    processors.startAsync();
    processors.awaitRunning();
    Assert.assertTrue(processors.isMetadataServiceConnected());
    Assert.assertTrue(processors.isBootstrapCompleted());
    Assert.assertTrue(processors.isRunning());
    Assert.assertTrue(processors.isReady());
}

Example 7 with TableMetadataTasks

use of io.pravega.controller.task.KeyValueTable.TableMetadataTasks in project pravega by pravega.

the class ControllerEventProcessorsTest method testHandleOrphaned.

@Test(timeout = 10000)
public void testHandleOrphaned() throws CheckpointStoreException {
    LocalController localController = mock(LocalController.class);
    CheckpointStore checkpointStore = mock(CheckpointStore.class);
    StreamMetadataStore streamStore = mock(StreamMetadataStore.class);
    BucketStore bucketStore = mock(BucketStore.class);
    ConnectionPool connectionPool = mock(ConnectionPool.class);
    StreamMetadataTasks streamMetadataTasks = mock(StreamMetadataTasks.class);
    StreamTransactionMetadataTasks streamTransactionMetadataTasks = mock(StreamTransactionMetadataTasks.class);
    KVTableMetadataStore kvtStore = mock(KVTableMetadataStore.class);
    TableMetadataTasks kvtTasks = mock(TableMetadataTasks.class);
    ControllerEventProcessorConfig config = ControllerEventProcessorConfigImpl.withDefault();
    EventProcessorSystem system = mock(EventProcessorSystem.class);
    EventProcessorGroup<ControllerEvent> processor = getProcessor();
    EventProcessorGroup<ControllerEvent> mockProcessor = spy(processor);
    doThrow(new CheckpointStoreException("host not found")).when(mockProcessor).notifyProcessFailure("host3");
    when(system.createEventProcessorGroup(any(), any(), any())).thenReturn(mockProcessor);
    @Cleanup ControllerEventProcessors processors = new ControllerEventProcessors("host1", config, localController, checkpointStore, streamStore, bucketStore, connectionPool, streamMetadataTasks, streamTransactionMetadataTasks, kvtStore, kvtTasks, system, executorService());
    // check for a case where init is not initialized so that kvtRequestProcessors don't get initialized and will be null
    assertTrue(Futures.await(processors.sweepFailedProcesses(() -> Sets.newHashSet("host1"))));
    Assert.assertFalse(processors.isReady());
    Assert.assertFalse(processors.isBootstrapCompleted());
    Assert.assertFalse(processors.isMetadataServiceConnected());
    processors.startAsync();
    processors.awaitRunning();
    assertTrue(Futures.await(processors.sweepFailedProcesses(() -> Sets.newHashSet("host1"))));
    assertTrue(Futures.await(processors.handleFailedProcess("host1")));
    AssertExtensions.assertFutureThrows("host not found", processors.handleFailedProcess("host3"), e -> e instanceof CheckpointStoreException);
    processors.shutDown();
}

Example 8 with TableMetadataTasks

use of io.pravega.controller.task.KeyValueTable.TableMetadataTasks in project pravega by pravega.

the class ControllerGrpcAuthFocusedTest method setup.

@Before
public void setup() throws IOException {
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createInMemoryStore(EXECUTOR);
    StreamMetadataStore streamStore = StreamStoreFactory.createInMemoryStore();
    this.kvtStore = spy(KVTableStoreFactory.createInMemoryStore(streamStore, EXECUTOR));
    BucketStore bucketStore = StreamStoreFactory.createInMemoryBucketStore();
    SegmentHelper segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    RequestTracker requestTracker = new RequestTracker(true);
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    GrpcAuthHelper authHelper = new GrpcAuthHelper(true, "secret", 300);
    EventHelper helper = EventHelperMock.getEventHelperMock(EXECUTOR, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, bucketStore, taskMetadataStore, segmentHelper, EXECUTOR, "host", authHelper, helper);
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, segmentHelper, EXECUTOR, "host", authHelper);
    kvtMetadataTasks = new TableMetadataTasks(kvtStore, segmentHelper, EXECUTOR, EXECUTOR, "host", authHelper, helper);
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, EXECUTOR), new ScaleOperationTask(streamMetadataTasks, streamStore, EXECUTOR), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, EXECUTOR), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, EXECUTOR), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, EXECUTOR), new TruncateStreamTask(streamMetadataTasks, streamStore, EXECUTOR), new CreateReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, EXECUTOR), EXECUTOR);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, EXECUTOR));
    streamTransactionMetadataTasks.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    Cluster mockCluster = mock(Cluster.class);
    when(mockCluster.getClusterMembers()).thenReturn(Collections.singleton(new Host("localhost", 9090, null)));
    ControllerServiceGrpc.ControllerServiceImplBase controllerServiceImplBase = new ControllerServiceImpl(new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelper, EXECUTOR, mockCluster, requestTracker), authHelper, requestTracker, true, true, 2);
    ControllerServiceGrpc.ControllerServiceImplBase controllerServiceImplBaseStrict = new ControllerServiceImpl(new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelper, EXECUTOR, mockCluster, requestTracker), authHelper, requestTracker, true, false, 2);
    PasswordAuthHandler authHandler = new PasswordAuthHandler();
    authHandler.initialize(AUTH_FILE.getAbsolutePath());
    String uniqueServerName = String.format("Test server name: %s", getClass());
    String uniqueServerNameStrict = String.format("Test server name: %sStrict", getClass());
    // Using a builder that creates a server for servicing in-process requests.
    // Also, using a direct executor which executes app code directly in transport thread. See
    // https://grpc.io/grpc-java/javadoc/io/grpc/inprocess/InProcessServerBuilder.html for more information.
    grpcServer = InProcessServerBuilder.forName(uniqueServerName).addService(ServerInterceptors.intercept(controllerServiceImplBase, new AuthInterceptor(authHandler))).directExecutor().build().start();
    grpcServerStrict = InProcessServerBuilder.forName(uniqueServerNameStrict).addService(ServerInterceptors.intercept(controllerServiceImplBaseStrict, new AuthInterceptor(authHandler))).directExecutor().build().start();
    inProcessChannel = InProcessChannelBuilder.forName(uniqueServerName).directExecutor().build();
    inProcessChannelStrict = InProcessChannelBuilder.forName(uniqueServerNameStrict).directExecutor().build();
}

Example 9 with TableMetadataTasks

use of io.pravega.controller.task.KeyValueTable.TableMetadataTasks in project pravega by pravega.

the class RequestHandlersTest method scopeDeleteTest.

@Test
public void scopeDeleteTest() {
    final String testScope = "testScope";
    final String testStream = "testStream";
    final String testRG = "_RGTestRG";
    final String testKVT = "testKVT";
    StreamMetadataStore streamStoreSpied = spy(getStore());
    KVTableMetadataStore kvtStoreSpied = spy(getKvtStore());
    StreamMetadataTasks streamMetadataTasks1 = mock(StreamMetadataTasks.class);
    TableMetadataTasks kvtTasksMocked = mock(TableMetadataTasks.class);
    streamStoreSpied.createScope(testScope, null, executor).join();
    OperationContext ctx = new OperationContext() {

        @Override
        public long getOperationStartTime() {
            return 0;
        }

        @Override
        public long getRequestId() {
            return 0;
        }
    };
    UUID scopeId = streamStoreSpied.getScopeId(testScope, ctx, executor).join();
    doAnswer(x -> {
        CompletableFuture<UUID> cf = new CompletableFuture<>();
        cf.complete(scopeId);
        return cf;
    }).when(streamStoreSpied).getScopeId(eq(testScope), eq(ctx), eq(executor));
    doAnswer(invocation -> {
        CompletableFuture<Boolean> cf = new CompletableFuture<>();
        cf.complete(true);
        return cf;
    }).when(streamStoreSpied).isScopeSealed(eq(testScope), any(), any());
    createStreamInStore(testStream, testScope);
    createStreamInStore(testRG, testScope);
    assertTrue(streamStore.checkStreamExists(testScope, testStream, ctx, executor).join());
    doAnswer(invocation -> {
        CompletableFuture<Controller.UpdateStreamStatus.Status> future = new CompletableFuture<>();
        future.complete(Controller.UpdateStreamStatus.Status.SUCCESS);
        return future;
    }).when(streamMetadataTasks1).sealStream(anyString(), anyString(), anyLong());
    doAnswer(invocation -> {
        CompletableFuture<Controller.DeleteStreamStatus.Status> future = new CompletableFuture<>();
        future.complete(Controller.DeleteStreamStatus.Status.SUCCESS);
        return future;
    }).when(streamMetadataTasks1).deleteStream(anyString(), anyString(), anyLong());
    // Create Reader Group
    ReaderGroupConfig rgConfig = ReaderGroupConfig.builder().stream(NameUtils.getScopedStreamName(testScope, testStream)).build();
    final ReaderGroupConfig config = ReaderGroupConfig.cloneConfig(rgConfig, UUID.randomUUID(), 123L);
    Controller.ReaderGroupConfiguration expectedConfig = ModelHelper.decode(testScope, testRG, config);
    doAnswer(invocationOnMock -> {
        CompletableFuture<Controller.CreateReaderGroupResponse.Status> createRG = new CompletableFuture<>();
        createRG.complete(Controller.CreateReaderGroupResponse.Status.SUCCESS);
        return createRG;
    }).when(streamMetadataTasks1).createReaderGroup(anyString(), any(), any(), anyLong(), anyLong());
    doAnswer(invocation -> CompletableFuture.completedFuture(Controller.ReaderGroupConfigResponse.newBuilder().setStatus(Controller.ReaderGroupConfigResponse.Status.SUCCESS).setConfig(expectedConfig).build())).when(streamMetadataTasks1).getReaderGroupConfig(eq(testScope), anyString(), anyLong());
    doAnswer(invocationOnMock -> {
        CompletableFuture<Controller.DeleteReaderGroupStatus.Status> future = new CompletableFuture<>();
        future.complete(Controller.DeleteReaderGroupStatus.Status.SUCCESS);
        return future;
    }).when(streamMetadataTasks1).deleteReaderGroup(anyString(), anyString(), anyString(), anyLong());
    // Create KVT
    KeyValueTableConfiguration kvtConfig = KeyValueTableConfiguration.builder().partitionCount(1).primaryKeyLength(1).secondaryKeyLength(1).build();
    doAnswer(invocationOnMock -> {
        CompletableFuture<Controller.CreateKeyValueTableStatus.Status> fut = new CompletableFuture<>();
        fut.complete(Controller.CreateKeyValueTableStatus.Status.SUCCESS);
        return fut;
    }).when(kvtTasksMocked).createKeyValueTable(anyString(), anyString(), any(), anyLong(), anyLong());
    List<String> tableList = new ArrayList<>();
    tableList.add(testKVT);
    Pair<List<String>, String> listOfKVTables = new ImmutablePair<>(tableList, "");
    doAnswer(invocationOnMock -> CompletableFuture.completedFuture(listOfKVTables)).doAnswer(invocationOnMock -> CompletableFuture.completedFuture(new ImmutablePair<>(Collections.emptyList(), invocationOnMock.getArgument(0)))).when(kvtStoreSpied).listKeyValueTables(anyString(), any(), anyInt(), any(), any());
    doAnswer(invocationOnMock -> {
        CompletableFuture<Controller.DeleteKVTableStatus.Status> future = new CompletableFuture<>();
        future.complete(Controller.DeleteKVTableStatus.Status.SUCCESS);
        return future;
    }).when(kvtTasksMocked).deleteKeyValueTable(anyString(), anyString(), anyLong());
    Controller.CreateKeyValueTableStatus.Status status = kvtTasksMocked.createKeyValueTable(testScope, testKVT, kvtConfig, System.currentTimeMillis(), 123L).join();
    assertEquals(status, Controller.CreateKeyValueTableStatus.Status.SUCCESS);
    DeleteScopeTask requestHandler = new DeleteScopeTask(streamMetadataTasks1, streamStoreSpied, kvtStoreSpied, kvtTasksMocked, executor);
    DeleteScopeEvent event = new DeleteScopeEvent(testScope, 123L, scopeId);
    CompletableFuture<Void> future = requestHandler.execute(event);
    future.join();
}

Example 10 with TableMetadataTasks

use of io.pravega.controller.task.KeyValueTable.TableMetadataTasks in project pravega by pravega.

the class EventProcessorHealthContributorTest method setup.

@SneakyThrows
@Before
public void setup() {
    Host host = mock(Host.class);
    LocalController localController = mock(LocalController.class);
    CheckpointStore checkpointStore = mock(CheckpointStore.class);
    StreamMetadataStore streamStore = mock(StreamMetadataStore.class);
    BucketStore bucketStore = mock(BucketStore.class);
    ConnectionPool connectionPool = mock(ConnectionPool.class);
    StreamMetadataTasks streamMetadataTasks = mock(StreamMetadataTasks.class);
    StreamTransactionMetadataTasks streamTransactionMetadataTasks = mock(StreamTransactionMetadataTasks.class);
    ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
    KVTableMetadataStore kvtMetadataStore = mock(KVTableMetadataStore.class);
    TableMetadataTasks kvtMetadataTasks = mock(TableMetadataTasks.class);
    EventProcessorSystem system = mock(EventProcessorSystemImpl.class);
    ControllerEventProcessorConfig config = ControllerEventProcessorConfigImpl.withDefault();
    EventProcessorGroup<ControllerEvent> processor = getProcessor();
    EventProcessorGroup<ControllerEvent> mockProcessor = spy(processor);
    doThrow(new CheckpointStoreException("host not found")).when(mockProcessor).notifyProcessFailure("host3");
    when(system.createEventProcessorGroup(any(), any(), any())).thenReturn(mockProcessor);
    eventProcessors = spy(new ControllerEventProcessors(host.getHostId(), config, localController, checkpointStore, streamStore, bucketStore, connectionPool, streamMetadataTasks, streamTransactionMetadataTasks, kvtMetadataStore, kvtMetadataTasks, system, executorService()));
    doReturn(true).when(eventProcessors).isReady();
    contributor = new EventProcessorHealthContributor("eventprocessors", eventProcessors);
    builder = Health.builder().name("eventprocessors");
}