Examples with MetadataCacheImpl org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl used on opensource projects

Example 1 with MetadataCacheImpl

use of org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl in project pulsar by apache.

the class NamespacesTest method testGetNamespaces.

@Test
public void testGetNamespaces() throws Exception {
    List<String> expectedList = Lists.newArrayList(this.testLocalNamespaces.get(0).toString(), this.testLocalNamespaces.get(1).toString());
    expectedList.sort(null);
    assertEquals(namespaces.getNamespacesForCluster(this.testTenant, this.testLocalCluster), expectedList);
    expectedList = Lists.newArrayList(this.testLocalNamespaces.get(0).toString(), this.testLocalNamespaces.get(1).toString(), this.testLocalNamespaces.get(2).toString(), this.testGlobalNamespaces.get(0).toString());
    expectedList.sort(null);
    assertEquals(namespaces.getTenantNamespaces(this.testTenant), expectedList);
    try {
        // check the tenant name is valid
        namespaces.getTenantNamespaces(this.testTenant + "/default");
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    try {
        namespaces.getTenantNamespaces("non-existing-tenant");
        fail("should have failed");
    } catch (RestException e) {
    // Ok, does not exist
    }
    try {
        namespaces.getNamespacesForCluster(this.testTenant, "other-cluster");
        fail("should have failed");
    } catch (RestException e) {
    // Ok, does not exist
    }
    // ZK Errors
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET_CHILDREN && path.equals("/admin/policies/my-tenant");
    });
    // clear caches to load data from metadata-store again
    MetadataCacheImpl<TenantInfo> tenantCache = (MetadataCacheImpl<TenantInfo>) pulsar.getPulsarResources().getTenantResources().getCache();
    AbstractMetadataStore store = (AbstractMetadataStore) tenantCache.getStore();
    tenantCache.invalidateAll();
    store.invalidateAll();
    try {
        namespaces.getTenantNamespaces(this.testTenant);
        fail("should have failed");
    } catch (RestException e) {
    // Ok
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET_CHILDREN && path.equals("/admin/policies/my-tenant/use");
    });
    try {
        namespaces.getNamespacesForCluster(this.testTenant, this.testLocalCluster);
        fail("should have failed");
    } catch (RestException e) {
    // Ok
    }
}

Example 2 with MetadataCacheImpl

use of org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl in project pulsar by apache.

the class AdminTest method clusters.

@Test
public void clusters() throws Exception {
    assertEquals(clusters.getClusters(), Lists.newArrayList());
    verify(clusters, never()).validateSuperUserAccess();
    clusters.createCluster("use", ClusterDataImpl.builder().serviceUrl("http://broker.messaging.use.example.com:8080").build());
    verify(clusters, times(1)).validateSuperUserAccess();
    // ensure to read from ZooKeeper directly
    // clusters.clustersListCache().clear();
    assertEquals(clusters.getClusters(), Lists.newArrayList("use"));
    // Check creating existing cluster
    try {
        clusters.createCluster("use", ClusterDataImpl.builder().serviceUrl("http://broker.messaging.use.example.com:8080").build());
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode());
    }
    // Check deleting non-existing cluster
    try {
        clusters.deleteCluster("usc");
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
    }
    assertEquals(clusters.getCluster("use"), ClusterDataImpl.builder().serviceUrl("http://broker.messaging.use.example.com:8080").build());
    verify(clusters, times(4)).validateSuperUserAccess();
    clusters.updateCluster("use", ClusterDataImpl.builder().serviceUrl("http://new-broker.messaging.use.example.com:8080").build());
    verify(clusters, times(5)).validateSuperUserAccess();
    assertEquals(clusters.getCluster("use"), ClusterData.builder().serviceUrl("http://new-broker.messaging.use.example.com:8080").build());
    verify(clusters, times(6)).validateSuperUserAccess();
    try {
        clusters.getNamespaceIsolationPolicies("use");
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), 404);
    }
    Map<String, String> parameters1 = new HashMap<>();
    parameters1.put("min_limit", "1");
    parameters1.put("usage_threshold", "90");
    NamespaceIsolationDataImpl policyData = NamespaceIsolationDataImpl.builder().namespaces(Collections.singletonList("dummy/colo/ns")).primary(Collections.singletonList("localhost" + ":" + pulsar.getListenPortHTTP())).autoFailoverPolicy(AutoFailoverPolicyData.builder().policyType(AutoFailoverPolicyType.min_available).parameters(parameters1).build()).build();
    AsyncResponse response = mock(AsyncResponse.class);
    clusters.setNamespaceIsolationPolicy(response, "use", "policy1", policyData);
    clusters.getNamespaceIsolationPolicies("use");
    try {
        clusters.deleteCluster("use");
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), 412);
    }
    clusters.deleteNamespaceIsolationPolicy("use", "policy1");
    assertTrue(clusters.getNamespaceIsolationPolicies("use").isEmpty());
    clusters.deleteCluster("use");
    verify(clusters, times(13)).validateSuperUserAccess();
    assertEquals(clusters.getClusters(), Lists.newArrayList());
    try {
        clusters.getCluster("use");
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), 404);
    }
    try {
        clusters.updateCluster("use", ClusterDataImpl.builder().build());
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), 404);
    }
    try {
        clusters.getNamespaceIsolationPolicies("use");
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), 404);
    }
    // Test zk failures
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET_CHILDREN && path.equals("/admin/clusters");
    });
    // clear caches to load data from metadata-store again
    MetadataCacheImpl<ClusterData> clusterCache = (MetadataCacheImpl<ClusterData>) pulsar.getPulsarResources().getClusterResources().getCache();
    MetadataCacheImpl isolationPolicyCache = (MetadataCacheImpl) pulsar.getPulsarResources().getNamespaceResources().getIsolationPolicies().getCache();
    AbstractMetadataStore store = (AbstractMetadataStore) clusterCache.getStore();
    clusterCache.invalidateAll();
    store.invalidateAll();
    try {
        clusters.getClusters();
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.CREATE && path.equals("/admin/clusters/test");
    });
    try {
        clusters.createCluster("test", ClusterDataImpl.builder().serviceUrl("http://broker.messaging.test.example.com:8080").build());
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET && path.equals("/admin/clusters/test");
    });
    clusterCache.invalidateAll();
    store.invalidateAll();
    try {
        clusters.updateCluster("test", ClusterDataImpl.builder().serviceUrl("http://broker.messaging.test.example.com").build());
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET && path.equals("/admin/clusters/test");
    });
    try {
        clusters.getCluster("test");
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET_CHILDREN && path.equals("/admin/policies");
    });
    try {
        clusters.deleteCluster("use");
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET && path.equals("/admin/clusters/use/namespaceIsolationPolicies");
    });
    clusterCache.invalidateAll();
    isolationPolicyCache.invalidateAll();
    store.invalidateAll();
    try {
        clusters.deleteCluster("use");
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    // Check name validations
    try {
        clusters.createCluster("bf@", ClusterDataImpl.builder().serviceUrl("http://dummy.messaging.example.com").build());
        fail("should have filed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    // Check authentication and listener name
    try {
        clusters.createCluster("auth", ClusterDataImpl.builder().serviceUrl("http://dummy.web.example.com").serviceUrlTls("").brokerServiceUrl("http://dummy.messaging.example.com").brokerServiceUrlTls("").authenticationPlugin("authenticationPlugin").authenticationParameters("authenticationParameters").listenerName("listenerName").build());
        ClusterData cluster = clusters.getCluster("auth");
        assertEquals(cluster.getAuthenticationPlugin(), "authenticationPlugin");
        assertEquals(cluster.getAuthenticationParameters(), "authenticationParameters");
        assertEquals(cluster.getListenerName(), "listenerName");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
}

Example 3 with MetadataCacheImpl

use of org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl in project pulsar by apache.

the class MetadataCacheTest method crossStoreUpdates.

@Test(dataProvider = "zk")
public void crossStoreUpdates(String provider, Supplier<String> urlSupplier) throws Exception {
    String testName = "cross store updates";
    @Cleanup MetadataStore store1 = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().build());
    @Cleanup MetadataStore store2 = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().build());
    MetadataCacheImpl<MyClass> objCache1 = (MetadataCacheImpl<MyClass>) store1.getMetadataCache(MyClass.class);
    MetadataCacheImpl<MyClass> objCache2 = (MetadataCacheImpl<MyClass>) store2.getMetadataCache(MyClass.class);
    AtomicReference<MyClass> storeObj = new AtomicReference<MyClass>();
    store2.registerListener(n -> {
        if (n.getType() == NotificationType.Modified) {
            CompletableFuture.runAsync(() -> {
                try {
                    MyClass obj = objCache2.get(n.getPath()).get().get();
                    storeObj.set(obj);
                } catch (Exception e) {
                    log.error("Got exception {}", e.getMessage());
                }
            });
        }
    });
    String key1 = "/test-key1";
    assertEquals(objCache1.getIfCached(key1), Optional.empty());
    assertEquals(objCache2.getIfCached(key1), Optional.empty());
    MyClass value1 = new MyClass(testName, 1);
    objCache1.create(key1, value1).join();
    Awaitility.await().ignoreNoExceptions().untilAsserted(() -> {
        assertEquals(objCache1.getIfCached(key1), Optional.of(value1));
        assertEquals(objCache2.get(key1).join(), Optional.of(value1));
        assertEquals(objCache2.getIfCached(key1), Optional.of(value1));
    });
    MyClass value2 = new MyClass(testName, 2);
    objCache1.readModifyUpdate(key1, (oldData) -> value2).join();
    Awaitility.await().ignoreNoExceptions().untilAsserted(() -> assertEquals(storeObj.get(), value2));
}

Example 4 with MetadataCacheImpl

use of org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl in project pulsar by yahoo.

the class AdminTest method clusters.

@Test
@SuppressWarnings("unchecked")
public void clusters() throws Exception {
    assertEquals(asyncRequests(ctx -> clusters.getClusters(ctx)), Sets.newHashSet());
    verify(clusters, never()).validateSuperUserAccessAsync();
    asyncRequests(ctx -> clusters.createCluster(ctx, "use", ClusterDataImpl.builder().serviceUrl("http://broker.messaging.use.example.com:8080").build()));
    // ensure to read from ZooKeeper directly
    // clusters.clustersListCache().clear();
    assertEquals(asyncRequests(ctx -> clusters.getClusters(ctx)), Sets.newHashSet("use"));
    // Check creating existing cluster
    try {
        asyncRequests(ctx -> clusters.createCluster(ctx, "use", ClusterDataImpl.builder().serviceUrl("http://broker.messaging.use.example.com:8080").build()));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode());
    }
    // Check deleting non-existing cluster
    try {
        asyncRequests(ctx -> clusters.deleteCluster(ctx, "usc"));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
    }
    assertEquals(asyncRequests(ctx -> clusters.getCluster(ctx, "use")), ClusterDataImpl.builder().serviceUrl("http://broker.messaging.use.example.com:8080").build());
    asyncRequests(ctx -> clusters.updateCluster(ctx, "use", ClusterDataImpl.builder().serviceUrl("http://new-broker.messaging.use.example.com:8080").build()));
    assertEquals(asyncRequests(ctx -> clusters.getCluster(ctx, "use")), ClusterData.builder().serviceUrl("http://new-broker.messaging.use.example.com:8080").build());
    try {
        asyncRequests(ctx -> clusters.getNamespaceIsolationPolicies(ctx, "use"));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), 404);
    }
    Map<String, String> parameters1 = new HashMap<>();
    parameters1.put("min_limit", "1");
    parameters1.put("usage_threshold", "90");
    NamespaceIsolationDataImpl policyData = NamespaceIsolationDataImpl.builder().namespaces(Collections.singletonList("dummy/colo/ns")).primary(Collections.singletonList("localhost" + ":" + pulsar.getListenPortHTTP())).autoFailoverPolicy(AutoFailoverPolicyData.builder().policyType(AutoFailoverPolicyType.min_available).parameters(parameters1).build()).build();
    asyncRequests(ctx -> clusters.setNamespaceIsolationPolicy(ctx, "use", "policy1", policyData));
    asyncRequests(ctx -> clusters.getNamespaceIsolationPolicies(ctx, "use"));
    try {
        asyncRequests(ctx -> clusters.deleteCluster(ctx, "use"));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), 412);
    }
    asyncRequests(ctx -> clusters.deleteNamespaceIsolationPolicy(ctx, "use", "policy1"));
    assertTrue(((Map<String, NamespaceIsolationDataImpl>) asyncRequests(ctx -> clusters.getNamespaceIsolationPolicies(ctx, "use"))).isEmpty());
    asyncRequests(ctx -> clusters.deleteCluster(ctx, "use"));
    assertEquals(asyncRequests(ctx -> clusters.getClusters(ctx)), Sets.newHashSet());
    try {
        asyncRequests(ctx -> clusters.getCluster(ctx, "use"));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), 404);
    }
    try {
        asyncRequests(ctx -> clusters.updateCluster(ctx, "use", ClusterDataImpl.builder().build()));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), 404);
    }
    try {
        asyncRequests(ctx -> clusters.getNamespaceIsolationPolicies(ctx, "use"));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), 404);
    }
    // Test zk failures
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET_CHILDREN && path.equals("/admin/clusters");
    });
    // clear caches to load data from metadata-store again
    MetadataCacheImpl<ClusterData> clusterCache = (MetadataCacheImpl<ClusterData>) pulsar.getPulsarResources().getClusterResources().getCache();
    MetadataCacheImpl isolationPolicyCache = (MetadataCacheImpl) pulsar.getPulsarResources().getNamespaceResources().getIsolationPolicies().getCache();
    AbstractMetadataStore store = (AbstractMetadataStore) clusterCache.getStore();
    clusterCache.invalidateAll();
    store.invalidateAll();
    try {
        asyncRequests(ctx -> clusters.getClusters(ctx));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.CREATE && path.equals("/admin/clusters/test");
    });
    try {
        asyncRequests(ctx -> clusters.createCluster(ctx, "test", ClusterDataImpl.builder().serviceUrl("http://broker.messaging.test.example.com:8080").build()));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET && path.equals("/admin/clusters/test");
    });
    clusterCache.invalidateAll();
    store.invalidateAll();
    try {
        asyncRequests(ctx -> clusters.updateCluster(ctx, "test", ClusterDataImpl.builder().serviceUrl("http://broker.messaging.test.example.com").build()));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET && path.equals("/admin/clusters/test");
    });
    try {
        asyncRequests(ctx -> clusters.getCluster(ctx, "test"));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET_CHILDREN && path.equals("/admin/policies");
    });
    try {
        asyncRequests(ctx -> clusters.deleteCluster(ctx, "use"));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET && path.equals("/admin/clusters/use/namespaceIsolationPolicies");
    });
    clusterCache.invalidateAll();
    isolationPolicyCache.invalidateAll();
    store.invalidateAll();
    try {
        asyncRequests(ctx -> clusters.deleteCluster(ctx, "use"));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    // Check name validations
    try {
        asyncRequests(ctx -> clusters.createCluster(ctx, "bf@", ClusterDataImpl.builder().serviceUrl("http://dummy.messaging.example.com").build()));
        fail("should have filed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    // Check authentication and listener name
    try {
        asyncRequests(ctx -> clusters.createCluster(ctx, "auth", ClusterDataImpl.builder().serviceUrl("http://dummy.web.example.com").serviceUrlTls("").brokerServiceUrl("http://dummy.messaging.example.com").brokerServiceUrlTls("").authenticationPlugin("authenticationPlugin").authenticationParameters("authenticationParameters").listenerName("listenerName").build()));
        ClusterData cluster = (ClusterData) asyncRequests(ctx -> clusters.getCluster(ctx, "auth"));
        assertEquals(cluster.getAuthenticationPlugin(), "authenticationPlugin");
        assertEquals(cluster.getAuthenticationParameters(), "authenticationParameters");
        assertEquals(cluster.getListenerName(), "listenerName");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    verify(clusters, times(24)).validateSuperUserAccessAsync();
}

Example 5 with MetadataCacheImpl

use of org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl in project pulsar by yahoo.

the class AdminTest method properties.

@Test
public void properties() throws Throwable {
    Object response = asyncRequests(ctx -> properties.getTenants(ctx));
    assertEquals(response, Lists.newArrayList());
    verify(properties, times(1)).validateSuperUserAccessAsync();
    // create local cluster
    asyncRequests(ctx -> clusters.createCluster(ctx, configClusterName, ClusterDataImpl.builder().build()));
    Set<String> allowedClusters = Sets.newHashSet();
    allowedClusters.add(configClusterName);
    TenantInfoImpl tenantInfo = TenantInfoImpl.builder().adminRoles(Sets.newHashSet("role1", "role2")).allowedClusters(allowedClusters).build();
    response = asyncRequests(ctx -> properties.createTenant(ctx, "test-property", tenantInfo));
    verify(properties, times(2)).validateSuperUserAccessAsync();
    response = asyncRequests(ctx -> properties.getTenants(ctx));
    assertEquals(response, Lists.newArrayList("test-property"));
    verify(properties, times(3)).validateSuperUserAccessAsync();
    response = asyncRequests(ctx -> properties.getTenantAdmin(ctx, "test-property"));
    assertEquals(response, tenantInfo);
    verify(properties, times(4)).validateSuperUserAccessAsync();
    final TenantInfoImpl newPropertyAdmin = TenantInfoImpl.builder().adminRoles(Sets.newHashSet("role1", "other-role")).allowedClusters(allowedClusters).build();
    response = asyncRequests(ctx -> properties.updateTenant(ctx, "test-property", newPropertyAdmin));
    verify(properties, times(5)).validateSuperUserAccessAsync();
    // Wait for updateTenant to take effect
    Thread.sleep(100);
    response = asyncRequests(ctx -> properties.getTenantAdmin(ctx, "test-property"));
    assertEquals(response, newPropertyAdmin);
    response = asyncRequests(ctx -> properties.getTenantAdmin(ctx, "test-property"));
    assertNotSame(response, tenantInfo);
    verify(properties, times(7)).validateSuperUserAccessAsync();
    // Check creating existing property
    try {
        response = asyncRequests(ctx -> properties.createTenant(ctx, "test-property", tenantInfo));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode());
    }
    // Check non-existing property
    try {
        response = asyncRequests(ctx -> properties.getTenantAdmin(ctx, "non-existing"));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
    }
    try {
        response = asyncRequests(ctx -> properties.updateTenant(ctx, "xxx-non-existing", newPropertyAdmin));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
    }
    // Check deleting non-existing property
    try {
        response = asyncRequests(ctx -> properties.deleteTenant(ctx, "non-existing", false));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
    }
    // clear caches to load data from metadata-store again
    MetadataCacheImpl<TenantInfo> cache = (MetadataCacheImpl<TenantInfo>) pulsar.getPulsarResources().getTenantResources().getCache();
    AbstractMetadataStore store = (AbstractMetadataStore) cache.getStore();
    cache.invalidateAll();
    store.invalidateAll();
    // Test zk failures
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET_CHILDREN && path.equals("/admin/policies");
    });
    try {
        response = asyncRequests(ctx -> properties.getTenants(ctx));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET && path.equals("/admin/policies/my-tenant");
    });
    try {
        response = asyncRequests(ctx -> properties.getTenantAdmin(ctx, "my-tenant"));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET && path.equals("/admin/policies/my-tenant");
    });
    try {
        response = asyncRequests(ctx -> properties.updateTenant(ctx, "my-tenant", newPropertyAdmin));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.CREATE && path.equals("/admin/policies/test");
    });
    try {
        response = asyncRequests(ctx -> properties.createTenant(ctx, "test", tenantInfo));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.GET_CHILDREN && path.equals("/admin/policies/test-property");
    });
    try {
        cache.invalidateAll();
        store.invalidateAll();
        response = asyncRequests(ctx -> properties.deleteTenant(ctx, "test-property", false));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    response = asyncRequests(ctx -> properties.createTenant(ctx, "error-property", tenantInfo));
    mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> {
        return op == MockZooKeeper.Op.DELETE && path.equals("/admin/policies/error-property");
    });
    try {
        response = asyncRequests(ctx -> properties.deleteTenant(ctx, "error-property", false));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
    }
    response = asyncRequests(ctx -> properties.deleteTenant(ctx, "test-property", false));
    response = asyncRequests(ctx -> properties.deleteTenant(ctx, "error-property", false));
    response = Lists.newArrayList();
    response = asyncRequests(ctx -> properties.getTenants(ctx));
    assertEquals(response, Lists.newArrayList());
    // Create a namespace to test deleting a non-empty property
    TenantInfoImpl newPropertyAdmin2 = TenantInfoImpl.builder().adminRoles(Sets.newHashSet("role1", "other-role")).allowedClusters(Sets.newHashSet("use")).build();
    response = asyncRequests(ctx -> properties.createTenant(ctx, "my-tenant", newPropertyAdmin2));
    response = asyncRequests(ctx -> namespaces.createNamespace(ctx, "my-tenant", "use", "my-namespace", BundlesData.builder().build()));
    try {
        response = asyncRequests(ctx -> properties.deleteTenant(ctx, "my-tenant", false));
        fail("should have failed");
    } catch (RestException e) {
    // Ok
    }
    // Check name validation
    try {
        response = asyncRequests(ctx -> properties.createTenant(ctx, "test&", tenantInfo));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    // Check tenantInfo is null
    try {
        response = asyncRequests(ctx -> properties.createTenant(ctx, "tenant-config-is-null", null));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    // Check tenantInfo with empty cluster
    String blankCluster = "";
    Set<String> blankClusters = Sets.newHashSet(blankCluster);
    TenantInfoImpl tenantWithEmptyCluster = TenantInfoImpl.builder().adminRoles(Sets.newHashSet("role1", "role2")).allowedClusters(blankClusters).build();
    try {
        response = asyncRequests(ctx -> properties.createTenant(ctx, "tenant-config-is-empty", tenantWithEmptyCluster));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    // Check tenantInfo contains empty cluster
    Set<String> containBlankClusters = Sets.newHashSet(blankCluster);
    containBlankClusters.add(configClusterName);
    TenantInfoImpl tenantContainEmptyCluster = TenantInfoImpl.builder().adminRoles(Sets.newHashSet()).allowedClusters(containBlankClusters).build();
    try {
        response = asyncRequests(ctx -> properties.createTenant(ctx, "tenant-config-contain-empty", tenantContainEmptyCluster));
        fail("should have failed");
    } catch (RestException e) {
        assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    AsyncResponse response2 = mock(AsyncResponse.class);
    namespaces.deleteNamespace(response2, "my-tenant", "use", "my-namespace", false, false);
    ArgumentCaptor<Response> captor = ArgumentCaptor.forClass(Response.class);
    verify(response2, timeout(5000).times(1)).resume(captor.capture());
    assertEquals(captor.getValue().getStatus(), Status.NO_CONTENT.getStatusCode());
    response = asyncRequests(ctx -> properties.deleteTenant(ctx, "my-tenant", false));
}