use of com.yahoo.pulsar.common.policies.data.PersistencePolicies in project pulsar by yahoo.
the class PulsarAdminToolTest method namespaces.
@Test
void namespaces() throws Exception {
PulsarAdmin admin = Mockito.mock(PulsarAdmin.class);
Namespaces mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
Lookup mockLookup = mock(Lookup.class);
when(admin.lookups()).thenReturn(mockLookup);
CmdNamespaces namespaces = new CmdNamespaces(admin);
namespaces.run(split("list myprop"));
verify(mockNamespaces).getNamespaces("myprop");
namespaces.run(split("list-cluster myprop/clust"));
verify(mockNamespaces).getNamespaces("myprop", "clust");
namespaces.run(split("destinations myprop/clust/ns1"));
verify(mockNamespaces).getDestinations("myprop/clust/ns1");
namespaces.run(split("policies myprop/clust/ns1"));
verify(mockNamespaces).getPolicies("myprop/clust/ns1");
namespaces.run(split("create myprop/clust/ns1"));
verify(mockNamespaces).createNamespace("myprop/clust/ns1");
namespaces.run(split("delete myprop/clust/ns1"));
verify(mockNamespaces).deleteNamespace("myprop/clust/ns1");
namespaces.run(split("permissions myprop/clust/ns1"));
verify(mockNamespaces).getPermissions("myprop/clust/ns1");
namespaces.run(split("grant-permission myprop/clust/ns1 --role role1 --actions produce,consume"));
verify(mockNamespaces).grantPermissionOnNamespace("myprop/clust/ns1", "role1", EnumSet.of(AuthAction.produce, AuthAction.consume));
namespaces.run(split("revoke-permission myprop/clust/ns1 --role role1"));
verify(mockNamespaces).revokePermissionsOnNamespace("myprop/clust/ns1", "role1");
namespaces.run(split("set-clusters myprop/clust/ns1 -c use,usw,usc"));
verify(mockNamespaces).setNamespaceReplicationClusters("myprop/clust/ns1", Lists.newArrayList("use", "usw", "usc"));
namespaces.run(split("get-clusters myprop/clust/ns1"));
verify(mockNamespaces).getNamespaceReplicationClusters("myprop/clust/ns1");
namespaces.run(split("unload myprop/clust/ns1"));
verify(mockNamespaces).unload("myprop/clust/ns1");
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff"));
verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff");
namespaces.run(split("split-bundle myprop/clust/ns1 -b 0x00000000_0xffffffff"));
verify(mockNamespaces).splitNamespaceBundle("myprop/clust/ns1", "0x00000000_0xffffffff");
namespaces.run(split("get-backlog-quotas myprop/clust/ns1"));
verify(mockNamespaces).getBacklogQuotaMap("myprop/clust/ns1");
namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_request_hold -l 10"));
verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10, RetentionPolicy.producer_request_hold));
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10K"));
verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024, RetentionPolicy.producer_exception));
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10M"));
verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024 * 1024, RetentionPolicy.producer_exception));
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10G"));
verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10l * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
namespaces.run(split("set-persistence myprop/clust/ns1 -e 2 -w 1 -a 1 -r 100.0"));
verify(mockNamespaces).setPersistence("myprop/clust/ns1", new PersistencePolicies(2, 1, 1, 100.0d));
namespaces.run(split("get-persistence myprop/clust/ns1"));
verify(mockNamespaces).getPersistence("myprop/clust/ns1");
namespaces.run(split("set-message-ttl myprop/clust/ns1 -ttl 300"));
verify(mockNamespaces).setNamespaceMessageTTL("myprop/clust/ns1", 300);
namespaces.run(split("get-message-ttl myprop/clust/ns1"));
verify(mockNamespaces).getNamespaceMessageTTL("myprop/clust/ns1");
namespaces.run(split("set-retention myprop/clust/ns1 -t 1h -s 1M"));
verify(mockNamespaces).setRetention("myprop/clust/ns1", new RetentionPolicies(60, 1));
namespaces.run(split("get-retention myprop/clust/ns1"));
verify(mockNamespaces).getRetention("myprop/clust/ns1");
namespaces.run(split("clear-backlog myprop/clust/ns1 -force"));
verify(mockNamespaces).clearNamespaceBacklog("myprop/clust/ns1");
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff myprop/clust/ns1 -force"));
verify(mockNamespaces).clearNamespaceBundleBacklog("myprop/clust/ns1", "0x80000000_0xffffffff");
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("clear-backlog -s my-sub myprop/clust/ns1 -force"));
verify(mockNamespaces).clearNamespaceBacklogForSubscription("myprop/clust/ns1", "my-sub");
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1 -force"));
verify(mockNamespaces).clearNamespaceBundleBacklogForSubscription("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
namespaces.run(split("unsubscribe -s my-sub myprop/clust/ns1"));
verify(mockNamespaces).unsubscribeNamespace("myprop/clust/ns1", "my-sub");
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("unsubscribe -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1"));
verify(mockNamespaces).unsubscribeNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
}
use of com.yahoo.pulsar.common.policies.data.PersistencePolicies in project pulsar by yahoo.
the class BrokerService method getManagedLedgerConfig.
public CompletableFuture<ManagedLedgerConfig> getManagedLedgerConfig(DestinationName topicName) {
CompletableFuture<ManagedLedgerConfig> future = new CompletableFuture<>();
// Execute in background thread, since getting the policies might block if the z-node wasn't already cached
pulsar.getOrderedExecutor().submitOrdered(topicName, safeRun(() -> {
NamespaceName namespace = topicName.getNamespaceObject();
ServiceConfiguration serviceConfig = pulsar.getConfiguration();
// Get persistence policy for this destination
Policies policies;
try {
policies = pulsar.getConfigurationCache().policiesCache().get(AdminResource.path("policies", namespace.getProperty(), namespace.getCluster(), namespace.getLocalName())).orElse(null);
} catch (Throwable t) {
// Ignoring since if we don't have policies, we fallback on the default
log.warn("Got exception when reading persistence policy for {}: {}", topicName, t.getMessage(), t);
future.completeExceptionally(t);
return;
}
PersistencePolicies persistencePolicies = policies != null ? policies.persistence : null;
RetentionPolicies retentionPolicies = policies != null ? policies.retention_policies : null;
if (persistencePolicies == null) {
// Apply default values
persistencePolicies = new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), serviceConfig.getManagedLedgerDefaultWriteQuorum(), serviceConfig.getManagedLedgerDefaultAckQuorum(), serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit());
}
if (retentionPolicies == null) {
retentionPolicies = new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), serviceConfig.getDefaultRetentionSizeInMB());
}
ManagedLedgerConfig config = new ManagedLedgerConfig();
config.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble());
config.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum());
config.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum());
config.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate());
config.setDigestType(DigestType.CRC32);
config.setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist());
config.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger());
config.setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
config.setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
config.setMaxSizePerLedgerMb(2048);
config.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize());
config.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum());
config.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum());
config.setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger());
config.setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds());
config.setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES);
config.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB());
future.complete(config);
}, (exception) -> future.completeExceptionally(exception)));
return future;
}
use of com.yahoo.pulsar.common.policies.data.PersistencePolicies in project pulsar by yahoo.
the class Namespaces method setPersistence.
@POST
@Path("/{property}/{cluster}/{namespace}/persistence")
@ApiOperation(value = "Set the persistence configuration for all the destinations on a namespace.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") })
public void setPersistence(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, PersistencePolicies persistence) {
validatePoliciesReadOnlyAccess();
try {
Stat nodeStat = new Stat();
final String path = path("policies", property, cluster, namespace);
byte[] content = globalZk().getData(path, null, nodeStat);
Policies policies = jsonMapper().readValue(content, Policies.class);
policies.persistence = persistence;
globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion());
policiesCache().invalidate(path("policies", property, cluster, namespace));
log.info("[{}] Successfully updated persistence configuration: namespace={}/{}/{}, map={}", clientAppId(), property, cluster, namespace, jsonMapper().writeValueAsString(policies.persistence));
} catch (KeeperException.NoNodeException e) {
log.warn("[{}] Failed to update persistence configuration for namespace {}/{}/{}: does not exist", clientAppId(), property, cluster, namespace);
throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
} catch (KeeperException.BadVersionException e) {
log.warn("[{}] Failed to update persistence configuration for namespace {}/{}/{}: concurrent modification", clientAppId(), property, cluster, namespace);
throw new RestException(Status.CONFLICT, "Concurrent modification");
} catch (Exception e) {
log.error("[{}] Failed to update persistence configuration for namespace {}/{}/{}", clientAppId(), property, cluster, namespace, e);
throw new RestException(e);
}
}
use of com.yahoo.pulsar.common.policies.data.PersistencePolicies in project pulsar by yahoo.
the class AdminApiTest method namespaces.
@Test(invocationCount = 1)
public void namespaces() throws PulsarAdminException, PulsarServerException, Exception {
admin.clusters().createCluster("usw", new ClusterData());
PropertyAdmin propertyAdmin = new PropertyAdmin(Lists.newArrayList("role1", "role2"), Sets.newHashSet("use", "usw"));
admin.properties().updateProperty("prop-xyz", propertyAdmin);
assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1").bundles, Policies.defaultBundle());
admin.namespaces().createNamespace("prop-xyz/use/ns2");
admin.namespaces().createNamespace("prop-xyz/use/ns3", 4);
assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns3").bundles.numBundles, 4);
assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns3").bundles.boundaries.size(), 5);
admin.namespaces().deleteNamespace("prop-xyz/use/ns3");
try {
admin.namespaces().createNamespace("non-existing/usw/ns1");
fail("Should not have passed");
} catch (NotFoundException e) {
// Ok
}
assertEquals(admin.namespaces().getNamespaces("prop-xyz"), Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2"));
assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2"));
try {
admin.namespaces().createNamespace("prop-xyz/usc/ns1");
fail("Should not have passed");
} catch (NotAuthorizedException e) {
// Ok, got the non authorized exception since usc cluster is not in the allowed clusters list.
}
admin.namespaces().grantPermissionOnNamespace("prop-xyz/use/ns1", "my-role", EnumSet.allOf(AuthAction.class));
Policies policies = new Policies();
policies.auth_policies.namespace_auth.put("my-role", EnumSet.allOf(AuthAction.class));
assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies);
assertEquals(admin.namespaces().getPermissions("prop-xyz/use/ns1"), policies.auth_policies.namespace_auth);
assertEquals(admin.namespaces().getDestinations("prop-xyz/use/ns1"), Lists.newArrayList());
admin.namespaces().revokePermissionsOnNamespace("prop-xyz/use/ns1", "my-role");
policies.auth_policies.namespace_auth.remove("my-role");
assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies);
assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(1, 1, 1, 0.0));
admin.namespaces().setPersistence("prop-xyz/use/ns1", new PersistencePolicies(3, 2, 1, 10.0));
assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(3, 2, 1, 10.0));
// Force topic creation and namespace being loaded
Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns1/my-topic");
producer.close();
admin.persistentTopics().delete("persistent://prop-xyz/use/ns1/my-topic");
admin.namespaces().unloadNamespaceBundle("prop-xyz/use/ns1", "0x00000000_0xffffffff");
NamespaceName ns = new NamespaceName("prop-xyz/use/ns1");
// Now, w/ bundle policies, we will use default bundle
NamespaceBundle defaultBundle = bundleFactory.getFullBundle(ns);
int i = 0;
for (; i < 10; i++) {
Optional<NamespaceEphemeralData> data1 = pulsar.getNamespaceService().getOwnershipCache().getOwnerAsync(defaultBundle).get();
if (!data1.isPresent()) {
// Already unloaded
break;
}
LOG.info("Waiting for unload namespace {} to complete. Current service unit isDisabled: {}", defaultBundle, data1.get().isDisabled());
Thread.sleep(1000);
}
assertTrue(i < 10);
admin.namespaces().deleteNamespace("prop-xyz/use/ns1");
assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), Lists.newArrayList("prop-xyz/use/ns2"));
try {
admin.namespaces().unload("prop-xyz/use/ns1");
fail("should have raised exception");
} catch (Exception e) {
// OK excepted
}
// Force topic creation and namespace being loaded
producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns2/my-topic");
producer.close();
admin.persistentTopics().delete("persistent://prop-xyz/use/ns2/my-topic");
// both unload and delete should succeed for ns2 on other broker with a redirect
// otheradmin.namespaces().unload("prop-xyz/use/ns2");
}
use of com.yahoo.pulsar.common.policies.data.PersistencePolicies in project pulsar by yahoo.
the class Namespaces method getPersistence.
@GET
@Path("/{property}/{cluster}/{namespace}/persistence")
@ApiOperation(value = "Get the persistence configuration for a namespace.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") })
public PersistencePolicies getPersistence(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace) {
validateAdminAccessOnProperty(property);
Policies policies = getNamespacePolicies(property, cluster, namespace);
if (policies.persistence == null) {
return new PersistencePolicies(config().getManagedLedgerDefaultEnsembleSize(), config().getManagedLedgerDefaultWriteQuorum(), config().getManagedLedgerDefaultAckQuorum(), 0.0d);
} else {
return policies.persistence;
}
}
Aggregations