Examples with Policies com.yahoo.pulsar.common.policies.data.Policies used on opensource projects

Example 6 with Policies

use of com.yahoo.pulsar.common.policies.data.Policies in project pulsar by yahoo.

the class AdminResource method getNamespacePolicies.

protected Policies getNamespacePolicies(String property, String cluster, String namespace) {
    try {
        Policies policies = policiesCache().get(AdminResource.path("policies", property, cluster, namespace)).orElseThrow(() -> new RestException(Status.NOT_FOUND, "Namespace does not exist"));
        // fetch bundles from LocalZK-policies
        NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory().getBundles(new NamespaceName(property, cluster, namespace));
        BundlesData bundleData = NamespaceBundleFactory.getBundlesData(bundles);
        policies.bundles = bundleData != null ? bundleData : policies.bundles;
        return policies;
    } catch (RestException re) {
        throw re;
    } catch (Exception e) {
        log.error("[{}] Failed to get namespace policies {}/{}/{}", clientAppId(), property, cluster, namespace, e);
        throw new RestException(e);
    }
}

Example 7 with Policies

use of com.yahoo.pulsar.common.policies.data.Policies in project pulsar by yahoo.

the class BrokerService method getManagedLedgerConfig.

public CompletableFuture<ManagedLedgerConfig> getManagedLedgerConfig(DestinationName topicName) {
    CompletableFuture<ManagedLedgerConfig> future = new CompletableFuture<>();
    // Execute in background thread, since getting the policies might block if the z-node wasn't already cached
    pulsar.getOrderedExecutor().submitOrdered(topicName, safeRun(() -> {
        NamespaceName namespace = topicName.getNamespaceObject();
        ServiceConfiguration serviceConfig = pulsar.getConfiguration();
        // Get persistence policy for this destination
        Policies policies;
        try {
            policies = pulsar.getConfigurationCache().policiesCache().get(AdminResource.path("policies", namespace.getProperty(), namespace.getCluster(), namespace.getLocalName())).orElse(null);
        } catch (Throwable t) {
            // Ignoring since if we don't have policies, we fallback on the default
            log.warn("Got exception when reading persistence policy for {}: {}", topicName, t.getMessage(), t);
            future.completeExceptionally(t);
            return;
        }
        PersistencePolicies persistencePolicies = policies != null ? policies.persistence : null;
        RetentionPolicies retentionPolicies = policies != null ? policies.retention_policies : null;
        if (persistencePolicies == null) {
            // Apply default values
            persistencePolicies = new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), serviceConfig.getManagedLedgerDefaultWriteQuorum(), serviceConfig.getManagedLedgerDefaultAckQuorum(), serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit());
        }
        if (retentionPolicies == null) {
            retentionPolicies = new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), serviceConfig.getDefaultRetentionSizeInMB());
        }
        ManagedLedgerConfig config = new ManagedLedgerConfig();
        config.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble());
        config.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum());
        config.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum());
        config.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate());
        config.setDigestType(DigestType.CRC32);
        config.setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist());
        config.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger());
        config.setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
        config.setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
        config.setMaxSizePerLedgerMb(2048);
        config.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize());
        config.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum());
        config.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum());
        config.setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger());
        config.setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds());
        config.setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES);
        config.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB());
        future.complete(config);
    }, (exception) -> future.completeExceptionally(exception)));
    return future;
}

Example 8 with Policies

use of com.yahoo.pulsar.common.policies.data.Policies in project pulsar by yahoo.

the class Namespaces method setRetention.

@POST
@Path("/{property}/{cluster}/{namespace}/retention")
@ApiOperation(value = " Set retention configuration on a namespace.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "Retention Quota must exceed backlog quota") })
public void setRetention(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, RetentionPolicies retention) {
    validatePoliciesReadOnlyAccess();
    try {
        Stat nodeStat = new Stat();
        final String path = path("policies", property, cluster, namespace);
        byte[] content = globalZk().getData(path, null, nodeStat);
        Policies policies = jsonMapper().readValue(content, Policies.class);
        if (!checkQuotas(policies, retention)) {
            log.warn("[{}] Failed to update retention configuration for namespace {}/{}/{}: conflicts with backlog quota", clientAppId(), property, cluster, namespace);
            throw new RestException(Status.PRECONDITION_FAILED, "Retention Quota must exceed configured backlog quota for namespace.");
        }
        policies.retention_policies = retention;
        globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion());
        policiesCache().invalidate(path("policies", property, cluster, namespace));
        log.info("[{}] Successfully updated retention configuration: namespace={}/{}/{}, map={}", clientAppId(), property, cluster, namespace, jsonMapper().writeValueAsString(policies.retention_policies));
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to update retention configuration for namespace {}/{}/{}: does not exist", clientAppId(), property, cluster, namespace);
        throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
    } catch (KeeperException.BadVersionException e) {
        log.warn("[{}] Failed to update retention configuration for namespace {}/{}/{}: concurrent modification", clientAppId(), property, cluster, namespace);
        throw new RestException(Status.CONFLICT, "Concurrent modification");
    } catch (RestException pfe) {
        throw pfe;
    } catch (Exception e) {
        log.error("[{}] Failed to update retention configuration for namespace {}/{}/{}", clientAppId(), property, cluster, namespace, e);
        throw new RestException(e);
    }
}

Example 9 with Policies

use of com.yahoo.pulsar.common.policies.data.Policies in project pulsar by yahoo.

the class Namespaces method deleteNamespace.

@DELETE
@Path("/{property}/{cluster}/{namespace}")
@ApiOperation(value = "Delete a namespace and all the destinations under it.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist"), @ApiResponse(code = 409, message = "Namespace is not empty") })
public void deleteNamespace(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    NamespaceName nsName = new NamespaceName(property, cluster, namespace);
    validateAdminAccessOnProperty(property);
    validatePoliciesReadOnlyAccess();
    // ensure that non-global namespace is directed to the correct cluster
    validateClusterOwnership(cluster);
    Entry<Policies, Stat> policiesNode = null;
    Policies policies = null;
    // ensure the local cluster is the only cluster for the global namespace configuration
    try {
        policiesNode = policiesCache().getWithStat(path("policies", property, cluster, namespace)).orElseThrow(() -> new RestException(Status.NOT_FOUND, "Namespace " + nsName + " does not exist."));
        policies = policiesNode.getKey();
        if (cluster.equals(Namespaces.GLOBAL_CLUSTER)) {
            if (policies.replication_clusters.size() > 1) {
                // There are still more than one clusters configured for the global namespace
                throw new RestException(Status.PRECONDITION_FAILED, "Cannot delete the global namespace " + nsName + ". There are still more than one replication clusters configured.");
            }
            if (policies.replication_clusters.size() == 1 && !policies.replication_clusters.contains(config().getClusterName())) {
                // the only replication cluster is other cluster, redirect
                String replCluster = policies.replication_clusters.get(0);
                ClusterData replClusterData = clustersCache().get(AdminResource.path("clusters", replCluster)).orElseThrow(() -> new RestException(Status.NOT_FOUND, "Cluser " + replCluster + " does not exist"));
                URL replClusterUrl;
                if (!config().isTlsEnabled()) {
                    replClusterUrl = new URL(replClusterData.getServiceUrl());
                } else if (!replClusterData.getServiceUrlTls().isEmpty()) {
                    replClusterUrl = new URL(replClusterData.getServiceUrlTls());
                } else {
                    throw new RestException(Status.PRECONDITION_FAILED, "The replication cluster does not provide TLS encrypted service");
                }
                URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(replClusterUrl.getHost()).port(replClusterUrl.getPort()).replaceQueryParam("authoritative", false).build();
                log.debug("[{}] Redirecting the rest call to {}: cluster={}", clientAppId(), redirect, cluster);
                throw new WebApplicationException(Response.temporaryRedirect(redirect).build());
            }
        }
    } catch (WebApplicationException wae) {
        throw wae;
    } catch (Exception e) {
        throw new RestException(e);
    }
    List<String> destinations = getDestinations(property, cluster, namespace);
    if (!destinations.isEmpty()) {
        log.info("Found destinations: {}", destinations);
        throw new RestException(Status.CONFLICT, "Cannot delete non empty namespace");
    }
    // set the policies to deleted so that somebody else cannot acquire this namespace
    try {
        policies.deleted = true;
        globalZk().setData(path("policies", property, cluster, namespace), jsonMapper().writeValueAsBytes(policies), policiesNode.getValue().getVersion());
        policiesCache().invalidate(path("policies", property, cluster, namespace));
    } catch (Exception e) {
        log.error("[{}] Failed to delete namespace on global ZK {}/{}/{}", clientAppId(), property, cluster, namespace, e);
        throw new RestException(e);
    }
    // remove from owned namespace map and ephemeral node from ZK
    try {
        NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory().getBundles(nsName);
        for (NamespaceBundle bundle : bundles.getBundles()) {
            // check if the bundle is owned by any broker, if not then we do not need to delete the bundle
            if (pulsar().getNamespaceService().getOwner(bundle).isPresent()) {
                pulsar().getAdminClient().namespaces().deleteNamespaceBundle(nsName.toString(), bundle.getBundleRange());
            }
        }
        // we have successfully removed all the ownership for the namespace, the policies znode can be deleted now
        globalZk().delete(path("policies", property, cluster, namespace), -1);
        policiesCache().invalidate(path("policies", property, cluster, namespace));
    } catch (PulsarAdminException cae) {
        throw new RestException(cae);
    } catch (Exception e) {
        log.error(String.format("[%s] Failed to remove owned namespace %s/%s/%s", clientAppId(), property, cluster, namespace), e);
    // avoid throwing exception in case of the second failure
    }
}

Example 10 with Policies

use of com.yahoo.pulsar.common.policies.data.Policies in project pulsar by yahoo.

the class Namespaces method setPersistence.

@POST
@Path("/{property}/{cluster}/{namespace}/persistence")
@ApiOperation(value = "Set the persistence configuration for all the destinations on a namespace.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") })
public void setPersistence(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, PersistencePolicies persistence) {
    validatePoliciesReadOnlyAccess();
    try {
        Stat nodeStat = new Stat();
        final String path = path("policies", property, cluster, namespace);
        byte[] content = globalZk().getData(path, null, nodeStat);
        Policies policies = jsonMapper().readValue(content, Policies.class);
        policies.persistence = persistence;
        globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion());
        policiesCache().invalidate(path("policies", property, cluster, namespace));
        log.info("[{}] Successfully updated persistence configuration: namespace={}/{}/{}, map={}", clientAppId(), property, cluster, namespace, jsonMapper().writeValueAsString(policies.persistence));
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to update persistence configuration for namespace {}/{}/{}: does not exist", clientAppId(), property, cluster, namespace);
        throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
    } catch (KeeperException.BadVersionException e) {
        log.warn("[{}] Failed to update persistence configuration for namespace {}/{}/{}: concurrent modification", clientAppId(), property, cluster, namespace);
        throw new RestException(Status.CONFLICT, "Concurrent modification");
    } catch (Exception e) {
        log.error("[{}] Failed to update persistence configuration for namespace {}/{}/{}", clientAppId(), property, cluster, namespace, e);
        throw new RestException(e);
    }
}