Examples with RangerResourceTrie org.apache.ranger.plugin.policyengine.RangerResourceTrie used on opensource projects

Example 1 with RangerResourceTrie

use of org.apache.ranger.plugin.policyengine.RangerResourceTrie in project ranger by apache.

the class RangerTagEnricher method processServiceTags.

private void processServiceTags(ServiceTags serviceTags) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Processing all service-tags");
    }
    boolean isInError = false;
    if (CollectionUtils.isEmpty(serviceTags.getServiceResources())) {
        LOG.info("There are no tagged resources for service " + serviceName);
        enrichedServiceTags = null;
    } else {
        ResourceHierarchies hierarchies = new ResourceHierarchies();
        List<RangerServiceResourceMatcher> resourceMatchers = new ArrayList<>();
        List<RangerServiceResource> serviceResources = serviceTags.getServiceResources();
        for (RangerServiceResource serviceResource : serviceResources) {
            RangerServiceResourceMatcher serviceResourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies);
            if (serviceResourceMatcher != null) {
                resourceMatchers.add(serviceResourceMatcher);
            } else {
                LOG.error("Could not create service-resource-matcher for service-resource:[" + serviceResource + "]");
                isInError = true;
                break;
            }
        }
        if (isInError) {
            serviceTags.setTagVersion(-1L);
            LOG.error("Error in processing tag-deltas. Will continue to use old tags");
        } else {
            Map<String, RangerResourceTrie<RangerServiceResourceMatcher>> serviceResourceTrie = null;
            if (!disableTrieLookupPrefilter) {
                serviceResourceTrie = new HashMap<>();
                for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
                    serviceResourceTrie.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, resourceMatchers, getPolicyEngineOptions().optimizeTagTrieForRetrieval, getPolicyEngineOptions().optimizeTagTrieForSpace, null));
                }
            }
            enrichedServiceTags = new EnrichedServiceTags(serviceTags, resourceMatchers, serviceResourceTrie);
        }
    }
}

Example 2 with RangerResourceTrie

use of org.apache.ranger.plugin.policyengine.RangerResourceTrie in project ranger by apache.

the class RangerTagEnricher method processServiceTagDeltas.

private void processServiceTagDeltas(ServiceTags deltas, ServiceTags allServiceTags, Map<String, RangerResourceTrie<RangerServiceResourceMatcher>> serviceResourceTrie) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Delta contains changes other than tag attribute changes, [" + deltas.getTagsChangeExtent() + "]");
    }
    boolean isInError = false;
    ResourceHierarchies hierarchies = new ResourceHierarchies();
    List<RangerServiceResourceMatcher> resourceMatchers = new ArrayList<>();
    if (enrichedServiceTags != null) {
        resourceMatchers.addAll(enrichedServiceTags.getServiceResourceMatchers());
    }
    List<RangerServiceResource> changedServiceResources = deltas.getServiceResources();
    for (RangerServiceResource serviceResource : changedServiceResources) {
        final boolean removedOldServiceResource = MapUtils.isEmpty(serviceResource.getResourceElements()) || removeOldServiceResource(serviceResource, resourceMatchers, serviceResourceTrie);
        if (removedOldServiceResource) {
            if (!StringUtils.isEmpty(serviceResource.getResourceSignature())) {
                RangerServiceResourceMatcher resourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies);
                if (resourceMatcher != null) {
                    for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
                        RangerResourceTrie<RangerServiceResourceMatcher> trie = serviceResourceTrie.get(resourceDef.getName());
                        if (trie != null) {
                            trie.add(serviceResource.getResourceElements().get(resourceDef.getName()), resourceMatcher);
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Added resource-matcher for service-resource:[" + serviceResource + "]");
                            }
                        } else {
                            trie = new RangerResourceTrie<>(resourceDef, Collections.singletonList(resourceMatcher), getPolicyEngineOptions().optimizeTagTrieForRetrieval, getPolicyEngineOptions().optimizeTagTrieForSpace, null);
                            serviceResourceTrie.put(resourceDef.getName(), trie);
                        }
                    }
                    resourceMatchers.add(resourceMatcher);
                } else {
                    LOG.error("Could not create resource-matcher for resource: [" + serviceResource + "]. Should NOT happen!!");
                    LOG.error("Setting tagVersion to -1 to ensure that in the next download all tags are downloaded");
                    isInError = true;
                }
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Service-resource:[id=" + serviceResource.getId() + "] is deleted as its resource-signature is empty. No need to create it!");
                }
            }
        } else {
            isInError = true;
        }
        if (isInError) {
            break;
        }
    }
    if (isInError) {
        LOG.error("Error in processing tag-deltas. Will continue to use old tags");
        deltas.setTagVersion(-1L);
    } else {
        for (Map.Entry<String, RangerResourceTrie<RangerServiceResourceMatcher>> entry : serviceResourceTrie.entrySet()) {
            entry.getValue().wrapUpUpdate();
        }
        enrichedServiceTags = new EnrichedServiceTags(allServiceTags, resourceMatchers, serviceResourceTrie);
    }
}

Example 3 with RangerResourceTrie

use of org.apache.ranger.plugin.policyengine.RangerResourceTrie in project ranger by apache.

the class RangerTagEnricher method getEvaluators.

private List<RangerServiceResourceMatcher> getEvaluators(RangerAccessRequest request, EnrichedServiceTags enrichedServiceTags) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("==> RangerTagEnricher.getEvaluators(request=" + request + ")");
    }
    List<RangerServiceResourceMatcher> ret = Collections.EMPTY_LIST;
    RangerAccessResource resource = request.getResource();
    final Map<String, RangerResourceTrie<RangerServiceResourceMatcher>> serviceResourceTrie = enrichedServiceTags.getServiceResourceTrie();
    if (resource == null || resource.getKeys() == null || resource.getKeys().isEmpty() || serviceResourceTrie == null) {
        ret = enrichedServiceTags.getServiceResourceMatchers();
    } else {
        RangerPerfTracer perf = null;
        if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) {
            perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerTagEnricher.getEvaluators(resource=" + resource.getAsString() + ")");
        }
        List<String> resourceKeys = serviceDefHelper.getOrderedResourceNames(resource.getKeys());
        Set<RangerServiceResourceMatcher> smallestList = null;
        if (CollectionUtils.isNotEmpty(resourceKeys)) {
            for (String resourceName : resourceKeys) {
                RangerResourceTrie<RangerServiceResourceMatcher> trie = serviceResourceTrie.get(resourceName);
                if (trie == null) {
                    // if no trie exists for this resource level, ignore and continue to next level
                    continue;
                }
                Set<RangerServiceResourceMatcher> serviceResourceMatchersForResource = trie.getEvaluatorsForResource(resource.getValue(resourceName), request.getResourceMatchingScope());
                Set<RangerServiceResourceMatcher> inheritedResourceMatchers = trie.getInheritedEvaluators();
                if (smallestList != null) {
                    if (CollectionUtils.isEmpty(inheritedResourceMatchers) && CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
                        smallestList = null;
                    } else if (CollectionUtils.isEmpty(inheritedResourceMatchers)) {
                        smallestList.retainAll(serviceResourceMatchersForResource);
                    } else if (CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
                        smallestList.retainAll(inheritedResourceMatchers);
                    } else {
                        Set<RangerServiceResourceMatcher> smaller, bigger;
                        if (serviceResourceMatchersForResource.size() < inheritedResourceMatchers.size()) {
                            smaller = serviceResourceMatchersForResource;
                            bigger = inheritedResourceMatchers;
                        } else {
                            smaller = inheritedResourceMatchers;
                            bigger = serviceResourceMatchersForResource;
                        }
                        Set<RangerServiceResourceMatcher> tmp = new HashSet<>();
                        if (smallestList.size() < smaller.size()) {
                            smallestList.stream().filter(smaller::contains).forEach(tmp::add);
                            smallestList.stream().filter(bigger::contains).forEach(tmp::add);
                        } else {
                            smaller.stream().filter(smallestList::contains).forEach(tmp::add);
                            if (smallestList.size() < bigger.size()) {
                                smallestList.stream().filter(bigger::contains).forEach(tmp::add);
                            } else {
                                bigger.stream().filter(smallestList::contains).forEach(tmp::add);
                            }
                        }
                        smallestList = tmp;
                    }
                } else {
                    if (CollectionUtils.isEmpty(inheritedResourceMatchers) || CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
                        Set<RangerServiceResourceMatcher> tmp = CollectionUtils.isEmpty(inheritedResourceMatchers) ? serviceResourceMatchersForResource : inheritedResourceMatchers;
                        smallestList = resourceKeys.size() == 1 || CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp);
                    } else {
                        smallestList = new HashSet<>(serviceResourceMatchersForResource);
                        smallestList.addAll(inheritedResourceMatchers);
                    }
                }
                if (CollectionUtils.isEmpty(smallestList)) {
                    // no tags for this resource, bail out
                    smallestList = null;
                    break;
                }
            }
        }
        if (smallestList != null) {
            ret = new ArrayList<>(smallestList);
        }
        RangerPerfTracer.logAlways(perf);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("<== RangerTagEnricher.getEvaluators(request=" + request + "): evaluatorCount=" + ret.size());
    }
    return ret;
}

Example 4 with RangerResourceTrie

use of org.apache.ranger.plugin.policyengine.RangerResourceTrie in project ranger by apache.

the class RangerSecurityZoneValidator method validateZoneServiceInAllZones.

private boolean validateZoneServiceInAllZones(List<RangerSecurityZone> zones, String serviceName, RangerServiceDef serviceDef, List<ValidationFailureDetails> failures) {
    if (LOG.isDebugEnabled()) {
        LOG.debug(String.format("==> RangerPolicyValidator.validateZoneServiceInAllZones(%s, %s, %s, %s)", zones, serviceName, serviceDef, failures));
    }
    boolean ret = true;
    // For each zone, get list-of-resources corresponding to serviceName.
    // For each list-of-resources:
    // get one resource (this is a map of <String, List<String>>); convert it into map of <String, RangerPolicyResource>. excludes is always false, recursive true only for HDFS
    // build a subclass of RangerPolicyResourceEvaluator with id of zone, zoneName as a member, and RangerDefaultResourceMatcher as matcher.
    // add this to list-of-evaluators
    Map<String, List<RangerZoneResourceMatcher>> matchersForResourceDef = new HashMap<>();
    for (RangerSecurityZone zone : zones) {
        List<HashMap<String, List<String>>> resources = zone.getServices().get(serviceName).getResources();
        for (Map<String, List<String>> resource : resources) {
            Map<String, RangerPolicy.RangerPolicyResource> policyResources = new HashMap<>();
            for (Map.Entry<String, List<String>> entry : resource.entrySet()) {
                String resourceDefName = entry.getKey();
                List<String> resourceValues = entry.getValue();
                RangerPolicy.RangerPolicyResource policyResource = new RangerPolicy.RangerPolicyResource();
                policyResource.setIsExcludes(false);
                policyResource.setIsRecursive(EmbeddedServiceDefsUtil.isRecursiveEnabled(serviceDef, resourceDefName));
                policyResource.setValues(resourceValues);
                policyResources.put(resourceDefName, policyResource);
                if (matchersForResourceDef.get(resourceDefName) == null) {
                    matchersForResourceDef.put(resourceDefName, new ArrayList<>());
                }
            }
            RangerZoneResourceMatcher matcher = new RangerZoneResourceMatcher(zone.getName(), policyResources, serviceDef);
            for (String resourceDefName : resource.keySet()) {
                matchersForResourceDef.get(resourceDefName).add(matcher);
            }
        }
    }
    // Build a map of trie with list-of-evaluators with one entry corresponds to one resource-def if it exists in the list-of-resources
    Map<String, RangerResourceTrie<RangerZoneResourceMatcher>> trieMap = new HashMap<>();
    List<RangerServiceDef.RangerResourceDef> resourceDefs = serviceDef.getResources();
    for (Map.Entry<String, List<RangerZoneResourceMatcher>> entry : matchersForResourceDef.entrySet()) {
        String resourceDefName = entry.getKey();
        List<RangerZoneResourceMatcher> matchers = entry.getValue();
        RangerServiceDef.RangerResourceDef resourceDef = null;
        for (RangerServiceDef.RangerResourceDef element : resourceDefs) {
            if (StringUtils.equals(element.getName(), resourceDefName)) {
                resourceDef = element;
                break;
            }
        }
        trieMap.put(entry.getKey(), new RangerResourceTrie<>(resourceDef, matchers));
    }
    // For each zone, get list-of-resources corresponding to serviceName
    // For each list-of-resources:
    // get one resource; for each level in the resource, run it through map of trie and get possible evaluators.
    // check each evaluator to see if the resource-match actually happens. If yes then add the zone-evaluator to matching evaluators.
    // flag error if there are more than one matching evaluators with different zone-ids.
    // 
    RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, true);
    for (RangerSecurityZone zone : zones) {
        List<HashMap<String, List<String>>> resources = zone.getServices().get(serviceName).getResources();
        for (Map<String, List<String>> resource : resources) {
            Set<RangerZoneResourceMatcher> smallestList = null;
            List<String> resourceKeys = serviceDefHelper.getOrderedResourceNames(resource.keySet());
            for (String resourceDefName : resourceKeys) {
                List<String> resourceValues = resource.get(resourceDefName);
                RangerResourceTrie<RangerZoneResourceMatcher> trie = trieMap.get(resourceDefName);
                Set<RangerZoneResourceMatcher> zoneMatchersForResource = trie.getEvaluatorsForResource(resourceValues);
                Set<RangerZoneResourceMatcher> inheritedZoneMatchers = trie.getInheritedEvaluators();
                if (LOG.isDebugEnabled()) {
                    LOG.debug("ResourceDefName:[" + resourceDefName + "], values:[" + resourceValues + "], matched-zones:[" + zoneMatchersForResource + "], inherited-zones:[" + inheritedZoneMatchers + "]");
                }
                if (smallestList != null) {
                    if (CollectionUtils.isEmpty(inheritedZoneMatchers) && CollectionUtils.isEmpty(zoneMatchersForResource)) {
                        smallestList = null;
                    } else if (CollectionUtils.isEmpty(inheritedZoneMatchers)) {
                        smallestList.retainAll(zoneMatchersForResource);
                    } else if (CollectionUtils.isEmpty(zoneMatchersForResource)) {
                        smallestList.retainAll(inheritedZoneMatchers);
                    } else {
                        Set<RangerZoneResourceMatcher> smaller, bigger;
                        if (zoneMatchersForResource.size() < inheritedZoneMatchers.size()) {
                            smaller = zoneMatchersForResource;
                            bigger = inheritedZoneMatchers;
                        } else {
                            smaller = inheritedZoneMatchers;
                            bigger = zoneMatchersForResource;
                        }
                        Set<RangerZoneResourceMatcher> tmp = new HashSet<>();
                        if (smallestList.size() < smaller.size()) {
                            smallestList.stream().filter(smaller::contains).forEach(tmp::add);
                            smallestList.stream().filter(bigger::contains).forEach(tmp::add);
                        } else {
                            smaller.stream().filter(smallestList::contains).forEach(tmp::add);
                            if (smallestList.size() < bigger.size()) {
                                smallestList.stream().filter(bigger::contains).forEach(tmp::add);
                            } else {
                                bigger.stream().filter(smallestList::contains).forEach(tmp::add);
                            }
                        }
                        smallestList = tmp;
                    }
                } else {
                    if (CollectionUtils.isEmpty(inheritedZoneMatchers) || CollectionUtils.isEmpty(zoneMatchersForResource)) {
                        Set<RangerZoneResourceMatcher> tmp = CollectionUtils.isEmpty(inheritedZoneMatchers) ? zoneMatchersForResource : inheritedZoneMatchers;
                        smallestList = resourceKeys.size() == 1 || CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp);
                    } else {
                        smallestList = new HashSet<>(zoneMatchersForResource);
                        smallestList.addAll(inheritedZoneMatchers);
                    }
                }
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Resource:[" + resource + "], matched-zones:[" + smallestList + "]");
            }
            if (CollectionUtils.isEmpty(smallestList) || smallestList.size() == 1) {
                continue;
            }
            final Set<RangerZoneResourceMatcher> intersection = smallestList;
            RangerAccessResourceImpl accessResource = new RangerAccessResourceImpl();
            accessResource.setServiceDef(serviceDef);
            for (Map.Entry<String, List<String>> entry : resource.entrySet()) {
                accessResource.setValue(entry.getKey(), entry.getValue());
            }
            Set<String> matchedZoneNames = new HashSet<>();
            for (RangerZoneResourceMatcher zoneMatcher : intersection) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Trying to match resource:[" + accessResource + "] using zoneMatcher:[" + zoneMatcher + "]");
                }
                // These are potential matches. Try to really match them
                if (zoneMatcher.getPolicyResourceMatcher().isMatch(accessResource, RangerPolicyResourceMatcher.MatchScope.ANY, null)) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Matched resource:[" + accessResource + "] using zoneMatcher:[" + zoneMatcher + "]");
                    }
                    // Actual match happened
                    matchedZoneNames.add(zoneMatcher.getSecurityZoneName());
                } else {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Did not match resource:[" + accessResource + "] using zoneMatcher:[" + zoneMatcher + "]");
                    }
                }
            }
            LOG.info("The following zone-names matched resource:[" + resource + "]: " + matchedZoneNames);
            if (matchedZoneNames.size() > 1) {
                ValidationErrorCode error = ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_ZONE_RESOURCE_CONFLICT;
                failures.add(new ValidationFailureDetailsBuilder().becauseOf(error.getMessage(matchedZoneNames, resource)).errorCode(error.getErrorCode()).build());
                ret = false;
                break;
            }
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(String.format("<== RangerPolicyValidator.validateZoneServiceInAllZones(%s, %s, %s, %s) : %s", zones, serviceName, serviceDef, failures, ret));
    }
    return ret;
}

Example 5 with RangerResourceTrie

use of org.apache.ranger.plugin.policyengine.RangerResourceTrie in project ranger by apache.

the class RangerTagEnricher method setServiceTags.

protected void setServiceTags(final ServiceTags serviceTags, final boolean rebuildOnlyIndex) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("==> RangerTagEnricher.setServiceTags(serviceTags=" + serviceTags + ", rebuildOnlyIndex=" + rebuildOnlyIndex + ")");
    }
    try (RangerReadWriteLock.RangerLock writeLock = this.lock.getWriteLock()) {
        if (LOG.isDebugEnabled()) {
            if (writeLock.isLockingEnabled()) {
                LOG.debug("Acquired lock - " + writeLock);
            }
        }
        RangerPerfTracer perf = null;
        if (RangerPerfTracer.isPerfTraceEnabled(PERF_SET_SERVICETAGS_LOG)) {
            perf = RangerPerfTracer.getPerfTracer(PERF_SET_SERVICETAGS_LOG, "RangerTagEnricher.setServiceTags(newTagVersion=" + serviceTags.getTagVersion() + ",isDelta=" + serviceTags.getIsDelta() + ")");
        }
        if (serviceTags == null) {
            LOG.info("ServiceTags is null for service " + serviceName);
            enrichedServiceTags = null;
        } else {
            if (!serviceTags.getIsDelta()) {
                processServiceTags(serviceTags);
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Received service-tag deltas:" + serviceTags);
                }
                ServiceTags oldServiceTags = enrichedServiceTags != null ? enrichedServiceTags.getServiceTags() : new ServiceTags();
                ServiceTags allServiceTags = rebuildOnlyIndex ? oldServiceTags : RangerServiceTagsDeltaUtil.applyDelta(oldServiceTags, serviceTags);
                if (serviceTags.getTagsChangeExtent() == ServiceTags.TagsChangeExtent.NONE) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("No change to service-tags other than version change");
                    }
                } else {
                    if (serviceTags.getTagsChangeExtent() != ServiceTags.TagsChangeExtent.TAGS) {
                        Map<String, RangerResourceTrie<RangerServiceResourceMatcher>> trieMap;
                        if (enrichedServiceTags == null) {
                            trieMap = new HashMap<>();
                        } else {
                            trieMap = writeLock.isLockingEnabled() ? enrichedServiceTags.getServiceResourceTrie() : copyServiceResourceTrie();
                        }
                        processServiceTagDeltas(serviceTags, allServiceTags, trieMap);
                    } else {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Delta contains only tag attribute changes");
                        }
                        List<RangerServiceResourceMatcher> resourceMatchers = enrichedServiceTags != null ? enrichedServiceTags.getServiceResourceMatchers() : new ArrayList<>();
                        Map<String, RangerResourceTrie<RangerServiceResourceMatcher>> serviceResourceTrie = enrichedServiceTags != null ? enrichedServiceTags.getServiceResourceTrie() : new HashMap<>();
                        enrichedServiceTags = new EnrichedServiceTags(allServiceTags, resourceMatchers, serviceResourceTrie);
                    }
                }
            }
        }
        setEnrichedServiceTagsInPlugin();
        RangerPerfTracer.logAlways(perf);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("<== RangerTagEnricher.setServiceTags(serviceTags=" + serviceTags + ", rebuildOnlyIndex=" + rebuildOnlyIndex + ")");
    }
}