Examples with MetadataEntity io.cdap.cdap.api.metadata.MetadataEntity used on opensource projects

Example 1 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class DatasetMetadataStorage method replaceInScope.

private MetadataDataset.Change replaceInScope(MetadataDatasetContext context, MetadataScope scope, MetadataEntity entity, Set<String> newTags, Map<String, String> newProperties, Map<ScopedNameOfKind, MetadataDirective> directives) {
    MetadataDataset dataset = context.getDataset(scope);
    MetadataDataset.Record before = dataset.getMetadata(entity);
    if (newTags.isEmpty() && newProperties.isEmpty()) {
        // this scope remains unchanged
        return new MetadataDataset.Change(before, before);
    }
    Set<String> existingTags = before.getTags();
    Set<String> tagsToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == TAG && (entry.getValue() == MetadataDirective.KEEP || entry.getValue() == MetadataDirective.PRESERVE)).map(Map.Entry::getKey).map(ScopedName::getName).filter(existingTags::contains).collect(Collectors.toSet());
    newTags = Sets.union(newTags, tagsToKeepOrPreserve);
    Map<String, String> existingProperties = before.getProperties();
    Map<String, String> propertiesToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == PROPERTY).filter(entry -> existingProperties.containsKey(entry.getKey().getName())).filter(entry -> entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP && !newProperties.containsKey(entry.getKey().getName())).map(Map.Entry::getKey).map(ScopedName::getName).collect(Collectors.toMap(name -> name, existingProperties::get));
    newProperties.putAll(propertiesToKeepOrPreserve);
    Set<String> tagsToRemove = Sets.difference(before.getTags(), newTags);
    Set<String> tagsToAdd = Sets.difference(newTags, before.getTags());
    Set<String> propertiesToRemove = Sets.difference(before.getProperties().keySet(), newProperties.keySet());
    @SuppressWarnings("ConstantConditions") Map<String, String> propertiesToAdd = Maps.filterEntries(newProperties, entry -> !entry.getValue().equals(existingProperties.get(entry.getKey())));
    MetadataDataset.Record after = before;
    if (!tagsToRemove.isEmpty()) {
        after = dataset.removeTags(entity, tagsToRemove).getLatest();
    }
    if (!tagsToAdd.isEmpty()) {
        after = dataset.addTags(entity, tagsToAdd).getLatest();
    }
    if (!propertiesToRemove.isEmpty()) {
        after = dataset.removeProperties(entity, propertiesToRemove).getLatest();
    }
    if (!propertiesToAdd.isEmpty()) {
        after = dataset.addProperties(entity, propertiesToAdd).getLatest();
    }
    return new MetadataDataset.Change(before, after);
}

Example 2 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class DatasetMetadataStorage method remove.

private MetadataChange remove(MetadataDatasetContext context, MetadataMutation.Remove remove) {
    MetadataEntity entity = remove.getEntity();
    MetadataDataset.Change userChange, systemChange;
    if (remove.getRemovals() != null) {
        Set<String> userTagsToRemove = new HashSet<>();
        Set<String> systemTagsToRemove = new HashSet<>();
        Set<String> userPropertiesToRemove = new HashSet<>();
        Set<String> systemPropertiesToRemove = new HashSet<>();
        remove.getRemovals().forEach(removal -> (TAG == removal.getKind() ? USER == removal.getScope() ? userTagsToRemove : systemTagsToRemove : USER == removal.getScope() ? userPropertiesToRemove : systemPropertiesToRemove).add(removal.getName()));
        userChange = removeInScope(context, USER, entity, userTagsToRemove, userPropertiesToRemove);
        systemChange = removeInScope(context, SYSTEM, entity, systemTagsToRemove, systemPropertiesToRemove);
    } else {
        Set<MetadataScope> scopes = remove.getScopes();
        Set<MetadataKind> kinds = remove.getKinds();
        userChange = removeScope(context, USER, entity, scopes, kinds);
        systemChange = removeScope(context, SYSTEM, entity, scopes, kinds);
    }
    return combineChanges(entity, userChange, systemChange);
}

Example 3 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class DatasetMetadataStorage method readScope.

private MetadataDataset.Record readScope(MetadataDatasetContext context, MetadataScope scope, Read read) {
    MetadataEntity entity = read.getEntity();
    if (read.getSelection() == null && (!read.getScopes().contains(scope) || read.getKinds().isEmpty())) {
        return new MetadataDataset.Record(entity);
    }
    Set<ScopedNameOfKind> selectionForScope = null;
    if (read.getSelection() != null) {
        // noinspection ConstantConditions
        selectionForScope = Sets.filter(read.getSelection(), entry -> entry.getScope() == scope);
        if (selectionForScope.isEmpty()) {
            return new MetadataDataset.Record(entity);
        }
    }
    // now we know we must read from the dataset
    MetadataDataset dataset = context.getDataset(scope);
    if (selectionForScope != null) {
        // request is for a specific set of tags and properties
        Set<String> tagsToRead = selectionForScope.stream().filter(entry -> TAG == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
        Set<String> propertiesToRead = selectionForScope.stream().filter(entry -> PROPERTY == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
        Set<String> tags = tagsToRead.isEmpty() ? Collections.emptySet() : Sets.intersection(tagsToRead, dataset.getTags(entity));
        Map<String, String> properties = propertiesToRead.isEmpty() ? Collections.emptyMap() : Maps.filterKeys(dataset.getProperties(entity), propertiesToRead::contains);
        return new MetadataDataset.Record(entity, properties, tags);
    }
    if (MetadataKind.ALL.equals(read.getKinds())) {
        // all metadata kinds requested
        return dataset.getMetadata(entity);
    }
    // exactly one kind is requested
    MetadataKind requestKind = read.getKinds().iterator().next();
    if (requestKind == TAG) {
        return new MetadataDataset.Record(entity, Collections.emptyMap(), dataset.getTags(entity));
    }
    if (requestKind == PROPERTY) {
        return new MetadataDataset.Record(entity, dataset.getProperties(entity), Collections.emptySet());
    }
    throw new IllegalStateException("Encountered metadata read request for unknown kind " + requestKind);
}

Example 4 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class SearchHelper method search.

private MetadataSearchResponse search(Set<MetadataScope> scopes, SearchRequest request) {
    List<MetadataEntry> results = new LinkedList<>();
    List<String> cursors = new LinkedList<>();
    for (MetadataScope scope : scopes) {
        SearchResults searchResults = execute(context -> context.getDataset(scope).search(request));
        results.addAll(searchResults.getResults());
        cursors.addAll(searchResults.getCursors());
    }
    int offset = request.getOffset();
    int limit = request.getLimit();
    SortInfo sortInfo = request.getSortInfo();
    // sort if required
    Set<MetadataEntity> sortedEntities = getSortedEntities(results, sortInfo);
    int total = sortedEntities.size();
    // pagination is not performed at the dataset level, because:
    // 1. scoring is needed for DEFAULT sort info. So perform it here for now.
    // 2. Even when using custom sorting, we need to remove elements from the beginning to the offset and the cursors
    // at the end
    // TODO: Figure out how all of this can be done server (HBase) side
    int startIndex = Math.min(request.getOffset(), sortedEntities.size());
    // Account for overflow
    int endIndex = (int) Math.min(Integer.MAX_VALUE, (long) offset + limit);
    endIndex = Math.min(endIndex, sortedEntities.size());
    // add 1 to maxIndex because end index is exclusive
    Set<MetadataEntity> subSortedEntities = new LinkedHashSet<>(ImmutableList.copyOf(sortedEntities).subList(startIndex, endIndex));
    // Fetch metadata for entities in the result list
    // Note: since the fetch is happening in a different transaction, the metadata for entities may have been
    // removed. It is okay not to have metadata for some results in case this happens.
    Set<MetadataSearchResultRecord> finalResults = execute(context -> addMetadataToEntities(subSortedEntities, fetchMetadata(context.getDataset(SYSTEM), subSortedEntities), fetchMetadata(context.getDataset(USER), subSortedEntities)));
    return new MetadataSearchResponse(sortInfo.getSortBy() + " " + sortInfo.getSortOrder(), offset, limit, request.getNumCursors(), total, finalResults, cursors, request.shouldShowHidden(), request.getEntityScopes());
}

Example 5 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataHttpHandler method makeBackwardCompatible.

/**
 * If the MetadataEntity Builder key-value represent an application or artifact which is versioned in CDAP i.e. their
 * metadata entity representation ends with 'version' rather than 'application' or 'artifact' and the type is also
 * set to 'version' then for backward compatibility of rest end points return an updated builder which has the
 * correct type. This is only needed in 5.0 for backward compatibility of the rest endpoint.
 * From 5.0 and later the rest end point must be called with a query parameter which specify the type of the
 * metadata entity if the type is not the last key. (CDAP-13678)
 */
@VisibleForTesting
static MetadataEntity.Builder makeBackwardCompatible(MetadataEntity.Builder builder) {
    MetadataEntity entity = builder.build();
    List<MetadataEntity.KeyValue> entityKeyValues = StreamSupport.stream(entity.spliterator(), false).collect(Collectors.toList());
    if (entityKeyValues.size() == 3 && entity.getType().equals(MetadataEntity.VERSION) && (entityKeyValues.get(1).getKey().equals(MetadataEntity.ARTIFACT) || entityKeyValues.get(1).getKey().equals(MetadataEntity.APPLICATION))) {
        // this is artifact or application so update the builder
        MetadataEntity.Builder actualEntityBuilder = MetadataEntity.builder();
        // namespace
        actualEntityBuilder.append(entityKeyValues.get(0).getKey(), entityKeyValues.get(0).getValue());
        // application or artifact (so append as type)
        actualEntityBuilder.appendAsType(entityKeyValues.get(1).getKey(), entityKeyValues.get(1).getValue());
        // version detail
        actualEntityBuilder.append(entityKeyValues.get(2).getKey(), entityKeyValues.get(2).getValue());
        return actualEntityBuilder;
    }
    return builder;
}