Search in sources :

Example 1 with MetadataDirective

use of io.cdap.cdap.spi.metadata.MetadataDirective in project cdap by caskdata.

the class DatasetMetadataStorage method replaceInScope.

private MetadataDataset.Change replaceInScope(MetadataDatasetContext context, MetadataScope scope, MetadataEntity entity, Set<String> newTags, Map<String, String> newProperties, Map<ScopedNameOfKind, MetadataDirective> directives) {
    MetadataDataset dataset = context.getDataset(scope);
    MetadataDataset.Record before = dataset.getMetadata(entity);
    if (newTags.isEmpty() && newProperties.isEmpty()) {
        // this scope remains unchanged
        return new MetadataDataset.Change(before, before);
    }
    Set<String> existingTags = before.getTags();
    Set<String> tagsToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == TAG && (entry.getValue() == MetadataDirective.KEEP || entry.getValue() == MetadataDirective.PRESERVE)).map(Map.Entry::getKey).map(ScopedName::getName).filter(existingTags::contains).collect(Collectors.toSet());
    newTags = Sets.union(newTags, tagsToKeepOrPreserve);
    Map<String, String> existingProperties = before.getProperties();
    Map<String, String> propertiesToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == PROPERTY).filter(entry -> existingProperties.containsKey(entry.getKey().getName())).filter(entry -> entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP && !newProperties.containsKey(entry.getKey().getName())).map(Map.Entry::getKey).map(ScopedName::getName).collect(Collectors.toMap(name -> name, existingProperties::get));
    newProperties.putAll(propertiesToKeepOrPreserve);
    Set<String> tagsToRemove = Sets.difference(before.getTags(), newTags);
    Set<String> tagsToAdd = Sets.difference(newTags, before.getTags());
    Set<String> propertiesToRemove = Sets.difference(before.getProperties().keySet(), newProperties.keySet());
    @SuppressWarnings("ConstantConditions") Map<String, String> propertiesToAdd = Maps.filterEntries(newProperties, entry -> !entry.getValue().equals(existingProperties.get(entry.getKey())));
    MetadataDataset.Record after = before;
    if (!tagsToRemove.isEmpty()) {
        after = dataset.removeTags(entity, tagsToRemove).getLatest();
    }
    if (!tagsToAdd.isEmpty()) {
        after = dataset.addTags(entity, tagsToAdd).getLatest();
    }
    if (!propertiesToRemove.isEmpty()) {
        after = dataset.removeProperties(entity, propertiesToRemove).getLatest();
    }
    if (!propertiesToAdd.isEmpty()) {
        after = dataset.addProperties(entity, propertiesToAdd).getLatest();
    }
    return new MetadataDataset.Change(before, after);
}
Also used : MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) ImmutablePair(io.cdap.cdap.common.utils.ImmutablePair) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Inject(com.google.inject.Inject) HashMap(java.util.HashMap) USER(io.cdap.cdap.api.metadata.MetadataScope.USER) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) HashSet(java.util.HashSet) TAG(io.cdap.cdap.spi.metadata.MetadataKind.TAG) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Metadata(io.cdap.cdap.spi.metadata.Metadata) Map(java.util.Map) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) Read(io.cdap.cdap.spi.metadata.Read) EnumSet(java.util.EnumSet) Sorting(io.cdap.cdap.spi.metadata.Sorting) SortInfo(io.cdap.cdap.data2.metadata.dataset.SortInfo) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) PROPERTY(io.cdap.cdap.spi.metadata.MetadataKind.PROPERTY) ImmutableMap(com.google.common.collect.ImmutableMap) Cursor(io.cdap.cdap.common.metadata.Cursor) SYSTEM(io.cdap.cdap.api.metadata.MetadataScope.SYSTEM) Set(java.util.Set) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) EntityScope(io.cdap.cdap.proto.EntityScope) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) Named(com.google.inject.name.Named) Constants(io.cdap.cdap.common.conf.Constants) VisibleForTesting(com.google.common.annotations.VisibleForTesting) DatasetDefinition(io.cdap.cdap.api.dataset.DatasetDefinition) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) MetadataDataset(io.cdap.cdap.data2.metadata.dataset.MetadataDataset) Collections(java.util.Collections) MetadataDataset(io.cdap.cdap.data2.metadata.dataset.MetadataDataset) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap)

Example 2 with MetadataDirective

use of io.cdap.cdap.spi.metadata.MetadataDirective in project cdap by caskdata.

the class ElasticsearchMetadataStorage method create.

/**
 * Creates the Elasticsearch index request for an entity creation or update.
 * See {@link MetadataMutation.Create} for detailed semantics.
 *
 * @param before the metadata for the mutation's entity before the change
 * @param create the mutation to apply
 *
 * @return an ElasticSearch request to be executed, and the change caused by the mutation
 */
private RequestAndChange create(VersionedMetadata before, MetadataMutation.Create create) {
    // if the entity did not exist before, none of the directives apply and this is equivalent to update()
    if (!before.existing()) {
        return update(create.getEntity(), before, create.getMetadata());
    }
    Metadata meta = create.getMetadata();
    Map<ScopedNameOfKind, MetadataDirective> directives = create.getDirectives();
    // determine the scopes that this mutation applies to (scopes that do not occur in the metadata are no changed)
    Set<MetadataScope> scopes = Stream.concat(meta.getTags().stream(), meta.getProperties().keySet().stream()).map(ScopedName::getScope).collect(Collectors.toSet());
    // compute what previously existing tags and properties have to be preserved (all others are replaced)
    Set<ScopedName> existingTagsToKeep = new HashSet<>();
    Map<ScopedName, String> existingPropertiesToKeep = new HashMap<>();
    // all tags and properties that are in a scope not affected by this mutation
    Sets.difference(MetadataScope.ALL, scopes).forEach(scope -> {
        before.getMetadata().getTags().stream().filter(tag -> tag.getScope().equals(scope)).forEach(existingTagsToKeep::add);
        before.getMetadata().getProperties().entrySet().stream().filter(entry -> entry.getKey().getScope().equals(scope)).forEach(entry -> existingPropertiesToKeep.put(entry.getKey(), entry.getValue()));
    });
    // tags and properties in affected scopes that must be kept or preserved
    directives.entrySet().stream().filter(entry -> scopes.contains(entry.getKey().getScope())).forEach(entry -> {
        ScopedNameOfKind key = entry.getKey();
        if (key.getKind() == MetadataKind.TAG && (entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP)) {
            ScopedName tag = new ScopedName(key.getScope(), key.getName());
            if (!meta.getTags().contains(tag) && before.getMetadata().getTags().contains(tag)) {
                existingTagsToKeep.add(tag);
            }
        } else if (key.getKind() == MetadataKind.PROPERTY) {
            ScopedName property = new ScopedName(key.getScope(), key.getName());
            String existingValue = before.getMetadata().getProperties().get(property);
            String newValue = meta.getProperties().get(property);
            if (existingValue != null && (entry.getValue() == MetadataDirective.PRESERVE && !existingValue.equals(newValue) || entry.getValue() == MetadataDirective.KEEP && newValue == null)) {
                existingPropertiesToKeep.put(property, existingValue);
            }
        }
    });
    // compute the new tags and properties
    Set<ScopedName> newTags = existingTagsToKeep.isEmpty() ? meta.getTags() : Sets.union(meta.getTags(), existingTagsToKeep);
    Map<ScopedName, String> newProperties = meta.getProperties();
    if (!existingPropertiesToKeep.isEmpty()) {
        newProperties = new HashMap<>(newProperties);
        newProperties.putAll(existingPropertiesToKeep);
    }
    Metadata after = new Metadata(newTags, newProperties);
    return new RequestAndChange(writeToIndex(create.getEntity(), before.getVersion(), after), new MetadataChange(create.getEntity(), before.getMetadata(), after));
}
Also used : Arrays(java.util.Arrays) SSLContext(javax.net.ssl.SSLContext) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) SearchHits(org.elasticsearch.search.SearchHits) GetMappingsRequest(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest) Bytes(io.cdap.cdap.api.common.Bytes) RetryStrategies(io.cdap.cdap.common.service.RetryStrategies) KeyStoreException(java.security.KeyStoreException) GsonBuilder(com.google.gson.GsonBuilder) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) DeleteRequest(org.elasticsearch.action.delete.DeleteRequest) IndexRequest(org.elasticsearch.action.index.IndexRequest) Map(java.util.Map) Closeables(com.google.common.io.Closeables) BulkResponse(org.elasticsearch.action.bulk.BulkResponse) Set(java.util.Set) KeyManagementException(java.security.KeyManagementException) ScoreMode(org.apache.lucene.search.join.ScoreMode) CreateIndexResponse(org.elasticsearch.action.admin.indices.create.CreateIndexResponse) Stream(java.util.stream.Stream) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataConstants(io.cdap.cdap.spi.metadata.MetadataConstants) RestStatus(org.elasticsearch.rest.RestStatus) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) CredentialsProvider(org.apache.http.client.CredentialsProvider) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) MatchQueryBuilder(org.elasticsearch.index.query.MatchQueryBuilder) ClearScrollRequest(org.elasticsearch.action.search.ClearScrollRequest) GetMappingsResponse(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse) Checksums(io.cdap.cdap.common.utils.Checksums) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) Read(io.cdap.cdap.spi.metadata.Read) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Resources(com.google.common.io.Resources) WildcardQueryBuilder(org.elasticsearch.index.query.WildcardQueryBuilder) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) DocWriteResponse(org.elasticsearch.action.DocWriteResponse) RestHighLevelClient(org.elasticsearch.client.RestHighLevelClient) SSLContextBuilder(org.apache.http.ssl.SSLContextBuilder) AuthScope(org.apache.http.auth.AuthScope) SearchScrollRequest(org.elasticsearch.action.search.SearchScrollRequest) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) HttpHost(org.apache.http.HttpHost) ActionListener(org.elasticsearch.action.ActionListener) GetResponse(org.elasticsearch.action.get.GetResponse) RestClientBuilder(org.elasticsearch.client.RestClientBuilder) MappingMetaData(org.elasticsearch.cluster.metadata.MappingMetaData) URL(java.net.URL) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) MetadataConflictException(io.cdap.cdap.common.metadata.MetadataConflictException) BasicCredentialsProvider(org.apache.http.impl.client.BasicCredentialsProvider) Settings(org.elasticsearch.common.settings.Settings) Gson(com.google.gson.Gson) Metadata(io.cdap.cdap.spi.metadata.Metadata) SearchResponse(org.elasticsearch.action.search.SearchResponse) RequestOptions(org.elasticsearch.client.RequestOptions) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) ElasticsearchStatusException(org.elasticsearch.ElasticsearchStatusException) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) Splitter(com.google.common.base.Splitter) Sorting(io.cdap.cdap.spi.metadata.Sorting) SearchHit(org.elasticsearch.search.SearchHit) GetRequest(org.elasticsearch.action.get.GetRequest) MultiGetResponse(org.elasticsearch.action.get.MultiGetResponse) ImmutableMap(com.google.common.collect.ImmutableMap) BulkItemResponse(org.elasticsearch.action.bulk.BulkItemResponse) MultiGetItemResponse(org.elasticsearch.action.get.MultiGetItemResponse) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) SortOrder(org.elasticsearch.search.sort.SortOrder) Pattern(java.util.regex.Pattern) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) RestClient(org.elasticsearch.client.RestClient) Retries(io.cdap.cdap.common.service.Retries) RetryStrategy(io.cdap.cdap.common.service.RetryStrategy) XContentType(org.elasticsearch.common.xcontent.XContentType) MultiGetRequest(org.elasticsearch.action.get.MultiGetRequest) HashMap(java.util.HashMap) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) HashSet(java.util.HashSet) WriteRequest(org.elasticsearch.action.support.WriteRequest) Operator(org.elasticsearch.index.query.Operator) CreateIndexRequest(org.elasticsearch.action.admin.indices.create.CreateIndexRequest) GetIndexResponse(org.elasticsearch.action.admin.indices.get.GetIndexResponse) DeleteIndexRequest(org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest) Logger(org.slf4j.Logger) Cursor(io.cdap.cdap.common.metadata.Cursor) GetIndexRequest(org.elasticsearch.action.admin.indices.get.GetIndexRequest) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) ProjectInfo(io.cdap.cdap.common.utils.ProjectInfo) DocWriteRequest(org.elasticsearch.action.DocWriteRequest) AcknowledgedResponse(org.elasticsearch.action.support.master.AcknowledgedResponse) Maps(com.google.common.collect.Maps) TermQueryBuilder(org.elasticsearch.index.query.TermQueryBuilder) UsernamePasswordCredentials(org.apache.http.auth.UsernamePasswordCredentials) TimeUnit(java.util.concurrent.TimeUnit) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) BulkRequest(org.elasticsearch.action.bulk.BulkRequest) Collections(java.util.Collections) MetadataUtil(io.cdap.cdap.common.metadata.MetadataUtil) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) Metadata(io.cdap.cdap.spi.metadata.Metadata) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) HashSet(java.util.HashSet)

Aggregations

VisibleForTesting (com.google.common.annotations.VisibleForTesting)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 Maps (com.google.common.collect.Maps)2 Sets (com.google.common.collect.Sets)2 Inject (com.google.inject.Inject)2 MetadataEntity (io.cdap.cdap.api.metadata.MetadataEntity)2 MetadataScope (io.cdap.cdap.api.metadata.MetadataScope)2 Cursor (io.cdap.cdap.common.metadata.Cursor)2 Metadata (io.cdap.cdap.spi.metadata.Metadata)2 MetadataChange (io.cdap.cdap.spi.metadata.MetadataChange)2 MetadataDirective (io.cdap.cdap.spi.metadata.MetadataDirective)2 MetadataKind (io.cdap.cdap.spi.metadata.MetadataKind)2 MetadataMutation (io.cdap.cdap.spi.metadata.MetadataMutation)2 MetadataRecord (io.cdap.cdap.spi.metadata.MetadataRecord)2 MetadataStorage (io.cdap.cdap.spi.metadata.MetadataStorage)2 MutationOptions (io.cdap.cdap.spi.metadata.MutationOptions)2 Read (io.cdap.cdap.spi.metadata.Read)2 ScopedName (io.cdap.cdap.spi.metadata.ScopedName)2 ScopedNameOfKind (io.cdap.cdap.spi.metadata.ScopedNameOfKind)2 SearchRequest (io.cdap.cdap.spi.metadata.SearchRequest)2