Search in sources :

Example 6 with ScopedName

use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by cdapio.

the class ElasticsearchMetadataStorage method create.

/**
 * Creates the Elasticsearch index request for an entity creation or update.
 * See {@link MetadataMutation.Create} for detailed semantics.
 *
 * @param before the metadata for the mutation's entity before the change
 * @param create the mutation to apply
 *
 * @return an ElasticSearch request to be executed, and the change caused by the mutation
 */
private RequestAndChange create(VersionedMetadata before, MetadataMutation.Create create) {
    // if the entity did not exist before, none of the directives apply and this is equivalent to update()
    if (!before.existing()) {
        return update(create.getEntity(), before, create.getMetadata());
    }
    Metadata meta = create.getMetadata();
    Map<ScopedNameOfKind, MetadataDirective> directives = create.getDirectives();
    // determine the scopes that this mutation applies to (scopes that do not occur in the metadata are no changed)
    Set<MetadataScope> scopes = Stream.concat(meta.getTags().stream(), meta.getProperties().keySet().stream()).map(ScopedName::getScope).collect(Collectors.toSet());
    // compute what previously existing tags and properties have to be preserved (all others are replaced)
    Set<ScopedName> existingTagsToKeep = new HashSet<>();
    Map<ScopedName, String> existingPropertiesToKeep = new HashMap<>();
    // all tags and properties that are in a scope not affected by this mutation
    Sets.difference(MetadataScope.ALL, scopes).forEach(scope -> {
        before.getMetadata().getTags().stream().filter(tag -> tag.getScope().equals(scope)).forEach(existingTagsToKeep::add);
        before.getMetadata().getProperties().entrySet().stream().filter(entry -> entry.getKey().getScope().equals(scope)).forEach(entry -> existingPropertiesToKeep.put(entry.getKey(), entry.getValue()));
    });
    // tags and properties in affected scopes that must be kept or preserved
    directives.entrySet().stream().filter(entry -> scopes.contains(entry.getKey().getScope())).forEach(entry -> {
        ScopedNameOfKind key = entry.getKey();
        if (key.getKind() == MetadataKind.TAG && (entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP)) {
            ScopedName tag = new ScopedName(key.getScope(), key.getName());
            if (!meta.getTags().contains(tag) && before.getMetadata().getTags().contains(tag)) {
                existingTagsToKeep.add(tag);
            }
        } else if (key.getKind() == MetadataKind.PROPERTY) {
            ScopedName property = new ScopedName(key.getScope(), key.getName());
            String existingValue = before.getMetadata().getProperties().get(property);
            String newValue = meta.getProperties().get(property);
            if (existingValue != null && (entry.getValue() == MetadataDirective.PRESERVE && !existingValue.equals(newValue) || entry.getValue() == MetadataDirective.KEEP && newValue == null)) {
                existingPropertiesToKeep.put(property, existingValue);
            }
        }
    });
    // compute the new tags and properties
    Set<ScopedName> newTags = existingTagsToKeep.isEmpty() ? meta.getTags() : Sets.union(meta.getTags(), existingTagsToKeep);
    Map<ScopedName, String> newProperties = meta.getProperties();
    if (!existingPropertiesToKeep.isEmpty()) {
        newProperties = new HashMap<>(newProperties);
        newProperties.putAll(existingPropertiesToKeep);
    }
    Metadata after = new Metadata(newTags, newProperties);
    return new RequestAndChange(writeToIndex(create.getEntity(), before.getVersion(), after), new MetadataChange(create.getEntity(), before.getMetadata(), after));
}
Also used : Arrays(java.util.Arrays) SSLContext(javax.net.ssl.SSLContext) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) SearchHits(org.elasticsearch.search.SearchHits) GetMappingsRequest(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest) Bytes(io.cdap.cdap.api.common.Bytes) RetryStrategies(io.cdap.cdap.common.service.RetryStrategies) KeyStoreException(java.security.KeyStoreException) GsonBuilder(com.google.gson.GsonBuilder) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) DeleteRequest(org.elasticsearch.action.delete.DeleteRequest) IndexRequest(org.elasticsearch.action.index.IndexRequest) Map(java.util.Map) Closeables(com.google.common.io.Closeables) BulkResponse(org.elasticsearch.action.bulk.BulkResponse) Set(java.util.Set) KeyManagementException(java.security.KeyManagementException) ScoreMode(org.apache.lucene.search.join.ScoreMode) CreateIndexResponse(org.elasticsearch.action.admin.indices.create.CreateIndexResponse) Stream(java.util.stream.Stream) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataConstants(io.cdap.cdap.spi.metadata.MetadataConstants) RestStatus(org.elasticsearch.rest.RestStatus) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) CredentialsProvider(org.apache.http.client.CredentialsProvider) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) MatchQueryBuilder(org.elasticsearch.index.query.MatchQueryBuilder) ClearScrollRequest(org.elasticsearch.action.search.ClearScrollRequest) GetMappingsResponse(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse) Checksums(io.cdap.cdap.common.utils.Checksums) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) Read(io.cdap.cdap.spi.metadata.Read) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Resources(com.google.common.io.Resources) WildcardQueryBuilder(org.elasticsearch.index.query.WildcardQueryBuilder) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) DocWriteResponse(org.elasticsearch.action.DocWriteResponse) RestHighLevelClient(org.elasticsearch.client.RestHighLevelClient) SSLContextBuilder(org.apache.http.ssl.SSLContextBuilder) AuthScope(org.apache.http.auth.AuthScope) SearchScrollRequest(org.elasticsearch.action.search.SearchScrollRequest) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) HttpHost(org.apache.http.HttpHost) ActionListener(org.elasticsearch.action.ActionListener) GetResponse(org.elasticsearch.action.get.GetResponse) RestClientBuilder(org.elasticsearch.client.RestClientBuilder) MappingMetaData(org.elasticsearch.cluster.metadata.MappingMetaData) URL(java.net.URL) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) MetadataConflictException(io.cdap.cdap.common.metadata.MetadataConflictException) BasicCredentialsProvider(org.apache.http.impl.client.BasicCredentialsProvider) Settings(org.elasticsearch.common.settings.Settings) Gson(com.google.gson.Gson) Metadata(io.cdap.cdap.spi.metadata.Metadata) SearchResponse(org.elasticsearch.action.search.SearchResponse) RequestOptions(org.elasticsearch.client.RequestOptions) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) ElasticsearchStatusException(org.elasticsearch.ElasticsearchStatusException) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) Splitter(com.google.common.base.Splitter) Sorting(io.cdap.cdap.spi.metadata.Sorting) SearchHit(org.elasticsearch.search.SearchHit) GetRequest(org.elasticsearch.action.get.GetRequest) MultiGetResponse(org.elasticsearch.action.get.MultiGetResponse) ImmutableMap(com.google.common.collect.ImmutableMap) BulkItemResponse(org.elasticsearch.action.bulk.BulkItemResponse) MultiGetItemResponse(org.elasticsearch.action.get.MultiGetItemResponse) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) SortOrder(org.elasticsearch.search.sort.SortOrder) Pattern(java.util.regex.Pattern) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) RestClient(org.elasticsearch.client.RestClient) Retries(io.cdap.cdap.common.service.Retries) RetryStrategy(io.cdap.cdap.common.service.RetryStrategy) XContentType(org.elasticsearch.common.xcontent.XContentType) MultiGetRequest(org.elasticsearch.action.get.MultiGetRequest) HashMap(java.util.HashMap) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) HashSet(java.util.HashSet) WriteRequest(org.elasticsearch.action.support.WriteRequest) Operator(org.elasticsearch.index.query.Operator) CreateIndexRequest(org.elasticsearch.action.admin.indices.create.CreateIndexRequest) GetIndexResponse(org.elasticsearch.action.admin.indices.get.GetIndexResponse) DeleteIndexRequest(org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest) Logger(org.slf4j.Logger) Cursor(io.cdap.cdap.common.metadata.Cursor) GetIndexRequest(org.elasticsearch.action.admin.indices.get.GetIndexRequest) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) ProjectInfo(io.cdap.cdap.common.utils.ProjectInfo) DocWriteRequest(org.elasticsearch.action.DocWriteRequest) AcknowledgedResponse(org.elasticsearch.action.support.master.AcknowledgedResponse) Maps(com.google.common.collect.Maps) TermQueryBuilder(org.elasticsearch.index.query.TermQueryBuilder) UsernamePasswordCredentials(org.apache.http.auth.UsernamePasswordCredentials) TimeUnit(java.util.concurrent.TimeUnit) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) BulkRequest(org.elasticsearch.action.bulk.BulkRequest) Collections(java.util.Collections) MetadataUtil(io.cdap.cdap.common.metadata.MetadataUtil) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) Metadata(io.cdap.cdap.spi.metadata.Metadata) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) HashSet(java.util.HashSet)

Example 7 with ScopedName

use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by cdapio.

the class ElasticsearchMetadataStorageTest method testFiltering.

@Test
public void testFiltering() {
    ScopedName sys = new ScopedName(MetadataScope.SYSTEM, "s");
    ScopedName user = new ScopedName(MetadataScope.USER, "u");
    String sval = "S";
    String uval = "U";
    Metadata before = new Metadata(tags(sys, user), props(sys, sval, user, uval));
    // test selection to remove
    Assert.assertEquals(new Metadata(tags(sys), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.NONE, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test selection is not affected by scopes or kinds
    Assert.assertEquals(new Metadata(tags(sys), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.ALL, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test selection to keep
    Assert.assertEquals(new Metadata(tags(user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.NONE, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test selection is not affected by scopes or kinds
    Assert.assertEquals(new Metadata(tags(user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.ALL, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test removing nothing
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.NONE, null));
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.ALL, null));
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.NONE, null));
    // test keeping all
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.ALL, null));
    // test removing all
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.ALL, null));
    // test keeping nothing
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.NONE, null));
    // test keeping nothing
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.NONE, null));
    // test keeping nothing
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.ALL, null));
    // test removing all SYSTEM
    Assert.assertEquals(new Metadata(tags(user), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, Collections.singleton(MetadataScope.SYSTEM), null));
    // test removing all USER
    Assert.assertEquals(new Metadata(tags(sys), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, Collections.singleton(MetadataScope.USER), null));
    // test keeping all SYSTEM
    Assert.assertEquals(new Metadata(tags(sys), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, Collections.singleton(MetadataScope.SYSTEM), null));
    // test keeping all USER
    Assert.assertEquals(new Metadata(tags(user), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, Collections.singleton(MetadataScope.USER), null));
    // test removing all tags
    Assert.assertEquals(new Metadata(tags(), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.TAG), MetadataScope.ALL, null));
    // test removing all properties
    Assert.assertEquals(new Metadata(tags(sys, user), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.PROPERTY), MetadataScope.ALL, null));
    // test keeping all tags
    Assert.assertEquals(new Metadata(tags(sys, user), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.TAG), MetadataScope.ALL, null));
    // test keeping all properties
    Assert.assertEquals(new Metadata(tags(), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.PROPERTY), MetadataScope.ALL, null));
    // test removing all tags in SYSTEM scope
    Assert.assertEquals(new Metadata(tags(user), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.TAG), Collections.singleton(MetadataScope.SYSTEM), null));
    // test removing all properties in USER scope
    Assert.assertEquals(new Metadata(tags(sys, user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.PROPERTY), Collections.singleton(MetadataScope.USER), null));
    // test keeping all tags in SYSTEM scope
    Assert.assertEquals(new Metadata(tags(sys), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.TAG), Collections.singleton(MetadataScope.SYSTEM), null));
    // test keeping all properties in USER scope
    Assert.assertEquals(new Metadata(tags(), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.PROPERTY), Collections.singleton(MetadataScope.USER), null));
}
Also used : ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Metadata(io.cdap.cdap.spi.metadata.Metadata) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) Test(org.junit.Test)

Example 8 with ScopedName

use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.

the class ElasticsearchMetadataStorage method create.

/**
 * Creates the Elasticsearch index request for an entity creation or update.
 * See {@link MetadataMutation.Create} for detailed semantics.
 *
 * @param before the metadata for the mutation's entity before the change
 * @param create the mutation to apply
 *
 * @return an ElasticSearch request to be executed, and the change caused by the mutation
 */
private RequestAndChange create(VersionedMetadata before, MetadataMutation.Create create) {
    // if the entity did not exist before, none of the directives apply and this is equivalent to update()
    if (!before.existing()) {
        return update(create.getEntity(), before, create.getMetadata());
    }
    Metadata meta = create.getMetadata();
    Map<ScopedNameOfKind, MetadataDirective> directives = create.getDirectives();
    // determine the scopes that this mutation applies to (scopes that do not occur in the metadata are no changed)
    Set<MetadataScope> scopes = Stream.concat(meta.getTags().stream(), meta.getProperties().keySet().stream()).map(ScopedName::getScope).collect(Collectors.toSet());
    // compute what previously existing tags and properties have to be preserved (all others are replaced)
    Set<ScopedName> existingTagsToKeep = new HashSet<>();
    Map<ScopedName, String> existingPropertiesToKeep = new HashMap<>();
    // all tags and properties that are in a scope not affected by this mutation
    Sets.difference(MetadataScope.ALL, scopes).forEach(scope -> {
        before.getMetadata().getTags().stream().filter(tag -> tag.getScope().equals(scope)).forEach(existingTagsToKeep::add);
        before.getMetadata().getProperties().entrySet().stream().filter(entry -> entry.getKey().getScope().equals(scope)).forEach(entry -> existingPropertiesToKeep.put(entry.getKey(), entry.getValue()));
    });
    // tags and properties in affected scopes that must be kept or preserved
    directives.entrySet().stream().filter(entry -> scopes.contains(entry.getKey().getScope())).forEach(entry -> {
        ScopedNameOfKind key = entry.getKey();
        if (key.getKind() == MetadataKind.TAG && (entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP)) {
            ScopedName tag = new ScopedName(key.getScope(), key.getName());
            if (!meta.getTags().contains(tag) && before.getMetadata().getTags().contains(tag)) {
                existingTagsToKeep.add(tag);
            }
        } else if (key.getKind() == MetadataKind.PROPERTY) {
            ScopedName property = new ScopedName(key.getScope(), key.getName());
            String existingValue = before.getMetadata().getProperties().get(property);
            String newValue = meta.getProperties().get(property);
            if (existingValue != null && (entry.getValue() == MetadataDirective.PRESERVE && !existingValue.equals(newValue) || entry.getValue() == MetadataDirective.KEEP && newValue == null)) {
                existingPropertiesToKeep.put(property, existingValue);
            }
        }
    });
    // compute the new tags and properties
    Set<ScopedName> newTags = existingTagsToKeep.isEmpty() ? meta.getTags() : Sets.union(meta.getTags(), existingTagsToKeep);
    Map<ScopedName, String> newProperties = meta.getProperties();
    if (!existingPropertiesToKeep.isEmpty()) {
        newProperties = new HashMap<>(newProperties);
        newProperties.putAll(existingPropertiesToKeep);
    }
    Metadata after = new Metadata(newTags, newProperties);
    return new RequestAndChange(writeToIndex(create.getEntity(), before.getVersion(), after), new MetadataChange(create.getEntity(), before.getMetadata(), after));
}
Also used : Arrays(java.util.Arrays) SSLContext(javax.net.ssl.SSLContext) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) SearchHits(org.elasticsearch.search.SearchHits) GetMappingsRequest(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest) Bytes(io.cdap.cdap.api.common.Bytes) RetryStrategies(io.cdap.cdap.common.service.RetryStrategies) KeyStoreException(java.security.KeyStoreException) GsonBuilder(com.google.gson.GsonBuilder) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) DeleteRequest(org.elasticsearch.action.delete.DeleteRequest) IndexRequest(org.elasticsearch.action.index.IndexRequest) Map(java.util.Map) Closeables(com.google.common.io.Closeables) BulkResponse(org.elasticsearch.action.bulk.BulkResponse) Set(java.util.Set) KeyManagementException(java.security.KeyManagementException) ScoreMode(org.apache.lucene.search.join.ScoreMode) CreateIndexResponse(org.elasticsearch.action.admin.indices.create.CreateIndexResponse) Stream(java.util.stream.Stream) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataConstants(io.cdap.cdap.spi.metadata.MetadataConstants) RestStatus(org.elasticsearch.rest.RestStatus) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) CredentialsProvider(org.apache.http.client.CredentialsProvider) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) MatchQueryBuilder(org.elasticsearch.index.query.MatchQueryBuilder) ClearScrollRequest(org.elasticsearch.action.search.ClearScrollRequest) GetMappingsResponse(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse) Checksums(io.cdap.cdap.common.utils.Checksums) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) Read(io.cdap.cdap.spi.metadata.Read) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Resources(com.google.common.io.Resources) WildcardQueryBuilder(org.elasticsearch.index.query.WildcardQueryBuilder) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) DocWriteResponse(org.elasticsearch.action.DocWriteResponse) RestHighLevelClient(org.elasticsearch.client.RestHighLevelClient) SSLContextBuilder(org.apache.http.ssl.SSLContextBuilder) AuthScope(org.apache.http.auth.AuthScope) SearchScrollRequest(org.elasticsearch.action.search.SearchScrollRequest) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) HttpHost(org.apache.http.HttpHost) ActionListener(org.elasticsearch.action.ActionListener) GetResponse(org.elasticsearch.action.get.GetResponse) RestClientBuilder(org.elasticsearch.client.RestClientBuilder) MappingMetaData(org.elasticsearch.cluster.metadata.MappingMetaData) URL(java.net.URL) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) MetadataConflictException(io.cdap.cdap.common.metadata.MetadataConflictException) BasicCredentialsProvider(org.apache.http.impl.client.BasicCredentialsProvider) Settings(org.elasticsearch.common.settings.Settings) Gson(com.google.gson.Gson) Metadata(io.cdap.cdap.spi.metadata.Metadata) SearchResponse(org.elasticsearch.action.search.SearchResponse) RequestOptions(org.elasticsearch.client.RequestOptions) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) ElasticsearchStatusException(org.elasticsearch.ElasticsearchStatusException) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) Splitter(com.google.common.base.Splitter) Sorting(io.cdap.cdap.spi.metadata.Sorting) SearchHit(org.elasticsearch.search.SearchHit) GetRequest(org.elasticsearch.action.get.GetRequest) MultiGetResponse(org.elasticsearch.action.get.MultiGetResponse) ImmutableMap(com.google.common.collect.ImmutableMap) BulkItemResponse(org.elasticsearch.action.bulk.BulkItemResponse) MultiGetItemResponse(org.elasticsearch.action.get.MultiGetItemResponse) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) SortOrder(org.elasticsearch.search.sort.SortOrder) Pattern(java.util.regex.Pattern) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) RestClient(org.elasticsearch.client.RestClient) Retries(io.cdap.cdap.common.service.Retries) RetryStrategy(io.cdap.cdap.common.service.RetryStrategy) XContentType(org.elasticsearch.common.xcontent.XContentType) MultiGetRequest(org.elasticsearch.action.get.MultiGetRequest) HashMap(java.util.HashMap) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) HashSet(java.util.HashSet) WriteRequest(org.elasticsearch.action.support.WriteRequest) Operator(org.elasticsearch.index.query.Operator) CreateIndexRequest(org.elasticsearch.action.admin.indices.create.CreateIndexRequest) GetIndexResponse(org.elasticsearch.action.admin.indices.get.GetIndexResponse) DeleteIndexRequest(org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest) Logger(org.slf4j.Logger) Cursor(io.cdap.cdap.common.metadata.Cursor) GetIndexRequest(org.elasticsearch.action.admin.indices.get.GetIndexRequest) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) ProjectInfo(io.cdap.cdap.common.utils.ProjectInfo) DocWriteRequest(org.elasticsearch.action.DocWriteRequest) AcknowledgedResponse(org.elasticsearch.action.support.master.AcknowledgedResponse) Maps(com.google.common.collect.Maps) TermQueryBuilder(org.elasticsearch.index.query.TermQueryBuilder) UsernamePasswordCredentials(org.apache.http.auth.UsernamePasswordCredentials) TimeUnit(java.util.concurrent.TimeUnit) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) BulkRequest(org.elasticsearch.action.bulk.BulkRequest) Collections(java.util.Collections) MetadataUtil(io.cdap.cdap.common.metadata.MetadataUtil) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) Metadata(io.cdap.cdap.spi.metadata.Metadata) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) HashSet(java.util.HashSet)

Example 9 with ScopedName

use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.

the class ElasticsearchMetadataStorage method update.

/**
 * Creates the Elasticsearch index request for updating the metadata of an entity.
 * This updates or adds the new metadata to the corresponding metadata document in the index.
 *
 * @param before the metadata for the mutation's entity before the change, and its version
 *
 * @return an ElasticSearch request to be executed, and the change caused by the mutation
 */
private RequestAndChange update(MetadataEntity entity, VersionedMetadata before, Metadata updates) {
    Set<ScopedName> tags = new HashSet<>(before.getMetadata().getTags());
    tags.addAll(updates.getTags());
    Map<ScopedName, String> properties = new HashMap<>(before.getMetadata().getProperties());
    properties.putAll(updates.getProperties());
    Metadata after = new Metadata(tags, properties);
    return new RequestAndChange(writeToIndex(entity, before.getVersion(), after), new MetadataChange(entity, before.getMetadata(), after));
}
Also used : MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) Metadata(io.cdap.cdap.spi.metadata.Metadata) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) HashSet(java.util.HashSet)

Example 10 with ScopedName

use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.

the class DefaultMetadataServiceClientTest method testUpdate.

@Test
public void testUpdate() throws Exception {
    final MetadataEntity updateEntity = MetadataEntity.builder().append("update", "test").build();
    createMetadataMutation(new MetadataMutation.Create(updateEntity, testMetadata, CREATE_DIRECTIVES));
    // confirm that the create was successful
    Assert.assertEquals(testMetadata.getProperties(MetadataScope.SYSTEM), getMetadataProperties(updateEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(testMetadata.getTags(MetadataScope.SYSTEM), getMetadataTags(updateEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(testMetadata.getProperties(MetadataScope.USER), getMetadataProperties(updateEntity, MetadataScope.USER));
    Assert.assertEquals(testMetadata.getTags(MetadataScope.USER), getMetadataTags(updateEntity, MetadataScope.USER));
    Metadata update = new Metadata(Collections.EMPTY_SET, ImmutableMap.of(new ScopedName(MetadataScope.SYSTEM, "x"), "10", new ScopedName(MetadataScope.SYSTEM, "y"), "11", new ScopedName(MetadataScope.USER, "z"), "4"));
    updateMetadataMutation(new MetadataMutation.Update(updateEntity, update));
    Assert.assertEquals(update.getProperties(MetadataScope.SYSTEM), getMetadataProperties(updateEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(testMetadata.getTags(MetadataScope.SYSTEM), getMetadataTags(updateEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(update.getProperties(MetadataScope.USER), getMetadataProperties(updateEntity, MetadataScope.USER));
    Assert.assertEquals(testMetadata.getTags(MetadataScope.USER), getMetadataTags(updateEntity, MetadataScope.USER));
}
Also used : MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) Metadata(io.cdap.cdap.spi.metadata.Metadata) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) Test(org.junit.Test)

Aggregations

Metadata (io.cdap.cdap.spi.metadata.Metadata)14 ScopedName (io.cdap.cdap.spi.metadata.ScopedName)14 MetadataEntity (io.cdap.cdap.api.metadata.MetadataEntity)10 MetadataMutation (io.cdap.cdap.spi.metadata.MetadataMutation)10 MetadataChange (io.cdap.cdap.spi.metadata.MetadataChange)8 ScopedNameOfKind (io.cdap.cdap.spi.metadata.ScopedNameOfKind)8 HashMap (java.util.HashMap)8 HashSet (java.util.HashSet)8 VisibleForTesting (com.google.common.annotations.VisibleForTesting)6 ImmutableMap (com.google.common.collect.ImmutableMap)6 Maps (com.google.common.collect.Maps)6 Sets (com.google.common.collect.Sets)6 Inject (com.google.inject.Inject)6 MetadataScope (io.cdap.cdap.api.metadata.MetadataScope)6 Cursor (io.cdap.cdap.common.metadata.Cursor)6 MetadataDirective (io.cdap.cdap.spi.metadata.MetadataDirective)6 MetadataKind (io.cdap.cdap.spi.metadata.MetadataKind)6 MetadataRecord (io.cdap.cdap.spi.metadata.MetadataRecord)6 MetadataStorage (io.cdap.cdap.spi.metadata.MetadataStorage)6 MutationOptions (io.cdap.cdap.spi.metadata.MutationOptions)6