Search in sources :

Example 1 with ScopedNameOfKind

use of io.cdap.cdap.spi.metadata.ScopedNameOfKind in project cdap by caskdata.

the class DatasetMetadataStorage method replaceInScope.

private MetadataDataset.Change replaceInScope(MetadataDatasetContext context, MetadataScope scope, MetadataEntity entity, Set<String> newTags, Map<String, String> newProperties, Map<ScopedNameOfKind, MetadataDirective> directives) {
    MetadataDataset dataset = context.getDataset(scope);
    MetadataDataset.Record before = dataset.getMetadata(entity);
    if (newTags.isEmpty() && newProperties.isEmpty()) {
        // this scope remains unchanged
        return new MetadataDataset.Change(before, before);
    }
    Set<String> existingTags = before.getTags();
    Set<String> tagsToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == TAG && (entry.getValue() == MetadataDirective.KEEP || entry.getValue() == MetadataDirective.PRESERVE)).map(Map.Entry::getKey).map(ScopedName::getName).filter(existingTags::contains).collect(Collectors.toSet());
    newTags = Sets.union(newTags, tagsToKeepOrPreserve);
    Map<String, String> existingProperties = before.getProperties();
    Map<String, String> propertiesToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == PROPERTY).filter(entry -> existingProperties.containsKey(entry.getKey().getName())).filter(entry -> entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP && !newProperties.containsKey(entry.getKey().getName())).map(Map.Entry::getKey).map(ScopedName::getName).collect(Collectors.toMap(name -> name, existingProperties::get));
    newProperties.putAll(propertiesToKeepOrPreserve);
    Set<String> tagsToRemove = Sets.difference(before.getTags(), newTags);
    Set<String> tagsToAdd = Sets.difference(newTags, before.getTags());
    Set<String> propertiesToRemove = Sets.difference(before.getProperties().keySet(), newProperties.keySet());
    @SuppressWarnings("ConstantConditions") Map<String, String> propertiesToAdd = Maps.filterEntries(newProperties, entry -> !entry.getValue().equals(existingProperties.get(entry.getKey())));
    MetadataDataset.Record after = before;
    if (!tagsToRemove.isEmpty()) {
        after = dataset.removeTags(entity, tagsToRemove).getLatest();
    }
    if (!tagsToAdd.isEmpty()) {
        after = dataset.addTags(entity, tagsToAdd).getLatest();
    }
    if (!propertiesToRemove.isEmpty()) {
        after = dataset.removeProperties(entity, propertiesToRemove).getLatest();
    }
    if (!propertiesToAdd.isEmpty()) {
        after = dataset.addProperties(entity, propertiesToAdd).getLatest();
    }
    return new MetadataDataset.Change(before, after);
}
Also used : MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) ImmutablePair(io.cdap.cdap.common.utils.ImmutablePair) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Inject(com.google.inject.Inject) HashMap(java.util.HashMap) USER(io.cdap.cdap.api.metadata.MetadataScope.USER) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) HashSet(java.util.HashSet) TAG(io.cdap.cdap.spi.metadata.MetadataKind.TAG) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Metadata(io.cdap.cdap.spi.metadata.Metadata) Map(java.util.Map) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) Read(io.cdap.cdap.spi.metadata.Read) EnumSet(java.util.EnumSet) Sorting(io.cdap.cdap.spi.metadata.Sorting) SortInfo(io.cdap.cdap.data2.metadata.dataset.SortInfo) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) PROPERTY(io.cdap.cdap.spi.metadata.MetadataKind.PROPERTY) ImmutableMap(com.google.common.collect.ImmutableMap) Cursor(io.cdap.cdap.common.metadata.Cursor) SYSTEM(io.cdap.cdap.api.metadata.MetadataScope.SYSTEM) Set(java.util.Set) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) EntityScope(io.cdap.cdap.proto.EntityScope) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) Named(com.google.inject.name.Named) Constants(io.cdap.cdap.common.conf.Constants) VisibleForTesting(com.google.common.annotations.VisibleForTesting) DatasetDefinition(io.cdap.cdap.api.dataset.DatasetDefinition) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) MetadataDataset(io.cdap.cdap.data2.metadata.dataset.MetadataDataset) Collections(java.util.Collections) MetadataDataset(io.cdap.cdap.data2.metadata.dataset.MetadataDataset) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap)

Example 2 with ScopedNameOfKind

use of io.cdap.cdap.spi.metadata.ScopedNameOfKind in project cdap by caskdata.

the class DatasetMetadataStorage method readScope.

private MetadataDataset.Record readScope(MetadataDatasetContext context, MetadataScope scope, Read read) {
    MetadataEntity entity = read.getEntity();
    if (read.getSelection() == null && (!read.getScopes().contains(scope) || read.getKinds().isEmpty())) {
        return new MetadataDataset.Record(entity);
    }
    Set<ScopedNameOfKind> selectionForScope = null;
    if (read.getSelection() != null) {
        // noinspection ConstantConditions
        selectionForScope = Sets.filter(read.getSelection(), entry -> entry.getScope() == scope);
        if (selectionForScope.isEmpty()) {
            return new MetadataDataset.Record(entity);
        }
    }
    // now we know we must read from the dataset
    MetadataDataset dataset = context.getDataset(scope);
    if (selectionForScope != null) {
        // request is for a specific set of tags and properties
        Set<String> tagsToRead = selectionForScope.stream().filter(entry -> TAG == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
        Set<String> propertiesToRead = selectionForScope.stream().filter(entry -> PROPERTY == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
        Set<String> tags = tagsToRead.isEmpty() ? Collections.emptySet() : Sets.intersection(tagsToRead, dataset.getTags(entity));
        Map<String, String> properties = propertiesToRead.isEmpty() ? Collections.emptyMap() : Maps.filterKeys(dataset.getProperties(entity), propertiesToRead::contains);
        return new MetadataDataset.Record(entity, properties, tags);
    }
    if (MetadataKind.ALL.equals(read.getKinds())) {
        // all metadata kinds requested
        return dataset.getMetadata(entity);
    }
    // exactly one kind is requested
    MetadataKind requestKind = read.getKinds().iterator().next();
    if (requestKind == TAG) {
        return new MetadataDataset.Record(entity, Collections.emptyMap(), dataset.getTags(entity));
    }
    if (requestKind == PROPERTY) {
        return new MetadataDataset.Record(entity, dataset.getProperties(entity), Collections.emptySet());
    }
    throw new IllegalStateException("Encountered metadata read request for unknown kind " + requestKind);
}
Also used : MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) ImmutablePair(io.cdap.cdap.common.utils.ImmutablePair) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Inject(com.google.inject.Inject) HashMap(java.util.HashMap) USER(io.cdap.cdap.api.metadata.MetadataScope.USER) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) HashSet(java.util.HashSet) TAG(io.cdap.cdap.spi.metadata.MetadataKind.TAG) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Metadata(io.cdap.cdap.spi.metadata.Metadata) Map(java.util.Map) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) Read(io.cdap.cdap.spi.metadata.Read) EnumSet(java.util.EnumSet) Sorting(io.cdap.cdap.spi.metadata.Sorting) SortInfo(io.cdap.cdap.data2.metadata.dataset.SortInfo) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) PROPERTY(io.cdap.cdap.spi.metadata.MetadataKind.PROPERTY) ImmutableMap(com.google.common.collect.ImmutableMap) Cursor(io.cdap.cdap.common.metadata.Cursor) SYSTEM(io.cdap.cdap.api.metadata.MetadataScope.SYSTEM) Set(java.util.Set) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) EntityScope(io.cdap.cdap.proto.EntityScope) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) Named(com.google.inject.name.Named) Constants(io.cdap.cdap.common.conf.Constants) VisibleForTesting(com.google.common.annotations.VisibleForTesting) DatasetDefinition(io.cdap.cdap.api.dataset.DatasetDefinition) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) MetadataDataset(io.cdap.cdap.data2.metadata.dataset.MetadataDataset) Collections(java.util.Collections) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataDataset(io.cdap.cdap.data2.metadata.dataset.MetadataDataset) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind)

Example 3 with ScopedNameOfKind

use of io.cdap.cdap.spi.metadata.ScopedNameOfKind in project cdap by caskdata.

the class ElasticsearchMetadataStorageTest method testFiltering.

@Test
public void testFiltering() {
    ScopedName sys = new ScopedName(MetadataScope.SYSTEM, "s");
    ScopedName user = new ScopedName(MetadataScope.USER, "u");
    String sval = "S";
    String uval = "U";
    Metadata before = new Metadata(tags(sys, user), props(sys, sval, user, uval));
    // test selection to remove
    Assert.assertEquals(new Metadata(tags(sys), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.NONE, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test selection is not affected by scopes or kinds
    Assert.assertEquals(new Metadata(tags(sys), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.ALL, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test selection to keep
    Assert.assertEquals(new Metadata(tags(user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.NONE, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test selection is not affected by scopes or kinds
    Assert.assertEquals(new Metadata(tags(user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.ALL, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test removing nothing
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.NONE, null));
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.ALL, null));
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.NONE, null));
    // test keeping all
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.ALL, null));
    // test removing all
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.ALL, null));
    // test keeping nothing
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.NONE, null));
    // test keeping nothing
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.NONE, null));
    // test keeping nothing
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.ALL, null));
    // test removing all SYSTEM
    Assert.assertEquals(new Metadata(tags(user), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, Collections.singleton(MetadataScope.SYSTEM), null));
    // test removing all USER
    Assert.assertEquals(new Metadata(tags(sys), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, Collections.singleton(MetadataScope.USER), null));
    // test keeping all SYSTEM
    Assert.assertEquals(new Metadata(tags(sys), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, Collections.singleton(MetadataScope.SYSTEM), null));
    // test keeping all USER
    Assert.assertEquals(new Metadata(tags(user), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, Collections.singleton(MetadataScope.USER), null));
    // test removing all tags
    Assert.assertEquals(new Metadata(tags(), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.TAG), MetadataScope.ALL, null));
    // test removing all properties
    Assert.assertEquals(new Metadata(tags(sys, user), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.PROPERTY), MetadataScope.ALL, null));
    // test keeping all tags
    Assert.assertEquals(new Metadata(tags(sys, user), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.TAG), MetadataScope.ALL, null));
    // test keeping all properties
    Assert.assertEquals(new Metadata(tags(), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.PROPERTY), MetadataScope.ALL, null));
    // test removing all tags in SYSTEM scope
    Assert.assertEquals(new Metadata(tags(user), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.TAG), Collections.singleton(MetadataScope.SYSTEM), null));
    // test removing all properties in USER scope
    Assert.assertEquals(new Metadata(tags(sys, user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.PROPERTY), Collections.singleton(MetadataScope.USER), null));
    // test keeping all tags in SYSTEM scope
    Assert.assertEquals(new Metadata(tags(sys), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.TAG), Collections.singleton(MetadataScope.SYSTEM), null));
    // test keeping all properties in USER scope
    Assert.assertEquals(new Metadata(tags(), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.PROPERTY), Collections.singleton(MetadataScope.USER), null));
}
Also used : ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Metadata(io.cdap.cdap.spi.metadata.Metadata) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) Test(org.junit.Test)

Example 4 with ScopedNameOfKind

use of io.cdap.cdap.spi.metadata.ScopedNameOfKind in project cdap by caskdata.

the class ElasticsearchMetadataStorage method create.

/**
 * Creates the Elasticsearch index request for an entity creation or update.
 * See {@link MetadataMutation.Create} for detailed semantics.
 *
 * @param before the metadata for the mutation's entity before the change
 * @param create the mutation to apply
 *
 * @return an ElasticSearch request to be executed, and the change caused by the mutation
 */
private RequestAndChange create(VersionedMetadata before, MetadataMutation.Create create) {
    // if the entity did not exist before, none of the directives apply and this is equivalent to update()
    if (!before.existing()) {
        return update(create.getEntity(), before, create.getMetadata());
    }
    Metadata meta = create.getMetadata();
    Map<ScopedNameOfKind, MetadataDirective> directives = create.getDirectives();
    // determine the scopes that this mutation applies to (scopes that do not occur in the metadata are no changed)
    Set<MetadataScope> scopes = Stream.concat(meta.getTags().stream(), meta.getProperties().keySet().stream()).map(ScopedName::getScope).collect(Collectors.toSet());
    // compute what previously existing tags and properties have to be preserved (all others are replaced)
    Set<ScopedName> existingTagsToKeep = new HashSet<>();
    Map<ScopedName, String> existingPropertiesToKeep = new HashMap<>();
    // all tags and properties that are in a scope not affected by this mutation
    Sets.difference(MetadataScope.ALL, scopes).forEach(scope -> {
        before.getMetadata().getTags().stream().filter(tag -> tag.getScope().equals(scope)).forEach(existingTagsToKeep::add);
        before.getMetadata().getProperties().entrySet().stream().filter(entry -> entry.getKey().getScope().equals(scope)).forEach(entry -> existingPropertiesToKeep.put(entry.getKey(), entry.getValue()));
    });
    // tags and properties in affected scopes that must be kept or preserved
    directives.entrySet().stream().filter(entry -> scopes.contains(entry.getKey().getScope())).forEach(entry -> {
        ScopedNameOfKind key = entry.getKey();
        if (key.getKind() == MetadataKind.TAG && (entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP)) {
            ScopedName tag = new ScopedName(key.getScope(), key.getName());
            if (!meta.getTags().contains(tag) && before.getMetadata().getTags().contains(tag)) {
                existingTagsToKeep.add(tag);
            }
        } else if (key.getKind() == MetadataKind.PROPERTY) {
            ScopedName property = new ScopedName(key.getScope(), key.getName());
            String existingValue = before.getMetadata().getProperties().get(property);
            String newValue = meta.getProperties().get(property);
            if (existingValue != null && (entry.getValue() == MetadataDirective.PRESERVE && !existingValue.equals(newValue) || entry.getValue() == MetadataDirective.KEEP && newValue == null)) {
                existingPropertiesToKeep.put(property, existingValue);
            }
        }
    });
    // compute the new tags and properties
    Set<ScopedName> newTags = existingTagsToKeep.isEmpty() ? meta.getTags() : Sets.union(meta.getTags(), existingTagsToKeep);
    Map<ScopedName, String> newProperties = meta.getProperties();
    if (!existingPropertiesToKeep.isEmpty()) {
        newProperties = new HashMap<>(newProperties);
        newProperties.putAll(existingPropertiesToKeep);
    }
    Metadata after = new Metadata(newTags, newProperties);
    return new RequestAndChange(writeToIndex(create.getEntity(), before.getVersion(), after), new MetadataChange(create.getEntity(), before.getMetadata(), after));
}
Also used : Arrays(java.util.Arrays) SSLContext(javax.net.ssl.SSLContext) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) SearchHits(org.elasticsearch.search.SearchHits) GetMappingsRequest(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest) Bytes(io.cdap.cdap.api.common.Bytes) RetryStrategies(io.cdap.cdap.common.service.RetryStrategies) KeyStoreException(java.security.KeyStoreException) GsonBuilder(com.google.gson.GsonBuilder) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) DeleteRequest(org.elasticsearch.action.delete.DeleteRequest) IndexRequest(org.elasticsearch.action.index.IndexRequest) Map(java.util.Map) Closeables(com.google.common.io.Closeables) BulkResponse(org.elasticsearch.action.bulk.BulkResponse) Set(java.util.Set) KeyManagementException(java.security.KeyManagementException) ScoreMode(org.apache.lucene.search.join.ScoreMode) CreateIndexResponse(org.elasticsearch.action.admin.indices.create.CreateIndexResponse) Stream(java.util.stream.Stream) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataConstants(io.cdap.cdap.spi.metadata.MetadataConstants) RestStatus(org.elasticsearch.rest.RestStatus) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) CredentialsProvider(org.apache.http.client.CredentialsProvider) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) MatchQueryBuilder(org.elasticsearch.index.query.MatchQueryBuilder) ClearScrollRequest(org.elasticsearch.action.search.ClearScrollRequest) GetMappingsResponse(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse) Checksums(io.cdap.cdap.common.utils.Checksums) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) Read(io.cdap.cdap.spi.metadata.Read) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Resources(com.google.common.io.Resources) WildcardQueryBuilder(org.elasticsearch.index.query.WildcardQueryBuilder) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) DocWriteResponse(org.elasticsearch.action.DocWriteResponse) RestHighLevelClient(org.elasticsearch.client.RestHighLevelClient) SSLContextBuilder(org.apache.http.ssl.SSLContextBuilder) AuthScope(org.apache.http.auth.AuthScope) SearchScrollRequest(org.elasticsearch.action.search.SearchScrollRequest) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) HttpHost(org.apache.http.HttpHost) ActionListener(org.elasticsearch.action.ActionListener) GetResponse(org.elasticsearch.action.get.GetResponse) RestClientBuilder(org.elasticsearch.client.RestClientBuilder) MappingMetaData(org.elasticsearch.cluster.metadata.MappingMetaData) URL(java.net.URL) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) MetadataConflictException(io.cdap.cdap.common.metadata.MetadataConflictException) BasicCredentialsProvider(org.apache.http.impl.client.BasicCredentialsProvider) Settings(org.elasticsearch.common.settings.Settings) Gson(com.google.gson.Gson) Metadata(io.cdap.cdap.spi.metadata.Metadata) SearchResponse(org.elasticsearch.action.search.SearchResponse) RequestOptions(org.elasticsearch.client.RequestOptions) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) ElasticsearchStatusException(org.elasticsearch.ElasticsearchStatusException) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) Splitter(com.google.common.base.Splitter) Sorting(io.cdap.cdap.spi.metadata.Sorting) SearchHit(org.elasticsearch.search.SearchHit) GetRequest(org.elasticsearch.action.get.GetRequest) MultiGetResponse(org.elasticsearch.action.get.MultiGetResponse) ImmutableMap(com.google.common.collect.ImmutableMap) BulkItemResponse(org.elasticsearch.action.bulk.BulkItemResponse) MultiGetItemResponse(org.elasticsearch.action.get.MultiGetItemResponse) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) SortOrder(org.elasticsearch.search.sort.SortOrder) Pattern(java.util.regex.Pattern) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) RestClient(org.elasticsearch.client.RestClient) Retries(io.cdap.cdap.common.service.Retries) RetryStrategy(io.cdap.cdap.common.service.RetryStrategy) XContentType(org.elasticsearch.common.xcontent.XContentType) MultiGetRequest(org.elasticsearch.action.get.MultiGetRequest) HashMap(java.util.HashMap) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) HashSet(java.util.HashSet) WriteRequest(org.elasticsearch.action.support.WriteRequest) Operator(org.elasticsearch.index.query.Operator) CreateIndexRequest(org.elasticsearch.action.admin.indices.create.CreateIndexRequest) GetIndexResponse(org.elasticsearch.action.admin.indices.get.GetIndexResponse) DeleteIndexRequest(org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest) Logger(org.slf4j.Logger) Cursor(io.cdap.cdap.common.metadata.Cursor) GetIndexRequest(org.elasticsearch.action.admin.indices.get.GetIndexRequest) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) ProjectInfo(io.cdap.cdap.common.utils.ProjectInfo) DocWriteRequest(org.elasticsearch.action.DocWriteRequest) AcknowledgedResponse(org.elasticsearch.action.support.master.AcknowledgedResponse) Maps(com.google.common.collect.Maps) TermQueryBuilder(org.elasticsearch.index.query.TermQueryBuilder) UsernamePasswordCredentials(org.apache.http.auth.UsernamePasswordCredentials) TimeUnit(java.util.concurrent.TimeUnit) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) BulkRequest(org.elasticsearch.action.bulk.BulkRequest) Collections(java.util.Collections) MetadataUtil(io.cdap.cdap.common.metadata.MetadataUtil) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) Metadata(io.cdap.cdap.spi.metadata.Metadata) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) HashSet(java.util.HashSet)

Example 5 with ScopedNameOfKind

use of io.cdap.cdap.spi.metadata.ScopedNameOfKind in project cdap by caskdata.

the class DatasetMetadataStorageTest method testSearchWeight.

// this tests is not in MetadataStorageTest,
// because it tests result scoring and sorting specific to the dataset-based implementation
@Test
public void testSearchWeight() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    String ns = "ns1";
    NamespaceId nsId = new NamespaceId(ns);
    MetadataEntity service1 = nsId.app("app1").service("service1").toMetadataEntity();
    MetadataEntity dataset1 = nsId.dataset("ds1").toMetadataEntity();
    MetadataEntity dataset2 = nsId.dataset("ds2").toMetadataEntity();
    // Add metadata
    String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    Map<String, String> userProps = ImmutableMap.of("key1", "value1", "key2", "value2", "multiword", multiWordValue);
    Map<String, String> systemProps = ImmutableMap.of("sysKey1", "sysValue1");
    Set<String> userTags = ImmutableSet.of("tag1", "tag2");
    Set<String> temporaryUserTags = ImmutableSet.of("tag3", "tag4");
    Map<String, String> dataset1UserProps = ImmutableMap.of("sKey1", "sValuee1 sValuee2");
    Map<String, String> dataset2UserProps = ImmutableMap.of("sKey1", "sValue1 sValue2", "Key1", "Value1");
    Set<String> sysTags = ImmutableSet.of("sysTag1");
    MetadataRecord service1Record = new MetadataRecord(service1, union(new Metadata(USER, userTags, userProps), new Metadata(SYSTEM, sysTags, systemProps)));
    mds.apply(new Update(service1Record.getEntity(), service1Record.getMetadata()), MutationOptions.DEFAULT);
    // dd and then remove some metadata for dataset2
    mds.apply(new Update(dataset2, new Metadata(USER, temporaryUserTags, userProps)), MutationOptions.DEFAULT);
    mds.apply(new Remove(dataset2, temporaryUserTags.stream().map(tag -> new ScopedNameOfKind(TAG, USER, tag)).collect(Collectors.toSet())), MutationOptions.DEFAULT);
    mds.apply(new Remove(dataset2, userProps.keySet().stream().map(tag -> new ScopedNameOfKind(PROPERTY, USER, tag)).collect(Collectors.toSet())), MutationOptions.DEFAULT);
    MetadataRecord dataset1Record = new MetadataRecord(dataset1, new Metadata(USER, tags(), dataset1UserProps));
    MetadataRecord dataset2Record = new MetadataRecord(dataset2, new Metadata(USER, tags(), dataset2UserProps));
    mds.batch(ImmutableList.of(new Update(dataset1Record.getEntity(), dataset1Record.getMetadata()), new Update(dataset2Record.getEntity(), dataset2Record.getMetadata())), MutationOptions.DEFAULT);
    // Test score and metadata match
    assertInOrder(mds, SearchRequest.of("value1 multiword:av2").addNamespace(ns).build(), service1Record, dataset2Record);
    assertInOrder(mds, SearchRequest.of("value1 sValue*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), dataset2Record, dataset1Record, service1Record);
    assertResults(mds, SearchRequest.of("*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), dataset2Record, dataset1Record, service1Record);
    // clean up
    mds.batch(ImmutableList.of(new Drop(service1), new Drop(dataset1), new Drop(dataset2)), MutationOptions.DEFAULT);
}
Also used : Module(com.google.inject.Module) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) ImmutablePair(io.cdap.cdap.common.utils.ImmutablePair) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) SystemDatasetRuntimeModule(io.cdap.cdap.data.runtime.SystemDatasetRuntimeModule) Metadata(io.cdap.cdap.spi.metadata.Metadata) Map(java.util.Map) Closeables(com.google.common.io.Closeables) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) NoOpMetricsCollectionService(io.cdap.cdap.common.metrics.NoOpMetricsCollectionService) EnumSet(java.util.EnumSet) PROPERTY(io.cdap.cdap.spi.metadata.MetadataKind.PROPERTY) AfterClass(org.junit.AfterClass) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) SYSTEM(io.cdap.cdap.api.metadata.MetadataScope.SYSTEM) Set(java.util.Set) Collectors(java.util.stream.Collectors) List(java.util.List) Drop(io.cdap.cdap.spi.metadata.MetadataMutation.Drop) BeforeClass(org.junit.BeforeClass) LocalLocationModule(io.cdap.cdap.common.guice.LocalLocationModule) AuthorizationTestModule(io.cdap.cdap.security.authorization.AuthorizationTestModule) USER(io.cdap.cdap.api.metadata.MetadataScope.USER) NamespaceAdminTestModule(io.cdap.cdap.common.guice.NamespaceAdminTestModule) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) TAG(io.cdap.cdap.spi.metadata.MetadataKind.TAG) AuthorizationEnforcementModule(io.cdap.cdap.security.authorization.AuthorizationEnforcementModule) ImmutableList(com.google.common.collect.ImmutableList) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) TransactionInMemoryModule(org.apache.tephra.runtime.TransactionInMemoryModule) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) StorageModule(io.cdap.cdap.data.runtime.StorageModule) TransactionManager(org.apache.tephra.TransactionManager) Iterator(java.util.Iterator) Cursor(io.cdap.cdap.common.metadata.Cursor) Update(io.cdap.cdap.spi.metadata.MetadataMutation.Update) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) Test(org.junit.Test) IOException(java.io.IOException) Scopes(com.google.inject.Scopes) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) Injector(com.google.inject.Injector) EntityScope(io.cdap.cdap.proto.EntityScope) Guice(com.google.inject.Guice) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) Assert(org.junit.Assert) Remove(io.cdap.cdap.spi.metadata.MetadataMutation.Remove) Collections(java.util.Collections) AuthenticationContextModules(io.cdap.cdap.security.auth.context.AuthenticationContextModules) AbstractModule(com.google.inject.AbstractModule) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Metadata(io.cdap.cdap.spi.metadata.Metadata) Remove(io.cdap.cdap.spi.metadata.MetadataMutation.Remove) Update(io.cdap.cdap.spi.metadata.MetadataMutation.Update) Drop(io.cdap.cdap.spi.metadata.MetadataMutation.Drop) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) Test(org.junit.Test)

Aggregations

ScopedNameOfKind (io.cdap.cdap.spi.metadata.ScopedNameOfKind)6 Metadata (io.cdap.cdap.spi.metadata.Metadata)5 VisibleForTesting (com.google.common.annotations.VisibleForTesting)4 ImmutableMap (com.google.common.collect.ImmutableMap)4 MetadataEntity (io.cdap.cdap.api.metadata.MetadataEntity)4 Cursor (io.cdap.cdap.common.metadata.Cursor)4 MetadataMutation (io.cdap.cdap.spi.metadata.MetadataMutation)4 MetadataRecord (io.cdap.cdap.spi.metadata.MetadataRecord)4 MetadataStorage (io.cdap.cdap.spi.metadata.MetadataStorage)4 MutationOptions (io.cdap.cdap.spi.metadata.MutationOptions)4 SearchRequest (io.cdap.cdap.spi.metadata.SearchRequest)4 IOException (java.io.IOException)4 Collections (java.util.Collections)4 Maps (com.google.common.collect.Maps)3 Sets (com.google.common.collect.Sets)3 Inject (com.google.inject.Inject)3 MetadataScope (io.cdap.cdap.api.metadata.MetadataScope)3 SYSTEM (io.cdap.cdap.api.metadata.MetadataScope.SYSTEM)3 USER (io.cdap.cdap.api.metadata.MetadataScope.USER)3 ScopedName (io.cdap.cdap.spi.metadata.ScopedName)3