use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by cdapio.
the class ElasticsearchMetadataStorage method create.
/**
* Creates the Elasticsearch index request for an entity creation or update.
* See {@link MetadataMutation.Create} for detailed semantics.
*
* @param before the metadata for the mutation's entity before the change
* @param create the mutation to apply
*
* @return an ElasticSearch request to be executed, and the change caused by the mutation
*/
private RequestAndChange create(VersionedMetadata before, MetadataMutation.Create create) {
// if the entity did not exist before, none of the directives apply and this is equivalent to update()
if (!before.existing()) {
return update(create.getEntity(), before, create.getMetadata());
}
Metadata meta = create.getMetadata();
Map<ScopedNameOfKind, MetadataDirective> directives = create.getDirectives();
// determine the scopes that this mutation applies to (scopes that do not occur in the metadata are no changed)
Set<MetadataScope> scopes = Stream.concat(meta.getTags().stream(), meta.getProperties().keySet().stream()).map(ScopedName::getScope).collect(Collectors.toSet());
// compute what previously existing tags and properties have to be preserved (all others are replaced)
Set<ScopedName> existingTagsToKeep = new HashSet<>();
Map<ScopedName, String> existingPropertiesToKeep = new HashMap<>();
// all tags and properties that are in a scope not affected by this mutation
Sets.difference(MetadataScope.ALL, scopes).forEach(scope -> {
before.getMetadata().getTags().stream().filter(tag -> tag.getScope().equals(scope)).forEach(existingTagsToKeep::add);
before.getMetadata().getProperties().entrySet().stream().filter(entry -> entry.getKey().getScope().equals(scope)).forEach(entry -> existingPropertiesToKeep.put(entry.getKey(), entry.getValue()));
});
// tags and properties in affected scopes that must be kept or preserved
directives.entrySet().stream().filter(entry -> scopes.contains(entry.getKey().getScope())).forEach(entry -> {
ScopedNameOfKind key = entry.getKey();
if (key.getKind() == MetadataKind.TAG && (entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP)) {
ScopedName tag = new ScopedName(key.getScope(), key.getName());
if (!meta.getTags().contains(tag) && before.getMetadata().getTags().contains(tag)) {
existingTagsToKeep.add(tag);
}
} else if (key.getKind() == MetadataKind.PROPERTY) {
ScopedName property = new ScopedName(key.getScope(), key.getName());
String existingValue = before.getMetadata().getProperties().get(property);
String newValue = meta.getProperties().get(property);
if (existingValue != null && (entry.getValue() == MetadataDirective.PRESERVE && !existingValue.equals(newValue) || entry.getValue() == MetadataDirective.KEEP && newValue == null)) {
existingPropertiesToKeep.put(property, existingValue);
}
}
});
// compute the new tags and properties
Set<ScopedName> newTags = existingTagsToKeep.isEmpty() ? meta.getTags() : Sets.union(meta.getTags(), existingTagsToKeep);
Map<ScopedName, String> newProperties = meta.getProperties();
if (!existingPropertiesToKeep.isEmpty()) {
newProperties = new HashMap<>(newProperties);
newProperties.putAll(existingPropertiesToKeep);
}
Metadata after = new Metadata(newTags, newProperties);
return new RequestAndChange(writeToIndex(create.getEntity(), before.getVersion(), after), new MetadataChange(create.getEntity(), before.getMetadata(), after));
}
use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by cdapio.
the class ElasticsearchMetadataStorageTest method testFiltering.
@Test
public void testFiltering() {
ScopedName sys = new ScopedName(MetadataScope.SYSTEM, "s");
ScopedName user = new ScopedName(MetadataScope.USER, "u");
String sval = "S";
String uval = "U";
Metadata before = new Metadata(tags(sys, user), props(sys, sval, user, uval));
// test selection to remove
Assert.assertEquals(new Metadata(tags(sys), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.NONE, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
// test selection is not affected by scopes or kinds
Assert.assertEquals(new Metadata(tags(sys), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.ALL, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
// test selection to keep
Assert.assertEquals(new Metadata(tags(user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.NONE, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
// test selection is not affected by scopes or kinds
Assert.assertEquals(new Metadata(tags(user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.ALL, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
// test removing nothing
Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.NONE, null));
Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.ALL, null));
Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.NONE, null));
// test keeping all
Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.ALL, null));
// test removing all
Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.ALL, null));
// test keeping nothing
Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.NONE, null));
// test keeping nothing
Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.NONE, null));
// test keeping nothing
Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.ALL, null));
// test removing all SYSTEM
Assert.assertEquals(new Metadata(tags(user), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, Collections.singleton(MetadataScope.SYSTEM), null));
// test removing all USER
Assert.assertEquals(new Metadata(tags(sys), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, Collections.singleton(MetadataScope.USER), null));
// test keeping all SYSTEM
Assert.assertEquals(new Metadata(tags(sys), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, Collections.singleton(MetadataScope.SYSTEM), null));
// test keeping all USER
Assert.assertEquals(new Metadata(tags(user), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, Collections.singleton(MetadataScope.USER), null));
// test removing all tags
Assert.assertEquals(new Metadata(tags(), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.TAG), MetadataScope.ALL, null));
// test removing all properties
Assert.assertEquals(new Metadata(tags(sys, user), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.PROPERTY), MetadataScope.ALL, null));
// test keeping all tags
Assert.assertEquals(new Metadata(tags(sys, user), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.TAG), MetadataScope.ALL, null));
// test keeping all properties
Assert.assertEquals(new Metadata(tags(), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.PROPERTY), MetadataScope.ALL, null));
// test removing all tags in SYSTEM scope
Assert.assertEquals(new Metadata(tags(user), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.TAG), Collections.singleton(MetadataScope.SYSTEM), null));
// test removing all properties in USER scope
Assert.assertEquals(new Metadata(tags(sys, user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.PROPERTY), Collections.singleton(MetadataScope.USER), null));
// test keeping all tags in SYSTEM scope
Assert.assertEquals(new Metadata(tags(sys), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.TAG), Collections.singleton(MetadataScope.SYSTEM), null));
// test keeping all properties in USER scope
Assert.assertEquals(new Metadata(tags(), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.PROPERTY), Collections.singleton(MetadataScope.USER), null));
}
use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.
the class ElasticsearchMetadataStorage method create.
/**
* Creates the Elasticsearch index request for an entity creation or update.
* See {@link MetadataMutation.Create} for detailed semantics.
*
* @param before the metadata for the mutation's entity before the change
* @param create the mutation to apply
*
* @return an ElasticSearch request to be executed, and the change caused by the mutation
*/
private RequestAndChange create(VersionedMetadata before, MetadataMutation.Create create) {
// if the entity did not exist before, none of the directives apply and this is equivalent to update()
if (!before.existing()) {
return update(create.getEntity(), before, create.getMetadata());
}
Metadata meta = create.getMetadata();
Map<ScopedNameOfKind, MetadataDirective> directives = create.getDirectives();
// determine the scopes that this mutation applies to (scopes that do not occur in the metadata are no changed)
Set<MetadataScope> scopes = Stream.concat(meta.getTags().stream(), meta.getProperties().keySet().stream()).map(ScopedName::getScope).collect(Collectors.toSet());
// compute what previously existing tags and properties have to be preserved (all others are replaced)
Set<ScopedName> existingTagsToKeep = new HashSet<>();
Map<ScopedName, String> existingPropertiesToKeep = new HashMap<>();
// all tags and properties that are in a scope not affected by this mutation
Sets.difference(MetadataScope.ALL, scopes).forEach(scope -> {
before.getMetadata().getTags().stream().filter(tag -> tag.getScope().equals(scope)).forEach(existingTagsToKeep::add);
before.getMetadata().getProperties().entrySet().stream().filter(entry -> entry.getKey().getScope().equals(scope)).forEach(entry -> existingPropertiesToKeep.put(entry.getKey(), entry.getValue()));
});
// tags and properties in affected scopes that must be kept or preserved
directives.entrySet().stream().filter(entry -> scopes.contains(entry.getKey().getScope())).forEach(entry -> {
ScopedNameOfKind key = entry.getKey();
if (key.getKind() == MetadataKind.TAG && (entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP)) {
ScopedName tag = new ScopedName(key.getScope(), key.getName());
if (!meta.getTags().contains(tag) && before.getMetadata().getTags().contains(tag)) {
existingTagsToKeep.add(tag);
}
} else if (key.getKind() == MetadataKind.PROPERTY) {
ScopedName property = new ScopedName(key.getScope(), key.getName());
String existingValue = before.getMetadata().getProperties().get(property);
String newValue = meta.getProperties().get(property);
if (existingValue != null && (entry.getValue() == MetadataDirective.PRESERVE && !existingValue.equals(newValue) || entry.getValue() == MetadataDirective.KEEP && newValue == null)) {
existingPropertiesToKeep.put(property, existingValue);
}
}
});
// compute the new tags and properties
Set<ScopedName> newTags = existingTagsToKeep.isEmpty() ? meta.getTags() : Sets.union(meta.getTags(), existingTagsToKeep);
Map<ScopedName, String> newProperties = meta.getProperties();
if (!existingPropertiesToKeep.isEmpty()) {
newProperties = new HashMap<>(newProperties);
newProperties.putAll(existingPropertiesToKeep);
}
Metadata after = new Metadata(newTags, newProperties);
return new RequestAndChange(writeToIndex(create.getEntity(), before.getVersion(), after), new MetadataChange(create.getEntity(), before.getMetadata(), after));
}
use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.
the class ElasticsearchMetadataStorage method update.
/**
* Creates the Elasticsearch index request for updating the metadata of an entity.
* This updates or adds the new metadata to the corresponding metadata document in the index.
*
* @param before the metadata for the mutation's entity before the change, and its version
*
* @return an ElasticSearch request to be executed, and the change caused by the mutation
*/
private RequestAndChange update(MetadataEntity entity, VersionedMetadata before, Metadata updates) {
Set<ScopedName> tags = new HashSet<>(before.getMetadata().getTags());
tags.addAll(updates.getTags());
Map<ScopedName, String> properties = new HashMap<>(before.getMetadata().getProperties());
properties.putAll(updates.getProperties());
Metadata after = new Metadata(tags, properties);
return new RequestAndChange(writeToIndex(entity, before.getVersion(), after), new MetadataChange(entity, before.getMetadata(), after));
}
use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.
the class DefaultMetadataServiceClientTest method testUpdate.
@Test
public void testUpdate() throws Exception {
final MetadataEntity updateEntity = MetadataEntity.builder().append("update", "test").build();
createMetadataMutation(new MetadataMutation.Create(updateEntity, testMetadata, CREATE_DIRECTIVES));
// confirm that the create was successful
Assert.assertEquals(testMetadata.getProperties(MetadataScope.SYSTEM), getMetadataProperties(updateEntity, MetadataScope.SYSTEM));
Assert.assertEquals(testMetadata.getTags(MetadataScope.SYSTEM), getMetadataTags(updateEntity, MetadataScope.SYSTEM));
Assert.assertEquals(testMetadata.getProperties(MetadataScope.USER), getMetadataProperties(updateEntity, MetadataScope.USER));
Assert.assertEquals(testMetadata.getTags(MetadataScope.USER), getMetadataTags(updateEntity, MetadataScope.USER));
Metadata update = new Metadata(Collections.EMPTY_SET, ImmutableMap.of(new ScopedName(MetadataScope.SYSTEM, "x"), "10", new ScopedName(MetadataScope.SYSTEM, "y"), "11", new ScopedName(MetadataScope.USER, "z"), "4"));
updateMetadataMutation(new MetadataMutation.Update(updateEntity, update));
Assert.assertEquals(update.getProperties(MetadataScope.SYSTEM), getMetadataProperties(updateEntity, MetadataScope.SYSTEM));
Assert.assertEquals(testMetadata.getTags(MetadataScope.SYSTEM), getMetadataTags(updateEntity, MetadataScope.SYSTEM));
Assert.assertEquals(update.getProperties(MetadataScope.USER), getMetadataProperties(updateEntity, MetadataScope.USER));
Assert.assertEquals(testMetadata.getTags(MetadataScope.USER), getMetadataTags(updateEntity, MetadataScope.USER));
}
Aggregations