use of io.cdap.cdap.spi.metadata.MetadataKind.PROPERTY in project cdap by caskdata.
the class DatasetMetadataStorage method replaceInScope.
private MetadataDataset.Change replaceInScope(MetadataDatasetContext context, MetadataScope scope, MetadataEntity entity, Set<String> newTags, Map<String, String> newProperties, Map<ScopedNameOfKind, MetadataDirective> directives) {
MetadataDataset dataset = context.getDataset(scope);
MetadataDataset.Record before = dataset.getMetadata(entity);
if (newTags.isEmpty() && newProperties.isEmpty()) {
// this scope remains unchanged
return new MetadataDataset.Change(before, before);
}
Set<String> existingTags = before.getTags();
Set<String> tagsToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == TAG && (entry.getValue() == MetadataDirective.KEEP || entry.getValue() == MetadataDirective.PRESERVE)).map(Map.Entry::getKey).map(ScopedName::getName).filter(existingTags::contains).collect(Collectors.toSet());
newTags = Sets.union(newTags, tagsToKeepOrPreserve);
Map<String, String> existingProperties = before.getProperties();
Map<String, String> propertiesToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == PROPERTY).filter(entry -> existingProperties.containsKey(entry.getKey().getName())).filter(entry -> entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP && !newProperties.containsKey(entry.getKey().getName())).map(Map.Entry::getKey).map(ScopedName::getName).collect(Collectors.toMap(name -> name, existingProperties::get));
newProperties.putAll(propertiesToKeepOrPreserve);
Set<String> tagsToRemove = Sets.difference(before.getTags(), newTags);
Set<String> tagsToAdd = Sets.difference(newTags, before.getTags());
Set<String> propertiesToRemove = Sets.difference(before.getProperties().keySet(), newProperties.keySet());
@SuppressWarnings("ConstantConditions") Map<String, String> propertiesToAdd = Maps.filterEntries(newProperties, entry -> !entry.getValue().equals(existingProperties.get(entry.getKey())));
MetadataDataset.Record after = before;
if (!tagsToRemove.isEmpty()) {
after = dataset.removeTags(entity, tagsToRemove).getLatest();
}
if (!tagsToAdd.isEmpty()) {
after = dataset.addTags(entity, tagsToAdd).getLatest();
}
if (!propertiesToRemove.isEmpty()) {
after = dataset.removeProperties(entity, propertiesToRemove).getLatest();
}
if (!propertiesToAdd.isEmpty()) {
after = dataset.addProperties(entity, propertiesToAdd).getLatest();
}
return new MetadataDataset.Change(before, after);
}
use of io.cdap.cdap.spi.metadata.MetadataKind.PROPERTY in project cdap by caskdata.
the class DatasetMetadataStorage method readScope.
private MetadataDataset.Record readScope(MetadataDatasetContext context, MetadataScope scope, Read read) {
MetadataEntity entity = read.getEntity();
if (read.getSelection() == null && (!read.getScopes().contains(scope) || read.getKinds().isEmpty())) {
return new MetadataDataset.Record(entity);
}
Set<ScopedNameOfKind> selectionForScope = null;
if (read.getSelection() != null) {
// noinspection ConstantConditions
selectionForScope = Sets.filter(read.getSelection(), entry -> entry.getScope() == scope);
if (selectionForScope.isEmpty()) {
return new MetadataDataset.Record(entity);
}
}
// now we know we must read from the dataset
MetadataDataset dataset = context.getDataset(scope);
if (selectionForScope != null) {
// request is for a specific set of tags and properties
Set<String> tagsToRead = selectionForScope.stream().filter(entry -> TAG == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
Set<String> propertiesToRead = selectionForScope.stream().filter(entry -> PROPERTY == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
Set<String> tags = tagsToRead.isEmpty() ? Collections.emptySet() : Sets.intersection(tagsToRead, dataset.getTags(entity));
Map<String, String> properties = propertiesToRead.isEmpty() ? Collections.emptyMap() : Maps.filterKeys(dataset.getProperties(entity), propertiesToRead::contains);
return new MetadataDataset.Record(entity, properties, tags);
}
if (MetadataKind.ALL.equals(read.getKinds())) {
// all metadata kinds requested
return dataset.getMetadata(entity);
}
// exactly one kind is requested
MetadataKind requestKind = read.getKinds().iterator().next();
if (requestKind == TAG) {
return new MetadataDataset.Record(entity, Collections.emptyMap(), dataset.getTags(entity));
}
if (requestKind == PROPERTY) {
return new MetadataDataset.Record(entity, dataset.getProperties(entity), Collections.emptySet());
}
throw new IllegalStateException("Encountered metadata read request for unknown kind " + requestKind);
}
use of io.cdap.cdap.spi.metadata.MetadataKind.PROPERTY in project cdap by caskdata.
the class MetadataStorageTest method testBatch.
@Test
public void testBatch() throws IOException {
MetadataEntity entity = MetadataEntity.ofDataset("a", "b");
Map<ScopedNameOfKind, MetadataDirective> directives = ImmutableMap.of(new ScopedNameOfKind(PROPERTY, SYSTEM, CREATION_TIME_KEY), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, DESCRIPTION_KEY), MetadataDirective.KEEP);
MetadataStorage mds = getMetadataStorage();
Create create = new Create(entity, new Metadata(SYSTEM, tags("batch"), props(CREATION_TIME_KEY, "12345678", DESCRIPTION_KEY, "hello", "other", "value")), directives);
MetadataChange change = mds.apply(create, MutationOptions.DEFAULT);
Assert.assertEquals(Metadata.EMPTY, change.getBefore());
Assert.assertEquals(create.getMetadata(), change.getAfter());
List<MetadataMutation> mutations = ImmutableList.of(new Update(entity, new Metadata(USER, tags("tag1", "tag2"))), new Drop(entity), new Create(entity, new Metadata(SYSTEM, tags("batch"), props(CREATION_TIME_KEY, "23456789", "other", "different")), directives), new Update(entity, new Metadata(USER, tags("tag3"), props("key", "value"))), new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "other"), new ScopedNameOfKind(TAG, USER, "tag2"))), new Create(entity, new Metadata(SYSTEM, tags("realtime"), props(CREATION_TIME_KEY, "33456789", DESCRIPTION_KEY, "new description", "other", "yet other")), directives));
// apply all mutations in sequence
List<MetadataChange> changes = mutations.stream().map(mutation -> {
try {
return mds.apply(mutation, MutationOptions.DEFAULT);
} catch (IOException e) {
throw Throwables.propagate(e);
}
}).collect(Collectors.toList());
// drop and recreate the entity
mds.apply(new Drop(entity), MutationOptions.DEFAULT);
change = mds.apply(create, MutationOptions.DEFAULT);
Assert.assertEquals(Metadata.EMPTY, change.getBefore());
Assert.assertEquals(create.getMetadata(), change.getAfter());
// apply all mutations in batch
List<MetadataChange> batchChanges = mds.batch(mutations, MutationOptions.DEFAULT);
// make sure the same mutations were applied
Assert.assertEquals(changes, batchChanges);
// clean up
mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}
use of io.cdap.cdap.spi.metadata.MetadataKind.PROPERTY in project cdap by caskdata.
the class DatasetMetadataStorageTest method testSearchWeight.
// this tests is not in MetadataStorageTest,
// because it tests result scoring and sorting specific to the dataset-based implementation
@Test
public void testSearchWeight() throws IOException {
MetadataStorage mds = getMetadataStorage();
String ns = "ns1";
NamespaceId nsId = new NamespaceId(ns);
MetadataEntity service1 = nsId.app("app1").service("service1").toMetadataEntity();
MetadataEntity dataset1 = nsId.dataset("ds1").toMetadataEntity();
MetadataEntity dataset2 = nsId.dataset("ds2").toMetadataEntity();
// Add metadata
String multiWordValue = "aV1 av2 , - , av3 - av4_av5 av6";
Map<String, String> userProps = ImmutableMap.of("key1", "value1", "key2", "value2", "multiword", multiWordValue);
Map<String, String> systemProps = ImmutableMap.of("sysKey1", "sysValue1");
Set<String> userTags = ImmutableSet.of("tag1", "tag2");
Set<String> temporaryUserTags = ImmutableSet.of("tag3", "tag4");
Map<String, String> dataset1UserProps = ImmutableMap.of("sKey1", "sValuee1 sValuee2");
Map<String, String> dataset2UserProps = ImmutableMap.of("sKey1", "sValue1 sValue2", "Key1", "Value1");
Set<String> sysTags = ImmutableSet.of("sysTag1");
MetadataRecord service1Record = new MetadataRecord(service1, union(new Metadata(USER, userTags, userProps), new Metadata(SYSTEM, sysTags, systemProps)));
mds.apply(new Update(service1Record.getEntity(), service1Record.getMetadata()), MutationOptions.DEFAULT);
// dd and then remove some metadata for dataset2
mds.apply(new Update(dataset2, new Metadata(USER, temporaryUserTags, userProps)), MutationOptions.DEFAULT);
mds.apply(new Remove(dataset2, temporaryUserTags.stream().map(tag -> new ScopedNameOfKind(TAG, USER, tag)).collect(Collectors.toSet())), MutationOptions.DEFAULT);
mds.apply(new Remove(dataset2, userProps.keySet().stream().map(tag -> new ScopedNameOfKind(PROPERTY, USER, tag)).collect(Collectors.toSet())), MutationOptions.DEFAULT);
MetadataRecord dataset1Record = new MetadataRecord(dataset1, new Metadata(USER, tags(), dataset1UserProps));
MetadataRecord dataset2Record = new MetadataRecord(dataset2, new Metadata(USER, tags(), dataset2UserProps));
mds.batch(ImmutableList.of(new Update(dataset1Record.getEntity(), dataset1Record.getMetadata()), new Update(dataset2Record.getEntity(), dataset2Record.getMetadata())), MutationOptions.DEFAULT);
// Test score and metadata match
assertInOrder(mds, SearchRequest.of("value1 multiword:av2").addNamespace(ns).build(), service1Record, dataset2Record);
assertInOrder(mds, SearchRequest.of("value1 sValue*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), dataset2Record, dataset1Record, service1Record);
assertResults(mds, SearchRequest.of("*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), dataset2Record, dataset1Record, service1Record);
// clean up
mds.batch(ImmutableList.of(new Drop(service1), new Drop(dataset1), new Drop(dataset2)), MutationOptions.DEFAULT);
}
Aggregations