Examples with ScopedName io.cdap.cdap.spi.metadata.ScopedName used on opensource projects

Example 1 with ScopedName

use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.

the class DatasetMetadataStorage method replaceInScope.

private MetadataDataset.Change replaceInScope(MetadataDatasetContext context, MetadataScope scope, MetadataEntity entity, Set<String> newTags, Map<String, String> newProperties, Map<ScopedNameOfKind, MetadataDirective> directives) {
    MetadataDataset dataset = context.getDataset(scope);
    MetadataDataset.Record before = dataset.getMetadata(entity);
    if (newTags.isEmpty() && newProperties.isEmpty()) {
        // this scope remains unchanged
        return new MetadataDataset.Change(before, before);
    }
    Set<String> existingTags = before.getTags();
    Set<String> tagsToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == TAG && (entry.getValue() == MetadataDirective.KEEP || entry.getValue() == MetadataDirective.PRESERVE)).map(Map.Entry::getKey).map(ScopedName::getName).filter(existingTags::contains).collect(Collectors.toSet());
    newTags = Sets.union(newTags, tagsToKeepOrPreserve);
    Map<String, String> existingProperties = before.getProperties();
    Map<String, String> propertiesToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == PROPERTY).filter(entry -> existingProperties.containsKey(entry.getKey().getName())).filter(entry -> entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP && !newProperties.containsKey(entry.getKey().getName())).map(Map.Entry::getKey).map(ScopedName::getName).collect(Collectors.toMap(name -> name, existingProperties::get));
    newProperties.putAll(propertiesToKeepOrPreserve);
    Set<String> tagsToRemove = Sets.difference(before.getTags(), newTags);
    Set<String> tagsToAdd = Sets.difference(newTags, before.getTags());
    Set<String> propertiesToRemove = Sets.difference(before.getProperties().keySet(), newProperties.keySet());
    @SuppressWarnings("ConstantConditions") Map<String, String> propertiesToAdd = Maps.filterEntries(newProperties, entry -> !entry.getValue().equals(existingProperties.get(entry.getKey())));
    MetadataDataset.Record after = before;
    if (!tagsToRemove.isEmpty()) {
        after = dataset.removeTags(entity, tagsToRemove).getLatest();
    }
    if (!tagsToAdd.isEmpty()) {
        after = dataset.addTags(entity, tagsToAdd).getLatest();
    }
    if (!propertiesToRemove.isEmpty()) {
        after = dataset.removeProperties(entity, propertiesToRemove).getLatest();
    }
    if (!propertiesToAdd.isEmpty()) {
        after = dataset.addProperties(entity, propertiesToAdd).getLatest();
    }
    return new MetadataDataset.Change(before, after);
}

Example 2 with ScopedName

use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.

the class DatasetMetadataStorage method readScope.

private MetadataDataset.Record readScope(MetadataDatasetContext context, MetadataScope scope, Read read) {
    MetadataEntity entity = read.getEntity();
    if (read.getSelection() == null && (!read.getScopes().contains(scope) || read.getKinds().isEmpty())) {
        return new MetadataDataset.Record(entity);
    }
    Set<ScopedNameOfKind> selectionForScope = null;
    if (read.getSelection() != null) {
        // noinspection ConstantConditions
        selectionForScope = Sets.filter(read.getSelection(), entry -> entry.getScope() == scope);
        if (selectionForScope.isEmpty()) {
            return new MetadataDataset.Record(entity);
        }
    }
    // now we know we must read from the dataset
    MetadataDataset dataset = context.getDataset(scope);
    if (selectionForScope != null) {
        // request is for a specific set of tags and properties
        Set<String> tagsToRead = selectionForScope.stream().filter(entry -> TAG == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
        Set<String> propertiesToRead = selectionForScope.stream().filter(entry -> PROPERTY == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
        Set<String> tags = tagsToRead.isEmpty() ? Collections.emptySet() : Sets.intersection(tagsToRead, dataset.getTags(entity));
        Map<String, String> properties = propertiesToRead.isEmpty() ? Collections.emptyMap() : Maps.filterKeys(dataset.getProperties(entity), propertiesToRead::contains);
        return new MetadataDataset.Record(entity, properties, tags);
    }
    if (MetadataKind.ALL.equals(read.getKinds())) {
        // all metadata kinds requested
        return dataset.getMetadata(entity);
    }
    // exactly one kind is requested
    MetadataKind requestKind = read.getKinds().iterator().next();
    if (requestKind == TAG) {
        return new MetadataDataset.Record(entity, Collections.emptyMap(), dataset.getTags(entity));
    }
    if (requestKind == PROPERTY) {
        return new MetadataDataset.Record(entity, dataset.getProperties(entity), Collections.emptySet());
    }
    throw new IllegalStateException("Encountered metadata read request for unknown kind " + requestKind);
}

Example 3 with ScopedName

use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.

the class DefaultMetadataServiceClientTest method testCreate.

@Test
public void testCreate() throws Exception {
    final MetadataEntity createEntity = MetadataEntity.builder().append("create", "test").build();
    createMetadataMutation(new MetadataMutation.Create(createEntity, testMetadata, CREATE_DIRECTIVES));
    Assert.assertEquals(testMetadata.getProperties(MetadataScope.SYSTEM), getMetadataProperties(createEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(testMetadata.getTags(MetadataScope.SYSTEM), getMetadataTags(createEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(testMetadata.getProperties(MetadataScope.USER), getMetadataProperties(createEntity, MetadataScope.USER));
    Assert.assertEquals(testMetadata.getTags(MetadataScope.USER), getMetadataTags(createEntity, MetadataScope.USER));
    // confirm that KEEP/PRESERVE work by trying to re-create the entity
    final Metadata recreate = new Metadata(Collections.EMPTY_SET, ImmutableMap.of(new ScopedName(MetadataScope.SYSTEM, "x"), "10", new ScopedName(MetadataScope.SYSTEM, "y"), "20", new ScopedName(MetadataScope.USER, "z"), "40"));
    createMetadataMutation(new MetadataMutation.Create(createEntity, recreate, CREATE_DIRECTIVES));
    Assert.assertEquals(recreate.getProperties(MetadataScope.SYSTEM), getMetadataProperties(createEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(recreate.getTags(MetadataScope.SYSTEM), getMetadataTags(createEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(testMetadata.getProperties(MetadataScope.USER), getMetadataProperties(createEntity, MetadataScope.USER));
    Assert.assertEquals(testMetadata.getTags(MetadataScope.USER), getMetadataTags(createEntity, MetadataScope.USER));
}

Example 4 with ScopedName

use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.

the class ElasticsearchMetadataStorageTest method testFiltering.

@Test
public void testFiltering() {
    ScopedName sys = new ScopedName(MetadataScope.SYSTEM, "s");
    ScopedName user = new ScopedName(MetadataScope.USER, "u");
    String sval = "S";
    String uval = "U";
    Metadata before = new Metadata(tags(sys, user), props(sys, sval, user, uval));
    // test selection to remove
    Assert.assertEquals(new Metadata(tags(sys), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.NONE, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test selection is not affected by scopes or kinds
    Assert.assertEquals(new Metadata(tags(sys), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.ALL, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test selection to keep
    Assert.assertEquals(new Metadata(tags(user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.NONE, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test selection is not affected by scopes or kinds
    Assert.assertEquals(new Metadata(tags(user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.ALL, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, user), new ScopedNameOfKind(MetadataKind.PROPERTY, sys))));
    // test removing nothing
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.NONE, null));
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.NONE, MetadataScope.ALL, null));
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.NONE, null));
    // test keeping all
    Assert.assertEquals(before, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.ALL, null));
    // test removing all
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, MetadataScope.ALL, null));
    // test keeping nothing
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.NONE, null));
    // test keeping nothing
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, MetadataScope.NONE, null));
    // test keeping nothing
    Assert.assertEquals(Metadata.EMPTY, ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.NONE, MetadataScope.ALL, null));
    // test removing all SYSTEM
    Assert.assertEquals(new Metadata(tags(user), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, Collections.singleton(MetadataScope.SYSTEM), null));
    // test removing all USER
    Assert.assertEquals(new Metadata(tags(sys), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, MetadataKind.ALL, Collections.singleton(MetadataScope.USER), null));
    // test keeping all SYSTEM
    Assert.assertEquals(new Metadata(tags(sys), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, Collections.singleton(MetadataScope.SYSTEM), null));
    // test keeping all USER
    Assert.assertEquals(new Metadata(tags(user), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, MetadataKind.ALL, Collections.singleton(MetadataScope.USER), null));
    // test removing all tags
    Assert.assertEquals(new Metadata(tags(), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.TAG), MetadataScope.ALL, null));
    // test removing all properties
    Assert.assertEquals(new Metadata(tags(sys, user), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.PROPERTY), MetadataScope.ALL, null));
    // test keeping all tags
    Assert.assertEquals(new Metadata(tags(sys, user), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.TAG), MetadataScope.ALL, null));
    // test keeping all properties
    Assert.assertEquals(new Metadata(tags(), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.PROPERTY), MetadataScope.ALL, null));
    // test removing all tags in SYSTEM scope
    Assert.assertEquals(new Metadata(tags(user), props(sys, sval, user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.TAG), Collections.singleton(MetadataScope.SYSTEM), null));
    // test removing all properties in USER scope
    Assert.assertEquals(new Metadata(tags(sys, user), props(sys, sval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.DISCARD, Collections.singleton(MetadataKind.PROPERTY), Collections.singleton(MetadataScope.USER), null));
    // test keeping all tags in SYSTEM scope
    Assert.assertEquals(new Metadata(tags(sys), props()), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.TAG), Collections.singleton(MetadataScope.SYSTEM), null));
    // test keeping all properties in USER scope
    Assert.assertEquals(new Metadata(tags(), props(user, uval)), ElasticsearchMetadataStorage.filterMetadata(before, ElasticsearchMetadataStorage.KEEP, Collections.singleton(MetadataKind.PROPERTY), Collections.singleton(MetadataScope.USER), null));
}

Example 5 with ScopedName

use of io.cdap.cdap.spi.metadata.ScopedName in project cdap by caskdata.

the class ElasticsearchMetadataStorage method create.

/**
 * Creates the Elasticsearch index request for an entity creation or update.
 * See {@link MetadataMutation.Create} for detailed semantics.
 *
 * @param before the metadata for the mutation's entity before the change
 * @param create the mutation to apply
 *
 * @return an ElasticSearch request to be executed, and the change caused by the mutation
 */
private RequestAndChange create(VersionedMetadata before, MetadataMutation.Create create) {
    // if the entity did not exist before, none of the directives apply and this is equivalent to update()
    if (!before.existing()) {
        return update(create.getEntity(), before, create.getMetadata());
    }
    Metadata meta = create.getMetadata();
    Map<ScopedNameOfKind, MetadataDirective> directives = create.getDirectives();
    // determine the scopes that this mutation applies to (scopes that do not occur in the metadata are no changed)
    Set<MetadataScope> scopes = Stream.concat(meta.getTags().stream(), meta.getProperties().keySet().stream()).map(ScopedName::getScope).collect(Collectors.toSet());
    // compute what previously existing tags and properties have to be preserved (all others are replaced)
    Set<ScopedName> existingTagsToKeep = new HashSet<>();
    Map<ScopedName, String> existingPropertiesToKeep = new HashMap<>();
    // all tags and properties that are in a scope not affected by this mutation
    Sets.difference(MetadataScope.ALL, scopes).forEach(scope -> {
        before.getMetadata().getTags().stream().filter(tag -> tag.getScope().equals(scope)).forEach(existingTagsToKeep::add);
        before.getMetadata().getProperties().entrySet().stream().filter(entry -> entry.getKey().getScope().equals(scope)).forEach(entry -> existingPropertiesToKeep.put(entry.getKey(), entry.getValue()));
    });
    // tags and properties in affected scopes that must be kept or preserved
    directives.entrySet().stream().filter(entry -> scopes.contains(entry.getKey().getScope())).forEach(entry -> {
        ScopedNameOfKind key = entry.getKey();
        if (key.getKind() == MetadataKind.TAG && (entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP)) {
            ScopedName tag = new ScopedName(key.getScope(), key.getName());
            if (!meta.getTags().contains(tag) && before.getMetadata().getTags().contains(tag)) {
                existingTagsToKeep.add(tag);
            }
        } else if (key.getKind() == MetadataKind.PROPERTY) {
            ScopedName property = new ScopedName(key.getScope(), key.getName());
            String existingValue = before.getMetadata().getProperties().get(property);
            String newValue = meta.getProperties().get(property);
            if (existingValue != null && (entry.getValue() == MetadataDirective.PRESERVE && !existingValue.equals(newValue) || entry.getValue() == MetadataDirective.KEEP && newValue == null)) {
                existingPropertiesToKeep.put(property, existingValue);
            }
        }
    });
    // compute the new tags and properties
    Set<ScopedName> newTags = existingTagsToKeep.isEmpty() ? meta.getTags() : Sets.union(meta.getTags(), existingTagsToKeep);
    Map<ScopedName, String> newProperties = meta.getProperties();
    if (!existingPropertiesToKeep.isEmpty()) {
        newProperties = new HashMap<>(newProperties);
        newProperties.putAll(existingPropertiesToKeep);
    }
    Metadata after = new Metadata(newTags, newProperties);
    return new RequestAndChange(writeToIndex(create.getEntity(), before.getVersion(), after), new MetadataChange(create.getEntity(), before.getMetadata(), after));
}