Examples with MetadataScope io.cdap.cdap.api.metadata.MetadataScope used on opensource projects

Example 11 with MetadataScope

use of io.cdap.cdap.api.metadata.MetadataScope in project cdap by caskdata.

the class TestFrameworkTestRun method testMetadataAccessInService.

@Category(SlowTests.class)
@Test
public void testMetadataAccessInService() throws Exception {
    ApplicationManager applicationManager = deployApplication(AppWithMetadataPrograms.class);
    LOG.info("Deployed.");
    ServiceManager serviceManager = applicationManager.getServiceManager(AppWithMetadataPrograms.METADATA_SERVICE_NAME).start();
    serviceManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS);
    LOG.info("Service Started");
    URL serviceURL = serviceManager.getServiceURL(15, TimeUnit.SECONDS);
    Assert.assertNotNull(serviceURL);
    // add some tags
    callServicePut(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET + "/tags", "tag1");
    callServicePut(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET + "/tags", "tag2");
    // add some properties
    Map<String, String> propertiesToAdd = ImmutableMap.of("k1", "v1", "k2", "v2");
    callServicePut(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET + "/properties", GSON.toJson(propertiesToAdd));
    // test service is able to read metadata
    String result = callServiceGet(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET);
    Map<MetadataScope, Metadata> scopeMetadata = GSON.fromJson(result, MAP_METADATASCOPE_METADATA_TYPE);
    // verify system metadata
    Assert.assertTrue(scopeMetadata.containsKey(MetadataScope.SYSTEM));
    Assert.assertTrue(scopeMetadata.containsKey(MetadataScope.USER));
    Assert.assertFalse(scopeMetadata.get(MetadataScope.SYSTEM).getProperties().isEmpty());
    Assert.assertFalse(scopeMetadata.get(MetadataScope.SYSTEM).getTags().isEmpty());
    Assert.assertTrue(scopeMetadata.get(MetadataScope.SYSTEM).getProperties().containsKey("entity-name"));
    Assert.assertEquals(AppWithMetadataPrograms.METADATA_SERVICE_DATASET, scopeMetadata.get(MetadataScope.SYSTEM).getProperties().get("entity-name"));
    Assert.assertTrue(scopeMetadata.get(MetadataScope.SYSTEM).getTags().containsAll(Arrays.asList("explore", "batch")));
    // verify user metadata
    Assert.assertFalse(scopeMetadata.get(MetadataScope.USER).getProperties().isEmpty());
    Assert.assertFalse(scopeMetadata.get(MetadataScope.USER).getTags().isEmpty());
    Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getTags().containsAll(Arrays.asList("tag1", "tag2")));
    Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getProperties().containsKey("k1"));
    Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getProperties().containsKey("k2"));
    Assert.assertEquals("v1", scopeMetadata.get(MetadataScope.USER).getProperties().get("k1"));
    Assert.assertEquals("v2", scopeMetadata.get(MetadataScope.USER).getProperties().get("k2"));
    // delete a tag
    callServiceDelete(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET + "/tags/" + "tag1");
    // delete a property
    callServiceDelete(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET + "/properties/" + "k1");
    // get metadata and verify
    result = callServiceGet(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET);
    scopeMetadata = GSON.fromJson(result, MAP_METADATASCOPE_METADATA_TYPE);
    Assert.assertEquals(1, scopeMetadata.get(MetadataScope.USER).getTags().size());
    Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getTags().contains("tag2"));
    Assert.assertEquals(1, scopeMetadata.get(MetadataScope.USER).getProperties().size());
    Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getProperties().containsKey("k2"));
    Assert.assertEquals("v2", scopeMetadata.get(MetadataScope.USER).getProperties().get("k2"));
    // delete all tags
    callServiceDelete(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET + "/tags");
    // get metadata and verify
    result = callServiceGet(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET);
    scopeMetadata = GSON.fromJson(result, MAP_METADATASCOPE_METADATA_TYPE);
    Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getTags().isEmpty());
    Assert.assertFalse(scopeMetadata.get(MetadataScope.USER).getProperties().isEmpty());
    // delete all properties
    callServiceDelete(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET + "/properties");
    // get metadata and verify
    result = callServiceGet(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET);
    scopeMetadata = GSON.fromJson(result, MAP_METADATASCOPE_METADATA_TYPE);
    Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getProperties().isEmpty());
    // add some tag and property again
    callServicePut(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET + "/tags", "tag1");
    callServicePut(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET + "/properties", GSON.toJson(propertiesToAdd));
    // get metadata and verify
    result = callServiceGet(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET);
    scopeMetadata = GSON.fromJson(result, MAP_METADATASCOPE_METADATA_TYPE);
    Assert.assertFalse(scopeMetadata.get(MetadataScope.USER).getTags().isEmpty());
    Assert.assertFalse(scopeMetadata.get(MetadataScope.USER).getProperties().isEmpty());
    // delete all metadata
    callServiceDelete(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET);
    // get metadata and verify
    result = callServiceGet(serviceManager.getServiceURL(), "metadata/" + AppWithMetadataPrograms.METADATA_SERVICE_DATASET);
    scopeMetadata = GSON.fromJson(result, MAP_METADATASCOPE_METADATA_TYPE);
    Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getTags().isEmpty());
    Assert.assertTrue(scopeMetadata.get(MetadataScope.USER).getProperties().isEmpty());
}

Example 12 with MetadataScope

use of io.cdap.cdap.api.metadata.MetadataScope in project cdap by cdapio.

the class ElasticsearchMetadataStorage method create.

/**
 * Creates the Elasticsearch index request for an entity creation or update.
 * See {@link MetadataMutation.Create} for detailed semantics.
 *
 * @param before the metadata for the mutation's entity before the change
 * @param create the mutation to apply
 *
 * @return an ElasticSearch request to be executed, and the change caused by the mutation
 */
private RequestAndChange create(VersionedMetadata before, MetadataMutation.Create create) {
    // if the entity did not exist before, none of the directives apply and this is equivalent to update()
    if (!before.existing()) {
        return update(create.getEntity(), before, create.getMetadata());
    }
    Metadata meta = create.getMetadata();
    Map<ScopedNameOfKind, MetadataDirective> directives = create.getDirectives();
    // determine the scopes that this mutation applies to (scopes that do not occur in the metadata are no changed)
    Set<MetadataScope> scopes = Stream.concat(meta.getTags().stream(), meta.getProperties().keySet().stream()).map(ScopedName::getScope).collect(Collectors.toSet());
    // compute what previously existing tags and properties have to be preserved (all others are replaced)
    Set<ScopedName> existingTagsToKeep = new HashSet<>();
    Map<ScopedName, String> existingPropertiesToKeep = new HashMap<>();
    // all tags and properties that are in a scope not affected by this mutation
    Sets.difference(MetadataScope.ALL, scopes).forEach(scope -> {
        before.getMetadata().getTags().stream().filter(tag -> tag.getScope().equals(scope)).forEach(existingTagsToKeep::add);
        before.getMetadata().getProperties().entrySet().stream().filter(entry -> entry.getKey().getScope().equals(scope)).forEach(entry -> existingPropertiesToKeep.put(entry.getKey(), entry.getValue()));
    });
    // tags and properties in affected scopes that must be kept or preserved
    directives.entrySet().stream().filter(entry -> scopes.contains(entry.getKey().getScope())).forEach(entry -> {
        ScopedNameOfKind key = entry.getKey();
        if (key.getKind() == MetadataKind.TAG && (entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP)) {
            ScopedName tag = new ScopedName(key.getScope(), key.getName());
            if (!meta.getTags().contains(tag) && before.getMetadata().getTags().contains(tag)) {
                existingTagsToKeep.add(tag);
            }
        } else if (key.getKind() == MetadataKind.PROPERTY) {
            ScopedName property = new ScopedName(key.getScope(), key.getName());
            String existingValue = before.getMetadata().getProperties().get(property);
            String newValue = meta.getProperties().get(property);
            if (existingValue != null && (entry.getValue() == MetadataDirective.PRESERVE && !existingValue.equals(newValue) || entry.getValue() == MetadataDirective.KEEP && newValue == null)) {
                existingPropertiesToKeep.put(property, existingValue);
            }
        }
    });
    // compute the new tags and properties
    Set<ScopedName> newTags = existingTagsToKeep.isEmpty() ? meta.getTags() : Sets.union(meta.getTags(), existingTagsToKeep);
    Map<ScopedName, String> newProperties = meta.getProperties();
    if (!existingPropertiesToKeep.isEmpty()) {
        newProperties = new HashMap<>(newProperties);
        newProperties.putAll(existingPropertiesToKeep);
    }
    Metadata after = new Metadata(newTags, newProperties);
    return new RequestAndChange(writeToIndex(create.getEntity(), before.getVersion(), after), new MetadataChange(create.getEntity(), before.getMetadata(), after));
}

Example 13 with MetadataScope

use of io.cdap.cdap.api.metadata.MetadataScope in project cdap by cdapio.

the class ScopedName method fromString.

public static ScopedName fromString(String s) {
    String[] parts = s.split(":", 2);
    if (parts.length != 2) {
        throw new IllegalArgumentException(String.format("Cannot parse '%s' as a ScopedName", s));
    }
    MetadataScope scope = MetadataScope.valueOf(parts[0]);
    return new ScopedName(scope, parts[1]);
}

Example 14 with MetadataScope

use of io.cdap.cdap.api.metadata.MetadataScope in project cdap by cdapio.

the class MetadataCompatibility method toV5Results.

/**
 * Convert a list of {@link MetadataRecord}s to an ordered set of 5.x {@link MetadataSearchResultRecord}s.
 *
 * The 5.x convention was that the results only contain non-empty records.
 */
private static Set<MetadataSearchResultRecord> toV5Results(List<MetadataRecord> results) {
    Set<MetadataSearchResultRecord> records = new LinkedHashSet<>();
    for (MetadataRecord record : results) {
        Map<MetadataScope, io.cdap.cdap.api.metadata.Metadata> map = toV5Metadata(record.getMetadata());
        records.add(new MetadataSearchResultRecord(record.getEntity(), Maps.filterValues(map, meta -> meta != null && !(meta.getProperties().isEmpty() && meta.getTags().isEmpty()))));
    }
    return records;
}

Example 15 with MetadataScope

use of io.cdap.cdap.api.metadata.MetadataScope in project cdap by cdapio.

the class DatasetMetadataStorage method search.

@Override
public SearchResponse search(SearchRequest request) {
    Cursor cursor = request.getCursor() != null && !request.getCursor().isEmpty() ? Cursor.fromString(request.getCursor()) : null;
    Set<String> namespaces = cursor == null ? request.getNamespaces() : cursor.getNamespaces();
    ImmutablePair<NamespaceId, Set<EntityScope>> namespaceAndScopes = determineNamespaceAndScopes(namespaces);
    CursorAndOffsetInfo cursorOffsetAndLimits = determineCursorOffsetAndLimits(request, cursor);
    String query = cursor != null ? cursor.getQuery() : request.getQuery() == null || request.getQuery().isEmpty() ? "*" : request.getQuery();
    Set<String> types = cursor != null ? cursor.getTypes() : request.getTypes();
    types = types == null ? Collections.emptySet() : types;
    Sorting sorting = cursor == null ? request.getSorting() : cursor.getSorting() == null ? null : Sorting.of(cursor.getSorting());
    SortInfo sortInfo = sorting == null ? SortInfo.DEFAULT : new SortInfo(sorting.getKey(), SortInfo.SortOrder.valueOf(sorting.getOrder().name()));
    boolean showHidden = cursor != null ? cursor.isShowHidden() : request.isShowHidden();
    MetadataScope scope = cursor != null ? cursor.getScope() : request.getScope();
    MetadataSearchResponse response = search(new io.cdap.cdap.data2.metadata.dataset.SearchRequest(namespaceAndScopes.getFirst(), query, types, sortInfo, cursorOffsetAndLimits.getOffsetToRequest(), cursorOffsetAndLimits.getLimitToRequest(), request.isCursorRequested() ? 1 : 0, cursorOffsetAndLimits.getCursor(), showHidden, namespaceAndScopes.getSecond()), scope);
    // translate results back and limit them to at most what was requested (see above where we add 1)
    int limitToRespond = cursorOffsetAndLimits.getLimitToRespond();
    int offsetToRespond = cursorOffsetAndLimits.getOffsetToRespond();
    List<MetadataRecord> results = response.getResults().stream().limit(limitToRespond).map(record -> {
        Metadata metadata = Metadata.EMPTY;
        for (Map.Entry<MetadataScope, io.cdap.cdap.api.metadata.Metadata> entry : record.getMetadata().entrySet()) {
            Metadata toAdd = new Metadata(entry.getKey(), entry.getValue().getTags(), entry.getValue().getProperties());
            metadata = mergeDisjointMetadata(metadata, toAdd);
        }
        return new MetadataRecord(record.getMetadataEntity(), metadata);
    }).collect(Collectors.toList());
    Cursor newCursor = null;
    if (response.getCursors() != null && !response.getCursors().isEmpty()) {
        String actualCursor = response.getCursors().get(0);
        if (cursor != null) {
            // the new cursor's offset is the previous cursor's offset plus the number of results
            newCursor = new Cursor(cursor, cursor.getOffset() + results.size(), actualCursor);
        } else {
            newCursor = new Cursor(offsetToRespond + results.size(), limitToRespond, showHidden, scope, namespaces, types, sorting == null ? null : sorting.toString(), actualCursor, query);
        }
    }
    // adjust the total results by the difference of requested offset and the true offset that we respond back
    int totalResults = offsetToRespond - cursorOffsetAndLimits.getOffsetToRequest() + response.getTotal();
    return new SearchResponse(request, newCursor == null ? null : newCursor.toString(), offsetToRespond, limitToRespond, totalResults, results);
}