Examples with Metadata io.cdap.cdap.spi.metadata.Metadata used on opensource projects

Example 1 with Metadata

use of io.cdap.cdap.spi.metadata.Metadata in project cdap by caskdata.

the class DatasetMetadataStorage method search.

@Override
public SearchResponse search(SearchRequest request) {
    Cursor cursor = request.getCursor() != null && !request.getCursor().isEmpty() ? Cursor.fromString(request.getCursor()) : null;
    Set<String> namespaces = cursor == null ? request.getNamespaces() : cursor.getNamespaces();
    ImmutablePair<NamespaceId, Set<EntityScope>> namespaceAndScopes = determineNamespaceAndScopes(namespaces);
    CursorAndOffsetInfo cursorOffsetAndLimits = determineCursorOffsetAndLimits(request, cursor);
    String query = cursor != null ? cursor.getQuery() : request.getQuery() == null || request.getQuery().isEmpty() ? "*" : request.getQuery();
    Set<String> types = cursor != null ? cursor.getTypes() : request.getTypes();
    types = types == null ? Collections.emptySet() : types;
    Sorting sorting = cursor == null ? request.getSorting() : cursor.getSorting() == null ? null : Sorting.of(cursor.getSorting());
    SortInfo sortInfo = sorting == null ? SortInfo.DEFAULT : new SortInfo(sorting.getKey(), SortInfo.SortOrder.valueOf(sorting.getOrder().name()));
    boolean showHidden = cursor != null ? cursor.isShowHidden() : request.isShowHidden();
    MetadataScope scope = cursor != null ? cursor.getScope() : request.getScope();
    MetadataSearchResponse response = search(new io.cdap.cdap.data2.metadata.dataset.SearchRequest(namespaceAndScopes.getFirst(), query, types, sortInfo, cursorOffsetAndLimits.getOffsetToRequest(), cursorOffsetAndLimits.getLimitToRequest(), request.isCursorRequested() ? 1 : 0, cursorOffsetAndLimits.getCursor(), showHidden, namespaceAndScopes.getSecond()), scope);
    // translate results back and limit them to at most what was requested (see above where we add 1)
    int limitToRespond = cursorOffsetAndLimits.getLimitToRespond();
    int offsetToRespond = cursorOffsetAndLimits.getOffsetToRespond();
    List<MetadataRecord> results = response.getResults().stream().limit(limitToRespond).map(record -> {
        Metadata metadata = Metadata.EMPTY;
        for (Map.Entry<MetadataScope, io.cdap.cdap.api.metadata.Metadata> entry : record.getMetadata().entrySet()) {
            Metadata toAdd = new Metadata(entry.getKey(), entry.getValue().getTags(), entry.getValue().getProperties());
            metadata = mergeDisjointMetadata(metadata, toAdd);
        }
        return new MetadataRecord(record.getMetadataEntity(), metadata);
    }).collect(Collectors.toList());
    Cursor newCursor = null;
    if (response.getCursors() != null && !response.getCursors().isEmpty()) {
        String actualCursor = response.getCursors().get(0);
        if (cursor != null) {
            // the new cursor's offset is the previous cursor's offset plus the number of results
            newCursor = new Cursor(cursor, cursor.getOffset() + results.size(), actualCursor);
        } else {
            newCursor = new Cursor(offsetToRespond + results.size(), limitToRespond, showHidden, scope, namespaces, types, sorting == null ? null : sorting.toString(), actualCursor, query);
        }
    }
    // adjust the total results by the difference of requested offset and the true offset that we respond back
    int totalResults = offsetToRespond - cursorOffsetAndLimits.getOffsetToRequest() + response.getTotal();
    return new SearchResponse(request, newCursor == null ? null : newCursor.toString(), offsetToRespond, limitToRespond, totalResults, results);
}

Example 2 with Metadata

use of io.cdap.cdap.spi.metadata.Metadata in project cdap by caskdata.

the class DatasetMetadataStorage method combineChanges.

private MetadataChange combineChanges(MetadataEntity entity, MetadataDataset.Change userChange, MetadataDataset.Change sysChange) {
    Metadata userBefore = new Metadata(USER, userChange.getExisting().getTags(), userChange.getExisting().getProperties());
    Metadata sysBefore = new Metadata(SYSTEM, sysChange.getExisting().getTags(), sysChange.getExisting().getProperties());
    Metadata before = mergeDisjointMetadata(userBefore, sysBefore);
    Metadata userAfter = new Metadata(USER, userChange.getLatest().getTags(), userChange.getLatest().getProperties());
    Metadata sysAfter = new Metadata(SYSTEM, sysChange.getLatest().getTags(), sysChange.getLatest().getProperties());
    Metadata after = mergeDisjointMetadata(userAfter, sysAfter);
    return new MetadataChange(entity, before, after);
}

Example 3 with Metadata

use of io.cdap.cdap.spi.metadata.Metadata in project cdap by caskdata.

the class DatasetMetadataStorage method read.

private Metadata read(MetadataDatasetContext context, Read read) {
    MetadataDataset.Record userMetadata = readScope(context, MetadataScope.USER, read);
    MetadataDataset.Record systemMetadata = readScope(context, MetadataScope.SYSTEM, read);
    return mergeDisjointMetadata(new Metadata(USER, userMetadata.getTags(), userMetadata.getProperties()), new Metadata(SYSTEM, systemMetadata.getTags(), systemMetadata.getProperties()));
}

Example 4 with Metadata

use of io.cdap.cdap.spi.metadata.Metadata in project cdap by caskdata.

the class DatasetMetadataStorage method readScope.

private MetadataDataset.Record readScope(MetadataDatasetContext context, MetadataScope scope, Read read) {
    MetadataEntity entity = read.getEntity();
    if (read.getSelection() == null && (!read.getScopes().contains(scope) || read.getKinds().isEmpty())) {
        return new MetadataDataset.Record(entity);
    }
    Set<ScopedNameOfKind> selectionForScope = null;
    if (read.getSelection() != null) {
        // noinspection ConstantConditions
        selectionForScope = Sets.filter(read.getSelection(), entry -> entry.getScope() == scope);
        if (selectionForScope.isEmpty()) {
            return new MetadataDataset.Record(entity);
        }
    }
    // now we know we must read from the dataset
    MetadataDataset dataset = context.getDataset(scope);
    if (selectionForScope != null) {
        // request is for a specific set of tags and properties
        Set<String> tagsToRead = selectionForScope.stream().filter(entry -> TAG == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
        Set<String> propertiesToRead = selectionForScope.stream().filter(entry -> PROPERTY == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
        Set<String> tags = tagsToRead.isEmpty() ? Collections.emptySet() : Sets.intersection(tagsToRead, dataset.getTags(entity));
        Map<String, String> properties = propertiesToRead.isEmpty() ? Collections.emptyMap() : Maps.filterKeys(dataset.getProperties(entity), propertiesToRead::contains);
        return new MetadataDataset.Record(entity, properties, tags);
    }
    if (MetadataKind.ALL.equals(read.getKinds())) {
        // all metadata kinds requested
        return dataset.getMetadata(entity);
    }
    // exactly one kind is requested
    MetadataKind requestKind = read.getKinds().iterator().next();
    if (requestKind == TAG) {
        return new MetadataDataset.Record(entity, Collections.emptyMap(), dataset.getTags(entity));
    }
    if (requestKind == PROPERTY) {
        return new MetadataDataset.Record(entity, dataset.getProperties(entity), Collections.emptySet());
    }
    throw new IllegalStateException("Encountered metadata read request for unknown kind " + requestKind);
}

Example 5 with Metadata

use of io.cdap.cdap.spi.metadata.Metadata in project cdap by caskdata.

the class MetadataSubscriberService method preProcess.

@Override
protected void preProcess() {
    if (didBackfill) {
        return;
    }
    if (backfillAttempts > 10) {
        LOG.info("Skipping attempt to back-fill plugin metadata after 10 failures.");
        return;
    }
    // Backfill plugin metadata
    backfillAttempts++;
    LOG.info("Starting back-fill process(attempt {}) for plugin metadata", backfillAttempts);
    boolean updateFailed = false;
    NamespaceStore namespaceStore = new DefaultNamespaceStore(this.transactionRunner);
    List<String> namespaces = namespaceStore.list().stream().map(NamespaceMeta::getName).collect(Collectors.toList());
    LOG.debug("Back-filling plugin metadata for {} namespaces", namespaces.size());
    for (String namespace : namespaces) {
        List<ApplicationMeta> apps = TransactionRunners.run(this.transactionRunner, context -> {
            AppMetadataStore appMetadataStore = AppMetadataStore.create(context);
            return appMetadataStore.getAllApplications(namespace);
        });
        LOG.debug("Back-filling plugin metadata for namespace '{}' with {} applications", namespace, apps.size());
        try {
            this.getPluginCounts(namespace, apps);
        } catch (IOException e) {
            updateFailed = true;
            LOG.warn("Failed to write plugin metadata updates for namespace '{}': {}", namespace, e);
        }
    }
    if (!updateFailed) {
        LOG.info("Successfully back-filled plugin metadata for {} namespaces.", namespaces.size());
        didBackfill = true;
        TransactionRunners.run(transactionRunner, (TxRunnable) context -> AppMetadataStore.create(context).persistSubscriberState(getTopicId().getTopic(), BACKFILL_SUBSCRIBER_NAME, "true"));
    }
}