Examples with MetadataChange io.cdap.cdap.spi.metadata.MetadataChange used on opensource projects

Example 1 with MetadataChange

use of io.cdap.cdap.spi.metadata.MetadataChange in project cdap by caskdata.

the class DatasetMetadataStorage method combineChanges.

private MetadataChange combineChanges(MetadataEntity entity, MetadataDataset.Change userChange, MetadataDataset.Change sysChange) {
    Metadata userBefore = new Metadata(USER, userChange.getExisting().getTags(), userChange.getExisting().getProperties());
    Metadata sysBefore = new Metadata(SYSTEM, sysChange.getExisting().getTags(), sysChange.getExisting().getProperties());
    Metadata before = mergeDisjointMetadata(userBefore, sysBefore);
    Metadata userAfter = new Metadata(USER, userChange.getLatest().getTags(), userChange.getLatest().getProperties());
    Metadata sysAfter = new Metadata(SYSTEM, sysChange.getLatest().getTags(), sysChange.getLatest().getProperties());
    Metadata after = mergeDisjointMetadata(userAfter, sysAfter);
    return new MetadataChange(entity, before, after);
}

Example 2 with MetadataChange

use of io.cdap.cdap.spi.metadata.MetadataChange in project cdap by caskdata.

the class AuditMetadataStorage method apply.

@Override
public MetadataChange apply(MetadataMutation mutation, MutationOptions options) throws IOException {
    MetadataChange change;
    try {
        change = storage.apply(mutation, options);
        emitMetrics(MUTATION_COUNT_MAP.get(mutation.getType()));
    } catch (Exception e) {
        emitMetrics(MUTATION_ERROR_MAP.get(mutation.getType()));
        throw e;
    }
    publishAudit(change);
    return change;
}

Example 3 with MetadataChange

use of io.cdap.cdap.spi.metadata.MetadataChange in project cdap by caskdata.

the class ElasticsearchMetadataStorage method batch.

@Override
public List<MetadataChange> batch(List<? extends MetadataMutation> mutations, MutationOptions options) throws IOException {
    if (mutations.isEmpty()) {
        return Collections.emptyList();
    }
    if (mutations.size() == 1) {
        return Collections.singletonList(apply(mutations.get(0), options));
    }
    // first detect whether there are duplicate entity ids. If so, execute in sequence
    Set<MetadataEntity> entities = new HashSet<>();
    LinkedHashMap<MetadataEntity, MetadataMutation> mutationMap = new LinkedHashMap<>(mutations.size());
    boolean duplicate = false;
    for (MetadataMutation mutation : mutations) {
        if (!entities.add(mutation.getEntity())) {
            duplicate = true;
            break;
        }
        mutationMap.put(mutation.getEntity(), mutation);
    }
    if (duplicate) {
        // if there are multiple mutations for the same entity, execute all in sequence
        List<MetadataChange> changes = new ArrayList<>(mutations.size());
        for (MetadataMutation mutation : mutations) {
            changes.add(apply(mutation, options));
        }
        return changes;
    }
    // collect all changes in an order-preserving map. The first time doBatch() is called, it will
    // enter all entities in the map. Every time it is retried, the change may get updated, but that
    // will not change the order of the map.
    LinkedHashMap<MetadataEntity, MetadataChange> changes = new LinkedHashMap<>(mutations.size());
    try {
        // repeatedly try to read current metadata, apply the mutations and reindex, until there is no conflict
        return Retries.callWithRetries(() -> doBatch(mutationMap, changes, options), RetryStrategies.limit(50, RetryStrategies.fixDelay(100, TimeUnit.MILLISECONDS)), e -> e instanceof MetadataConflictException);
    } catch (MetadataConflictException e) {
        throw new MetadataConflictException("After retries: " + e.getRawMessage(), e.getConflictingEntities());
    }
}

Example 4 with MetadataChange

use of io.cdap.cdap.spi.metadata.MetadataChange in project cdap by caskdata.

the class ElasticsearchMetadataStorage method create.

/**
 * Creates the Elasticsearch index request for an entity creation or update.
 * See {@link MetadataMutation.Create} for detailed semantics.
 *
 * @param before the metadata for the mutation's entity before the change
 * @param create the mutation to apply
 *
 * @return an ElasticSearch request to be executed, and the change caused by the mutation
 */
private RequestAndChange create(VersionedMetadata before, MetadataMutation.Create create) {
    // if the entity did not exist before, none of the directives apply and this is equivalent to update()
    if (!before.existing()) {
        return update(create.getEntity(), before, create.getMetadata());
    }
    Metadata meta = create.getMetadata();
    Map<ScopedNameOfKind, MetadataDirective> directives = create.getDirectives();
    // determine the scopes that this mutation applies to (scopes that do not occur in the metadata are no changed)
    Set<MetadataScope> scopes = Stream.concat(meta.getTags().stream(), meta.getProperties().keySet().stream()).map(ScopedName::getScope).collect(Collectors.toSet());
    // compute what previously existing tags and properties have to be preserved (all others are replaced)
    Set<ScopedName> existingTagsToKeep = new HashSet<>();
    Map<ScopedName, String> existingPropertiesToKeep = new HashMap<>();
    // all tags and properties that are in a scope not affected by this mutation
    Sets.difference(MetadataScope.ALL, scopes).forEach(scope -> {
        before.getMetadata().getTags().stream().filter(tag -> tag.getScope().equals(scope)).forEach(existingTagsToKeep::add);
        before.getMetadata().getProperties().entrySet().stream().filter(entry -> entry.getKey().getScope().equals(scope)).forEach(entry -> existingPropertiesToKeep.put(entry.getKey(), entry.getValue()));
    });
    // tags and properties in affected scopes that must be kept or preserved
    directives.entrySet().stream().filter(entry -> scopes.contains(entry.getKey().getScope())).forEach(entry -> {
        ScopedNameOfKind key = entry.getKey();
        if (key.getKind() == MetadataKind.TAG && (entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP)) {
            ScopedName tag = new ScopedName(key.getScope(), key.getName());
            if (!meta.getTags().contains(tag) && before.getMetadata().getTags().contains(tag)) {
                existingTagsToKeep.add(tag);
            }
        } else if (key.getKind() == MetadataKind.PROPERTY) {
            ScopedName property = new ScopedName(key.getScope(), key.getName());
            String existingValue = before.getMetadata().getProperties().get(property);
            String newValue = meta.getProperties().get(property);
            if (existingValue != null && (entry.getValue() == MetadataDirective.PRESERVE && !existingValue.equals(newValue) || entry.getValue() == MetadataDirective.KEEP && newValue == null)) {
                existingPropertiesToKeep.put(property, existingValue);
            }
        }
    });
    // compute the new tags and properties
    Set<ScopedName> newTags = existingTagsToKeep.isEmpty() ? meta.getTags() : Sets.union(meta.getTags(), existingTagsToKeep);
    Map<ScopedName, String> newProperties = meta.getProperties();
    if (!existingPropertiesToKeep.isEmpty()) {
        newProperties = new HashMap<>(newProperties);
        newProperties.putAll(existingPropertiesToKeep);
    }
    Metadata after = new Metadata(newTags, newProperties);
    return new RequestAndChange(writeToIndex(create.getEntity(), before.getVersion(), after), new MetadataChange(create.getEntity(), before.getMetadata(), after));
}

Example 5 with MetadataChange

use of io.cdap.cdap.spi.metadata.MetadataChange in project cdap by caskdata.

the class ElasticsearchMetadataStorage method update.

/**
 * Creates the Elasticsearch index request for updating the metadata of an entity.
 * This updates or adds the new metadata to the corresponding metadata document in the index.
 *
 * @param before the metadata for the mutation's entity before the change, and its version
 *
 * @return an ElasticSearch request to be executed, and the change caused by the mutation
 */
private RequestAndChange update(MetadataEntity entity, VersionedMetadata before, Metadata updates) {
    Set<ScopedName> tags = new HashSet<>(before.getMetadata().getTags());
    tags.addAll(updates.getTags());
    Map<ScopedName, String> properties = new HashMap<>(before.getMetadata().getProperties());
    properties.putAll(updates.getProperties());
    Metadata after = new Metadata(tags, properties);
    return new RequestAndChange(writeToIndex(entity, before.getVersion(), after), new MetadataChange(entity, before.getMetadata(), after));
}