Examples with MetadataEntity io.cdap.cdap.api.metadata.MetadataEntity used on opensource projects

Example 51 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class DefaultMetadataServiceClientTest method testRemove.

@Test
public void testRemove() throws Exception {
    final MetadataEntity removeEntity = MetadataEntity.builder().append("remove", "test").build();
    createMetadataMutation(new MetadataMutation.Create(removeEntity, testMetadata, CREATE_DIRECTIVES));
    // confirm that the create was successful
    Assert.assertEquals(testMetadata.getProperties(MetadataScope.SYSTEM), getMetadataProperties(removeEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(testMetadata.getTags(MetadataScope.SYSTEM), getMetadataTags(removeEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(testMetadata.getProperties(MetadataScope.USER), getMetadataProperties(removeEntity, MetadataScope.USER));
    Assert.assertEquals(testMetadata.getTags(MetadataScope.USER), getMetadataTags(removeEntity, MetadataScope.USER));
    // Remove only USER metadata
    removeMetadataMutation(new MetadataMutation.Remove(removeEntity, MetadataScope.USER));
    Assert.assertEquals(testMetadata.getProperties(MetadataScope.SYSTEM), getMetadataProperties(removeEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(testMetadata.getTags(MetadataScope.SYSTEM), getMetadataTags(removeEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(Collections.EMPTY_MAP, getMetadataProperties(removeEntity, MetadataScope.USER));
    Assert.assertEquals(Collections.EMPTY_SET, getMetadataTags(removeEntity, MetadataScope.USER));
    // Remove all metadata
    removeMetadataMutation(new MetadataMutation.Remove(removeEntity));
    Assert.assertEquals(Collections.EMPTY_MAP, getMetadataProperties(removeEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(Collections.EMPTY_SET, getMetadataTags(removeEntity, MetadataScope.SYSTEM));
    Assert.assertEquals(Collections.EMPTY_MAP, getMetadataProperties(removeEntity, MetadataScope.USER));
    Assert.assertEquals(Collections.EMPTY_SET, getMetadataTags(removeEntity, MetadataScope.USER));
}

Example 52 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class FieldLineageAdminTest method testFieldsWithDsSchema.

@Test
public void testFieldsWithDsSchema() throws Exception {
    FieldLineageAdmin fieldLineageAdmin = new FieldLineageAdmin(new FakeFieldLineageReader(getFieldNames(), Collections.emptySet(), Collections.emptySet()), metadataAdmin);
    EndPoint endPoint = EndPoint.of(NamespaceId.DEFAULT.getNamespace(), "file");
    // test that when there is no schema information present for the dataset and the we request for lineage with
    // includeCurrent set to true we get lineage fields correctly.
    Set<Field> expected = getFields(getFieldNames());
    // includeCurrent set to true
    Set<Field> actual = fieldLineageAdmin.getFields(endPoint, 0, Long.MAX_VALUE, null, true);
    Assert.assertEquals(expected, actual);
    // schema with fields which are different than known to lineage store
    Schema schema = Schema.recordOf("record", Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("address", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("addiffField1", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("diffField2", Schema.nullableOf(Schema.of(Schema.Type.INT))));
    // add the the dataset with the schema with fields known in lineage store
    TableProperties.Builder props = TableProperties.builder();
    TableProperties.setSchema(props, schema);
    TableProperties.setRowFieldName(props, "name");
    DatasetId datasetId = NamespaceId.DEFAULT.dataset("file");
    MetadataEntity entity = datasetId.toMetadataEntity();
    datasetFramework.addInstance("table", datasetId, props.build());
    // wait until the metadata for this dataset has been stored
    Tasks.waitFor(false, () -> metadataAdmin.getProperties(MetadataScope.SYSTEM, entity).isEmpty(), 5, TimeUnit.SECONDS);
    // test all fields expected should have all the fields which was known the lineage store but should not contains
    // any dataset schema field since the includeCurrent is set to false
    expected = getFields(getFieldNames());
    actual = fieldLineageAdmin.getFields(endPoint, 0, Long.MAX_VALUE, null, false);
    Assert.assertEquals(expected, actual);
    // test all fields expected should have all the fields which was known the lineage store and also the fields
    // which were only present in the dataset schema since includeCurrent is set to true.
    // this also test that for the fields which are common in lineage store and dataset schema for example address in
    // this case has their lineage info field set to true as we do have lineage for this field
    expected = getFields(getFieldNames());
    expected.addAll(new HashSet<>(Arrays.asList(new Field("addiffField1", false), new Field("diffField2", false))));
    actual = fieldLineageAdmin.getFields(endPoint, 0, Long.MAX_VALUE, null, true);
    Assert.assertEquals(expected, actual);
    // test fields prefixed with string "add" when includeCurrent not set then the ds field show not show up
    Assert.assertEquals(new HashSet<>(Arrays.asList(new Field("address", true), new Field("address_original", true))), fieldLineageAdmin.getFields(endPoint, 0, Long.MAX_VALUE, "add", false));
    // test fields prefixed with string "add" when includeCurrent is set the ds field should also show up
    Assert.assertEquals(new HashSet<>(Arrays.asList(new Field("address", true), new Field("address_original", true), new Field("addiffField1", false))), fieldLineageAdmin.getFields(endPoint, 0, Long.MAX_VALUE, "add", true));
    // test fields prefixed with string "ADD" (case insensitive)
    Assert.assertEquals(new HashSet<>(Arrays.asList(new Field("address", true), new Field("address_original", true), new Field("addiffField1", false))), fieldLineageAdmin.getFields(endPoint, 0, Long.MAX_VALUE, "ADD", true));
}

Example 53 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class DatasetMetadataStorage method replaceInScope.

private MetadataDataset.Change replaceInScope(MetadataDatasetContext context, MetadataScope scope, MetadataEntity entity, Set<String> newTags, Map<String, String> newProperties, Map<ScopedNameOfKind, MetadataDirective> directives) {
    MetadataDataset dataset = context.getDataset(scope);
    MetadataDataset.Record before = dataset.getMetadata(entity);
    if (newTags.isEmpty() && newProperties.isEmpty()) {
        // this scope remains unchanged
        return new MetadataDataset.Change(before, before);
    }
    Set<String> existingTags = before.getTags();
    Set<String> tagsToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == TAG && (entry.getValue() == MetadataDirective.KEEP || entry.getValue() == MetadataDirective.PRESERVE)).map(Map.Entry::getKey).map(ScopedName::getName).filter(existingTags::contains).collect(Collectors.toSet());
    newTags = Sets.union(newTags, tagsToKeepOrPreserve);
    Map<String, String> existingProperties = before.getProperties();
    Map<String, String> propertiesToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == PROPERTY).filter(entry -> existingProperties.containsKey(entry.getKey().getName())).filter(entry -> entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP && !newProperties.containsKey(entry.getKey().getName())).map(Map.Entry::getKey).map(ScopedName::getName).collect(Collectors.toMap(name -> name, existingProperties::get));
    newProperties.putAll(propertiesToKeepOrPreserve);
    Set<String> tagsToRemove = Sets.difference(before.getTags(), newTags);
    Set<String> tagsToAdd = Sets.difference(newTags, before.getTags());
    Set<String> propertiesToRemove = Sets.difference(before.getProperties().keySet(), newProperties.keySet());
    @SuppressWarnings("ConstantConditions") Map<String, String> propertiesToAdd = Maps.filterEntries(newProperties, entry -> !entry.getValue().equals(existingProperties.get(entry.getKey())));
    MetadataDataset.Record after = before;
    if (!tagsToRemove.isEmpty()) {
        after = dataset.removeTags(entity, tagsToRemove).getLatest();
    }
    if (!tagsToAdd.isEmpty()) {
        after = dataset.addTags(entity, tagsToAdd).getLatest();
    }
    if (!propertiesToRemove.isEmpty()) {
        after = dataset.removeProperties(entity, propertiesToRemove).getLatest();
    }
    if (!propertiesToAdd.isEmpty()) {
        after = dataset.addProperties(entity, propertiesToAdd).getLatest();
    }
    return new MetadataDataset.Change(before, after);
}

Example 54 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class SearchHelper method getSortedEntities.

private Set<MetadataEntity> getSortedEntities(List<MetadataEntry> results, SortInfo sortInfo) {
    // in this case, the backing storage is expected to return results in the expected order.
    if (SortInfo.SortOrder.WEIGHTED != sortInfo.getSortOrder()) {
        Set<MetadataEntity> entities = new LinkedHashSet<>(results.size());
        for (MetadataEntry metadataEntry : results) {
            entities.add(metadataEntry.getMetadataEntity());
        }
        return entities;
    }
    // if sort order is weighted, score results by weight, and return in descending order of weights
    // Score results
    final Map<MetadataEntity, Integer> weightedResults = new HashMap<>();
    for (MetadataEntry metadataEntry : results) {
        weightedResults.put(metadataEntry.getMetadataEntity(), weightedResults.getOrDefault(metadataEntry.getMetadataEntity(), 0) + 1);
    }
    // Sort the results by score
    List<Map.Entry<MetadataEntity, Integer>> resultList = new ArrayList<>(weightedResults.entrySet());
    resultList.sort(SEARCH_RESULT_DESC_SCORE_COMPARATOR);
    Set<MetadataEntity> result = new LinkedHashSet<>(resultList.size());
    for (Map.Entry<MetadataEntity, Integer> entry : resultList) {
        result.add(entry.getKey());
    }
    return result;
}

Example 55 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class ElasticsearchMetadataStorage method apply.

@Override
public MetadataChange apply(MetadataMutation mutation, MutationOptions options) throws IOException {
    MetadataEntity entity = mutation.getEntity();
    try {
        // repeatedly try to read current metadata, apply the mutation and reindex, until there is no conflict
        return Retries.callWithRetries(() -> {
            VersionedMetadata before = readFromIndex(entity);
            RequestAndChange intermediary = applyMutation(before, mutation);
            executeMutation(mutation.getEntity(), intermediary.getRequest(), options);
            return intermediary.getChange();
        }, retryStrategyOnConflict, e -> e instanceof MetadataConflictException);
    } catch (MetadataConflictException e) {
        throw new MetadataConflictException("After retries: " + e.getRawMessage(), e.getConflictingEntities());
    }
}