Examples with MetadataRecord io.cdap.cdap.common.metadata.MetadataRecord used on opensource projects

Example 6 with MetadataRecord

use of io.cdap.cdap.common.metadata.MetadataRecord in project cdap by cdapio.

the class MetadataHttpHandlerTestRun method testSystemScopeArtifacts.

@Test
public void testSystemScopeArtifacts() throws Exception {
    // add a system artifact. currently can't do this through the rest api (by design)
    // so bypass it and use the repository directly
    ArtifactId systemId = NamespaceId.SYSTEM.artifact("app", "1.0.0");
    File systemArtifact = createArtifactJarFile(AllProgramsApp.class, "app", "1.0.0", new Manifest());
    StandaloneTester tester = STANDALONE.get();
    tester.addSystemArtifact(systemId.getArtifact(), Id.Artifact.fromEntityId(systemId).getVersion(), systemArtifact, null);
    // wait until the system metadata has been processed
    Tasks.waitFor(false, () -> getProperties(systemId, MetadataScope.SYSTEM).isEmpty(), 10, TimeUnit.SECONDS);
    // verify that user metadata can be added for system-scope artifacts
    Map<String, String> userProperties = ImmutableMap.of("systemArtifactKey", "systemArtifactValue");
    Set<String> userTags = ImmutableSet.of();
    addProperties(systemId, userProperties);
    addTags(systemId, userTags);
    // verify that user and system metadata can be retrieved for system-scope artifacts
    Assert.assertEquals(ImmutableSet.of(new MetadataRecord(systemId, MetadataScope.USER, userProperties, userTags), new MetadataRecord(systemId, MetadataScope.SYSTEM, ImmutableMap.of(MetadataConstants.ENTITY_NAME_KEY, systemId.getEntityName()), ImmutableSet.of())), removeCreationTime(getMetadata(systemId.toMetadataEntity())));
    // verify that system scope artifacts can be returned by a search in the default namespace
    // with no target type
    MetadataSearchResponse response = searchMetadata(ImmutableList.of(NamespaceId.DEFAULT, NamespaceId.SYSTEM), "system*");
    MetadataEntity expectedEntity = systemId.toMetadataEntity();
    Set<MetadataEntity> actualEntities = new HashSet<>(extractMetadataEntities(response.getResults()));
    Assert.assertTrue(actualEntities.contains(expectedEntity));
    // with target type as artifact
    assertSearch(searchMetadata(ImmutableList.of(NamespaceId.DEFAULT, NamespaceId.SYSTEM), "system*", MetadataEntity.ARTIFACT), systemId);
    // verify that user metadata can be deleted for system-scope artifacts
    removeMetadata(systemId);
    Assert.assertEquals(ImmutableSet.of(new MetadataRecord(systemId, MetadataScope.USER, ImmutableMap.of(), ImmutableSet.of()), new MetadataRecord(systemId, MetadataScope.SYSTEM, ImmutableMap.of(MetadataConstants.ENTITY_NAME_KEY, systemId.getEntityName()), ImmutableSet.of())), removeCreationTime(getMetadata(systemId.toMetadataEntity())));
    artifactClient.delete(systemId);
}

Example 7 with MetadataRecord

use of io.cdap.cdap.common.metadata.MetadataRecord in project cdap by cdapio.

the class MetadataHttpHandlerTestRun method testMetadata.

@Test
public void testMetadata() throws Exception {
    assertCleanState();
    // Remove when nothing exists
    removeAllMetadata();
    assertCleanState();
    // Add some properties and tags
    Map<String, String> appProperties = ImmutableMap.of("aKey", "aValue");
    Map<String, String> serviceProperties = ImmutableMap.of("sKey", "sValue");
    Map<String, String> runProperties = ImmutableMap.of("runKey", "runValue");
    Map<String, String> datasetProperties = ImmutableMap.of("dKey", "dValue");
    Map<String, String> artifactProperties = ImmutableMap.of("rKey", "rValue");
    Map<String, String> fieldProperties = ImmutableMap.of("fKey", "fValue");
    Set<String> appTags = ImmutableSet.of("aTag");
    Set<String> serviceTags = ImmutableSet.of("sTag");
    Set<String> runTags = ImmutableSet.of("runTag");
    Set<String> datasetTags = ImmutableSet.of("dTag");
    Set<String> artifactTags = ImmutableSet.of("rTag");
    Set<String> fieldTags = ImmutableSet.of("fTag");
    addProperties(application, appProperties);
    addProperties(pingService, serviceProperties);
    addProperties(runId, runProperties);
    addProperties(myds, datasetProperties);
    addProperties(artifactId, artifactProperties);
    addProperties(fieldEntity, fieldProperties);
    addTags(application, appTags);
    addTags(pingService, serviceTags);
    addTags(runId, runTags);
    addTags(myds, datasetTags);
    addTags(artifactId, artifactTags);
    addTags(fieldEntity, fieldTags);
    // verify app
    Set<MetadataRecord> metadataRecords = getMetadata(application.toMetadataEntity(), MetadataScope.USER);
    Assert.assertEquals(1, metadataRecords.size());
    MetadataRecord metadata = metadataRecords.iterator().next();
    Assert.assertEquals(MetadataScope.USER, metadata.getScope());
    Assert.assertEquals(application.toMetadataEntity(), metadata.getMetadataEntity());
    Assert.assertEquals(appProperties, metadata.getProperties());
    Assert.assertEquals(appTags, metadata.getTags());
    // verify service
    metadataRecords = getMetadata(pingService.toMetadataEntity(), MetadataScope.USER);
    Assert.assertEquals(1, metadataRecords.size());
    metadata = metadataRecords.iterator().next();
    Assert.assertEquals(MetadataScope.USER, metadata.getScope());
    Assert.assertEquals(pingService.toMetadataEntity(), metadata.getMetadataEntity());
    Assert.assertEquals(serviceProperties, metadata.getProperties());
    Assert.assertEquals(serviceTags, metadata.getTags());
    // We specifically call the getMetadata where there is no aggregation for runId metadata.
    // For aggregated version of test see LineageHttpHandlerTest which calls getMetadataForRun().
    Set<MetadataRecord> runMetadataRecords = getMetadata(runId.toMetadataEntity(), MetadataScope.USER);
    Assert.assertEquals(1, runMetadataRecords.size());
    MetadataRecord runMetadata = runMetadataRecords.iterator().next();
    Assert.assertEquals(MetadataScope.USER, runMetadata.getScope());
    Assert.assertEquals(runId.toMetadataEntity(), runMetadata.getMetadataEntity());
    Assert.assertEquals(runProperties, runMetadata.getProperties());
    Assert.assertEquals(runTags, runMetadata.getTags());
    // verify dataset
    metadataRecords = getMetadata(myds.toMetadataEntity(), MetadataScope.USER);
    Assert.assertEquals(1, metadataRecords.size());
    metadata = metadataRecords.iterator().next();
    Assert.assertEquals(MetadataScope.USER, metadata.getScope());
    Assert.assertEquals(myds.toMetadataEntity(), metadata.getMetadataEntity());
    Assert.assertEquals(datasetProperties, metadata.getProperties());
    Assert.assertEquals(datasetTags, metadata.getTags());
    // verify artifact
    metadataRecords = getMetadata(artifactId.toMetadataEntity(), MetadataScope.USER);
    Assert.assertEquals(1, metadataRecords.size());
    metadata = metadataRecords.iterator().next();
    Assert.assertEquals(MetadataScope.USER, metadata.getScope());
    Assert.assertEquals(artifactId.toMetadataEntity(), metadata.getMetadataEntity());
    Assert.assertEquals(artifactProperties, metadata.getProperties());
    Assert.assertEquals(artifactTags, metadata.getTags());
    // verify custom entity
    metadataRecords = getMetadata(fieldEntity, MetadataScope.USER);
    Assert.assertEquals(1, metadataRecords.size());
    MetadataRecord metadataRecord = metadataRecords.iterator().next();
    Assert.assertEquals(MetadataScope.USER, metadataRecord.getScope());
    Assert.assertEquals(fieldEntity, metadataRecord.getMetadataEntity());
    Assert.assertEquals(fieldProperties, metadataRecord.getProperties());
    Assert.assertEquals(fieldTags, metadataRecord.getTags());
    // cleanup
    removeAllMetadata();
    assertCleanState();
}

Example 8 with MetadataRecord

use of io.cdap.cdap.common.metadata.MetadataRecord in project cdap by cdapio.

the class MetadataHttpHandlerTestRun method assertEmptyMetadata.

private void assertEmptyMetadata(Set<MetadataRecord> entityMetadata) {
    // should have two metadata records - one for each scope, both should have empty properties and tags
    Assert.assertEquals(1, entityMetadata.size());
    for (MetadataRecord metadataRecord : entityMetadata) {
        Assert.assertTrue(metadataRecord.getProperties().isEmpty());
        Assert.assertTrue(metadataRecord.getTags().isEmpty());
    }
}

Example 9 with MetadataRecord

use of io.cdap.cdap.common.metadata.MetadataRecord in project cdap by cdapio.

the class AuditMetadataStorage method publishAudit.

private void publishAudit(MetadataChange change, MetadataScope scope) {
    Map<String, String> propsBefore = change.getBefore().getProperties(scope);
    Map<String, String> propsAfter = change.getAfter().getProperties(scope);
    Set<String> tagsBefore = change.getBefore().getTags(scope);
    Set<String> tagsAfter = change.getAfter().getTags(scope);
    boolean propsChanged = !propsBefore.equals(propsAfter);
    boolean tagsChanged = !tagsBefore.equals(tagsAfter);
    if (!propsChanged && !tagsChanged) {
        // no change to log
        return;
    }
    // previous state is already given
    MetadataRecord previous = new MetadataRecord(change.getEntity(), scope, propsBefore, tagsBefore);
    // compute what was added
    @SuppressWarnings("ConstantConditions") Map<String, String> propsAdded = Maps.filterEntries(propsAfter, entry -> !entry.getValue().equals(propsBefore.get(entry.getKey())));
    Set<String> tagsAdded = Sets.difference(tagsAfter, tagsBefore);
    MetadataRecord additions = new MetadataRecord(change.getEntity(), scope, propsAdded, tagsAdded);
    // compute what was deleted
    @SuppressWarnings("ConstantConditions") Map<String, String> propsDeleted = Maps.filterEntries(propsBefore, entry -> !entry.getValue().equals(propsAfter.get(entry.getKey())));
    Set<String> tagsDeleted = Sets.difference(tagsBefore, tagsAfter);
    MetadataRecord deletions = new MetadataRecord(change.getEntity(), scope, propsDeleted, tagsDeleted);
    // and publish
    MetadataPayload payload = new MetadataPayloadBuilder().addPrevious(previous).addAdditions(additions).addDeletions(deletions).build();
    AuditPublishers.publishAudit(auditPublisher, previous.getMetadataEntity(), AuditType.METADATA_CHANGE, payload);
}

Example 10 with MetadataRecord

use of io.cdap.cdap.common.metadata.MetadataRecord in project cdap by caskdata.

the class GetMetadataCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    MetadataEntity metadataEntity = MetadataCommandHelper.toMetadataEntity(arguments.get(ArgumentName.ENTITY.toString()));
    String scope = arguments.getOptional(ArgumentName.METADATA_SCOPE.toString());
    Set<MetadataRecord> metadata = scope == null ? client.getMetadata(metadataEntity) : client.getMetadata(metadataEntity, MetadataScope.valueOf(scope.toUpperCase()));
    Table table = getTableBuilder().setRows(metadata.stream().map(record -> Lists.newArrayList(record.toString(), Joiner.on("\n").join(record.getTags()), Joiner.on("\n").withKeyValueSeparator(":").join(record.getProperties()), record.getScope().name())).collect(Collectors.toList())).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
}