use of co.cask.cdap.common.metadata.MetadataRecord in project cdap by caskdata.
the class ArtifactHttpHandlerTest method testDeletePropertiesAndArtifacts.
@Test
public void testDeletePropertiesAndArtifacts() throws Exception {
// add 2 versions of the same app that doesn't use config
ArtifactId wordcountId1 = NamespaceId.DEFAULT.artifact("wordcount", "1.0.0");
Assert.assertEquals(HttpResponseStatus.OK.code(), addAppArtifact(Id.Artifact.fromEntityId(wordcountId1), WordCountApp.class).getStatusLine().getStatusCode());
// test get /artifacts endpoint
Set<ArtifactSummary> expectedArtifacts = Sets.newHashSet(new ArtifactSummary("wordcount", "1.0.0"));
Set<ArtifactSummary> actualArtifacts = getArtifacts(NamespaceId.DEFAULT);
Assert.assertEquals(expectedArtifacts, actualArtifacts);
addArtifactProperties(Id.Artifact.fromEntityId(wordcountId1), ImmutableMap.of("key1", "value1", "key2", "value2", "key3", "value3"));
Assert.assertEquals(ImmutableMap.of("key1", "value1", "key2", "value2", "key3", "value3"), getArtifactProperties(wordcountId1));
// delete a single property
deleteArtifact(wordcountId1, false, "key1", 200);
Assert.assertEquals(ImmutableMap.of("key2", "value2", "key3", "value3"), getArtifactProperties(wordcountId1));
// delete all properties
deleteArtifact(wordcountId1, false, null, 200);
Assert.assertEquals(ImmutableMap.of(), getArtifactProperties(wordcountId1));
Set<MetadataRecord> metadataRecords = metadataClient.getMetadata(Id.Artifact.fromEntityId(wordcountId1), MetadataScope.USER);
Assert.assertEquals(1, metadataRecords.size());
Assert.assertEquals(new MetadataRecord(wordcountId1, MetadataScope.USER), metadataRecords.iterator().next());
// delete artifact
deleteArtifact(wordcountId1, true, null, 200);
try {
metadataClient.getMetadata(Id.Artifact.fromEntityId(wordcountId1), MetadataScope.USER);
Assert.fail("Should not reach here");
} catch (NotFoundException e) {
// no-op
}
actualArtifacts = getArtifacts(NamespaceId.DEFAULT);
Assert.assertTrue(actualArtifacts.isEmpty());
}
use of co.cask.cdap.common.metadata.MetadataRecord in project cdap by caskdata.
the class MetadataHttpHandlerTestRun method assertEmptyMetadata.
private void assertEmptyMetadata(Set<MetadataRecord> entityMetadata, @Nullable MetadataScope scope) {
// should have two metadata records - one for each scope, both should have empty properties and tags
int expectedRecords = (scope == null) ? 2 : 1;
Assert.assertEquals(expectedRecords, entityMetadata.size());
for (MetadataRecord metadataRecord : entityMetadata) {
Assert.assertTrue(metadataRecord.getProperties().isEmpty());
Assert.assertTrue(metadataRecord.getTags().isEmpty());
}
}
use of co.cask.cdap.common.metadata.MetadataRecord in project cdap by caskdata.
the class MetadataHttpHandlerTestRun method testSystemMetadataRetrieval.
@Test
public void testSystemMetadataRetrieval() throws Exception {
appClient.deploy(NamespaceId.DEFAULT, createAppJarFile(AllProgramsApp.class));
// verify stream system metadata
StreamId streamId = NamespaceId.DEFAULT.stream(AllProgramsApp.STREAM_NAME);
Set<String> streamSystemTags = getTags(streamId, MetadataScope.SYSTEM);
Assert.assertEquals(ImmutableSet.of(AbstractSystemMetadataWriter.EXPLORE_TAG), streamSystemTags);
Map<String, String> streamSystemProperties = getProperties(streamId, MetadataScope.SYSTEM);
// Verify create time exists, and is within the past hour
Assert.assertTrue("Expected creation time to exist but it does not", streamSystemProperties.containsKey(AbstractSystemMetadataWriter.CREATION_TIME_KEY));
long createTime = Long.parseLong(streamSystemProperties.get(AbstractSystemMetadataWriter.CREATION_TIME_KEY));
Assert.assertTrue("Stream create time should be within the last hour - " + createTime, createTime > System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1));
Assert.assertEquals(ImmutableMap.of(AbstractSystemMetadataWriter.SCHEMA_KEY, Schema.recordOf("stringBody", Schema.Field.of("body", Schema.of(Schema.Type.STRING))).toString(), AbstractSystemMetadataWriter.TTL_KEY, String.valueOf(Long.MAX_VALUE), AbstractSystemMetadataWriter.DESCRIPTION_KEY, "test stream", AbstractSystemMetadataWriter.CREATION_TIME_KEY, String.valueOf(createTime), AbstractSystemMetadataWriter.ENTITY_NAME_KEY, streamId.getEntityName()), streamSystemProperties);
// Update stream properties and verify metadata got updated (except creation time and description)
long newTtl = 100000L;
streamClient.setStreamProperties(streamId, new StreamProperties(newTtl, null, null));
streamSystemProperties = getProperties(streamId, MetadataScope.SYSTEM);
Assert.assertEquals(ImmutableMap.of(AbstractSystemMetadataWriter.SCHEMA_KEY, Schema.recordOf("stringBody", Schema.Field.of("body", Schema.of(Schema.Type.STRING))).toString(), AbstractSystemMetadataWriter.TTL_KEY, String.valueOf(newTtl * 1000), AbstractSystemMetadataWriter.DESCRIPTION_KEY, "test stream", AbstractSystemMetadataWriter.CREATION_TIME_KEY, String.valueOf(createTime), AbstractSystemMetadataWriter.ENTITY_NAME_KEY, streamId.getEntityName()), streamSystemProperties);
Set<MetadataRecord> streamSystemMetadata = getMetadata(streamId, MetadataScope.SYSTEM);
Assert.assertEquals(ImmutableSet.of(new MetadataRecord(streamId, MetadataScope.SYSTEM, streamSystemProperties, streamSystemTags)), streamSystemMetadata);
// create view and verify view system metadata
StreamViewId view = new StreamViewId(streamId.getNamespace(), streamId.getStream(), "view");
Schema viewSchema = Schema.recordOf("record", Schema.Field.of("viewBody", Schema.nullableOf(Schema.of(Schema.Type.BYTES))));
streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("format", viewSchema)));
ImmutableSet<String> viewUserTags = ImmutableSet.of("viewTag");
addTags(view, viewUserTags);
Assert.assertEquals(ImmutableSet.of(new MetadataRecord(view, MetadataScope.USER, ImmutableMap.<String, String>of(), viewUserTags), new MetadataRecord(view, MetadataScope.SYSTEM, ImmutableMap.of(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, view.getEntityName(), AbstractSystemMetadataWriter.SCHEMA_KEY, viewSchema.toString()), ImmutableSet.of(AllProgramsApp.STREAM_NAME))), removeCreationTime(getMetadata(view)));
// verify dataset system metadata
DatasetId datasetInstance = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME);
Set<String> dsSystemTags = getTags(datasetInstance, MetadataScope.SYSTEM);
Assert.assertEquals(ImmutableSet.of(DatasetSystemMetadataWriter.BATCH_TAG, AbstractSystemMetadataWriter.EXPLORE_TAG), dsSystemTags);
Map<String, String> dsSystemProperties = getProperties(datasetInstance, MetadataScope.SYSTEM);
// Verify create time exists, and is within the past hour
Assert.assertTrue("Expected creation time to exist but it does not", dsSystemProperties.containsKey(AbstractSystemMetadataWriter.CREATION_TIME_KEY));
createTime = Long.parseLong(dsSystemProperties.get(AbstractSystemMetadataWriter.CREATION_TIME_KEY));
Assert.assertTrue("Dataset create time should be within the last hour - " + createTime, createTime > System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1));
// Now remove create time and assert all other system properties
Assert.assertEquals(ImmutableMap.of("type", KeyValueTable.class.getName(), AbstractSystemMetadataWriter.DESCRIPTION_KEY, "test dataset", AbstractSystemMetadataWriter.CREATION_TIME_KEY, String.valueOf(createTime), AbstractSystemMetadataWriter.ENTITY_NAME_KEY, datasetInstance.getEntityName()), dsSystemProperties);
// Update properties, and make sure that system metadata gets updated (except create time)
datasetClient.update(datasetInstance, TableProperties.builder().setTTL(100000L).build().getProperties());
dsSystemProperties = getProperties(datasetInstance, MetadataScope.SYSTEM);
Assert.assertEquals(ImmutableMap.of("type", KeyValueTable.class.getName(), AbstractSystemMetadataWriter.DESCRIPTION_KEY, "test dataset", AbstractSystemMetadataWriter.TTL_KEY, "100000", AbstractSystemMetadataWriter.CREATION_TIME_KEY, String.valueOf(createTime), AbstractSystemMetadataWriter.ENTITY_NAME_KEY, datasetInstance.getEntityName()), dsSystemProperties);
// verify artifact metadata
ArtifactId artifactId = getArtifactId();
Assert.assertEquals(ImmutableSet.of(new MetadataRecord(artifactId, MetadataScope.SYSTEM, ImmutableMap.of(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, artifactId.getEntityName()), ImmutableSet.<String>of())), removeCreationTime(getMetadata(artifactId, MetadataScope.SYSTEM)));
// verify app system metadata
ApplicationId app = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
Assert.assertEquals(ImmutableMap.builder().put(ProgramType.FLOW.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpFlow.NAME, AllProgramsApp.NoOpFlow.NAME).put(ProgramType.MAPREDUCE.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpMR.NAME, AllProgramsApp.NoOpMR.NAME).put(ProgramType.MAPREDUCE.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpMR2.NAME, AllProgramsApp.NoOpMR2.NAME).put(ProgramType.SERVICE.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpService.NAME, AllProgramsApp.NoOpService.NAME).put(ProgramType.SPARK.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpSpark.NAME, AllProgramsApp.NoOpSpark.NAME).put(ProgramType.WORKER.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpWorker.NAME, AllProgramsApp.NoOpWorker.NAME).put(ProgramType.WORKFLOW.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.NoOpWorkflow.NAME, AllProgramsApp.NoOpWorkflow.NAME).put(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, app.getEntityName()).put(AbstractSystemMetadataWriter.VERSION_KEY, ApplicationId.DEFAULT_VERSION).put(AbstractSystemMetadataWriter.DESCRIPTION_KEY, AllProgramsApp.DESCRIPTION).put("schedule" + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.SCHEDULE_NAME, AllProgramsApp.SCHEDULE_NAME + MetadataDataset.KEYVALUE_SEPARATOR + AllProgramsApp.SCHEDULE_DESCRIPTION).build(), removeCreationTime(getProperties(app, MetadataScope.SYSTEM)));
Assert.assertEquals(ImmutableSet.of(AllProgramsApp.class.getSimpleName()), getTags(app, MetadataScope.SYSTEM));
// verify program system metadata
assertProgramSystemMetadata(app.flow(AllProgramsApp.NoOpFlow.NAME), "Realtime", AllProgramsApp.NoOpFlow.DESCRIPTION);
assertProgramSystemMetadata(app.worker(AllProgramsApp.NoOpWorker.NAME), "Realtime", null);
assertProgramSystemMetadata(app.service(AllProgramsApp.NoOpService.NAME), "Realtime", null);
assertProgramSystemMetadata(app.mr(AllProgramsApp.NoOpMR.NAME), "Batch", null);
assertProgramSystemMetadata(app.spark(AllProgramsApp.NoOpSpark.NAME), "Batch", null);
assertProgramSystemMetadata(app.workflow(AllProgramsApp.NoOpWorkflow.NAME), "Batch", AllProgramsApp.NoOpWorkflow.DESCRIPTION);
// update dataset properties to add the workflow.local.dataset property to it.
datasetClient.update(datasetInstance, ImmutableMap.of(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY, "true"));
dsSystemTags = getTags(datasetInstance, MetadataScope.SYSTEM);
Assert.assertEquals(ImmutableSet.of(DatasetSystemMetadataWriter.BATCH_TAG, AbstractSystemMetadataWriter.EXPLORE_TAG, DatasetSystemMetadataWriter.LOCAL_DATASET_TAG), dsSystemTags);
}
use of co.cask.cdap.common.metadata.MetadataRecord in project cdap by caskdata.
the class MetadataHttpHandlerTestRun method testSystemScopeArtifacts.
@Test
public void testSystemScopeArtifacts() throws Exception {
// add a system artifact. currently can't do this through the rest api (by design)
// so bypass it and use the repository directly
ArtifactId systemId = NamespaceId.SYSTEM.artifact("wordcount", "1.0.0");
File systemArtifact = createArtifactJarFile(WordCountApp.class, "wordcount", "1.0.0", new Manifest());
StandaloneTester tester = STANDALONE.get();
tester.addSystemArtifact(systemId.getArtifact(), Id.Artifact.fromEntityId(systemId).getVersion(), systemArtifact, null);
// verify that user metadata can be added for system-scope artifacts
Map<String, String> userProperties = ImmutableMap.of("systemArtifactKey", "systemArtifactValue");
Set<String> userTags = ImmutableSet.of();
addProperties(systemId, userProperties);
addTags(systemId, userTags);
// verify that user and system metadata can be retrieved for system-scope artifacts
Assert.assertEquals(ImmutableSet.of(new MetadataRecord(systemId, MetadataScope.USER, userProperties, userTags), new MetadataRecord(systemId, MetadataScope.SYSTEM, ImmutableMap.of(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, systemId.getEntityName()), ImmutableSet.<String>of())), removeCreationTime(getMetadata(systemId)));
// verify that system scope artifacts can be returned by a search in the default namespace
// with no target type
Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(systemId)), searchMetadata(NamespaceId.DEFAULT, "system*"));
// with target type as artifact
Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(systemId)), searchMetadata(NamespaceId.DEFAULT, "system*", EntityTypeSimpleName.ARTIFACT));
// verify that user metadata can be deleted for system-scope artifacts
removeMetadata(systemId);
Assert.assertEquals(ImmutableSet.of(new MetadataRecord(systemId, MetadataScope.USER, ImmutableMap.<String, String>of(), ImmutableSet.<String>of()), new MetadataRecord(systemId, MetadataScope.SYSTEM, ImmutableMap.of(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, systemId.getEntityName()), ImmutableSet.<String>of())), removeCreationTime(getMetadata(systemId)));
artifactClient.delete(systemId);
}
use of co.cask.cdap.common.metadata.MetadataRecord in project cdap by caskdata.
the class MetadataHttpHandlerTestRun method removeCreationTime.
private Set<MetadataRecord> removeCreationTime(Set<MetadataRecord> original) {
MetadataRecord systemRecord = null;
for (MetadataRecord record : original) {
if (MetadataScope.SYSTEM == record.getScope()) {
systemRecord = record;
}
}
Assert.assertNotNull(systemRecord);
removeCreationTime(systemRecord.getProperties());
return original;
}
Aggregations