Examples with MetadataSearchResultRecord co.cask.cdap.proto.metadata.MetadataSearchResultRecord used on opensource projects

Example 1 with MetadataSearchResultRecord

use of co.cask.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSystemScopeArtifacts.

@Test
public void testSystemScopeArtifacts() throws Exception {
    // add a system artifact. currently can't do this through the rest api (by design)
    // so bypass it and use the repository directly
    ArtifactId systemId = NamespaceId.SYSTEM.artifact("wordcount", "1.0.0");
    File systemArtifact = createArtifactJarFile(WordCountApp.class, "wordcount", "1.0.0", new Manifest());
    StandaloneTester tester = STANDALONE.get();
    tester.addSystemArtifact(systemId.getArtifact(), systemId.toId().getVersion(), systemArtifact, null);
    // verify that user metadata can be added for system-scope artifacts
    Map<String, String> userProperties = ImmutableMap.of("systemArtifactKey", "systemArtifactValue");
    Set<String> userTags = ImmutableSet.of();
    addProperties(systemId, userProperties);
    addTags(systemId, userTags);
    // verify that user and system metadata can be retrieved for system-scope artifacts
    Assert.assertEquals(ImmutableSet.of(new MetadataRecord(systemId, MetadataScope.USER, userProperties, userTags), new MetadataRecord(systemId, MetadataScope.SYSTEM, ImmutableMap.of(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, systemId.getEntityName()), ImmutableSet.<String>of())), removeCreationTime(getMetadata(systemId)));
    // verify that system scope artifacts can be returned by a search in the default namespace
    // with no target type
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(systemId)), searchMetadata(NamespaceId.DEFAULT, "system*"));
    // with target type as artifact
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(systemId)), searchMetadata(NamespaceId.DEFAULT, "system*", EntityTypeSimpleName.ARTIFACT));
    // verify that user metadata can be deleted for system-scope artifacts
    removeMetadata(systemId);
    Assert.assertEquals(ImmutableSet.of(new MetadataRecord(systemId, MetadataScope.USER, ImmutableMap.<String, String>of(), ImmutableSet.<String>of()), new MetadataRecord(systemId, MetadataScope.SYSTEM, ImmutableMap.of(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, systemId.getEntityName()), ImmutableSet.<String>of())), removeCreationTime(getMetadata(systemId)));
    artifactClient.delete(systemId);
}

Example 2 with MetadataSearchResultRecord

use of co.cask.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchMetadataDeleteNamespace.

@Test
public void testSearchMetadataDeleteNamespace() throws Exception {
    NamespaceId namespace = new NamespaceId("ns2");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    // Deploy app
    appClient.deploy(namespace, createAppJarFile(WordCountApp.class, WordCountApp.class.getSimpleName(), "1.0"));
    Set<String> tags = ImmutableSet.of("tag1", "tag2");
    ArtifactId artifact = namespace.artifact("WordCountApp", "1.0");
    ApplicationId app = namespace.app("WordCountApp");
    ProgramId flow = app.flow("WordCountFlow");
    ProgramId service = app.service("WordFrequencyService");
    StreamId stream = namespace.stream("text");
    DatasetId datasetInstance = namespace.dataset("mydataset");
    StreamViewId view = stream.view("view");
    streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("csv", null, null)));
    // Add metadata
    addTags(app, tags);
    addTags(flow, tags);
    addTags(stream, tags);
    addTags(datasetInstance, tags);
    addTags(view, tags);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(view)), searchMetadata(namespace, "text"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(datasetInstance)), searchMetadata(namespace, "mydataset"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(app), new MetadataSearchResultRecord(flow), new MetadataSearchResultRecord(artifact), new MetadataSearchResultRecord(service)), searchMetadata(namespace, "word*"));
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(app), new MetadataSearchResultRecord(flow), new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(datasetInstance), new MetadataSearchResultRecord(view)), searchMetadata(namespace, "tag1"));
    // Delete namespace
    namespaceClient.delete(namespace);
    // Assert no metadata
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "text"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "mydataset"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "word*"));
    Assert.assertEquals(ImmutableSet.of(), searchMetadata(namespace, "tag1"));
}

Example 3 with MetadataSearchResultRecord

use of co.cask.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchResultPaginationWithTargetType.

@Test
public void testSearchResultPaginationWithTargetType() throws Exception {
    // note that the ordering of the entity creations and the sort param used in this test case matter, in order to
    // reproduce the scenario that caused the issue CDAP-7881
    NamespaceId namespace = new NamespaceId("pagination_with_target_type");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    StreamId stream1 = namespace.stream("text1");
    StreamId stream2 = namespace.stream("text2");
    DatasetId trackerDataset = namespace.dataset("_auditLog");
    DatasetId mydataset = namespace.dataset("mydataset");
    // the creation order below will determine how we see the entities in search result sorted by entity creation time
    // in ascending order
    datasetClient.create(trackerDataset, new DatasetInstanceConfiguration(Table.class.getName(), Collections.<String, String>emptyMap()));
    datasetClient.create(mydataset, new DatasetInstanceConfiguration(Table.class.getName(), Collections.<String, String>emptyMap()));
    // create entities so system metadata is annotated
    streamClient.create(stream1);
    streamClient.create(stream2);
    // do sorting with creation time here since the testSearchResultPagination does with entity name
    // the sorted result order _auditLog mydataset text2 text1 (ascending: creation from earliest time)
    String sort = AbstractSystemMetadataWriter.CREATION_TIME_KEY + " " + SortInfo.SortOrder.ASC;
    // offset 1, limit 2, 2 cursors, should return 2nd result, with 0 cursors since we don't have enough data
    // set showHidden to true which will show the trackerDataset but will not be in search response since its not stream
    MetadataSearchResponse searchResponse = searchMetadata(namespace, "*", ImmutableSet.of(EntityTypeSimpleName.STREAM), sort, 1, 2, 2, null, true);
    List<MetadataSearchResultRecord> expectedResults = ImmutableList.of(new MetadataSearchResultRecord(stream2));
    List<String> expectedCursors = ImmutableList.of();
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertEquals(expectedCursors, searchResponse.getCursors());
    // offset 1, limit 2, 2 cursors, should return just the dataset created above other than trackerDataset even
    // though it was created before since showHidden is false and it should not affect pagination
    searchResponse = searchMetadata(namespace, "*", ImmutableSet.of(EntityTypeSimpleName.DATASET), sort, 0, 2, 2, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(mydataset));
    expectedCursors = ImmutableList.of();
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertEquals(expectedCursors, searchResponse.getCursors());
}

Example 4 with MetadataSearchResultRecord

use of co.cask.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchMetadata.

@Test
public void testSearchMetadata() throws Exception {
    appClient.deploy(NamespaceId.DEFAULT, createAppJarFile(AllProgramsApp.class));
    Map<NamespacedEntityId, Metadata> expectedUserMetadata = new HashMap<>();
    // Add metadata to app
    Map<String, String> props = ImmutableMap.of("key1", "value1");
    Set<String> tags = ImmutableSet.of("tag1", "tag2");
    ApplicationId appId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    addProperties(appId, props);
    addTags(appId, tags);
    expectedUserMetadata.put(appId, new Metadata(props, tags));
    // Add metadata to stream
    props = ImmutableMap.of("key10", "value10", "key11", "value11");
    tags = ImmutableSet.of("tag11");
    StreamId streamId = NamespaceId.DEFAULT.stream(AllProgramsApp.STREAM_NAME);
    addProperties(streamId, props);
    addTags(streamId, tags);
    expectedUserMetadata.put(streamId, new Metadata(props, tags));
    Set<MetadataSearchResultRecord> results = super.searchMetadata(NamespaceId.DEFAULT, "value*", ImmutableSet.<EntityTypeSimpleName>of());
    // Verify results
    Assert.assertEquals(expectedUserMetadata.keySet(), getEntities(results));
    for (MetadataSearchResultRecord result : results) {
        // User metadata has to match exactly since we know what we have set
        Assert.assertEquals(expectedUserMetadata.get(result.getEntityId()), result.getMetadata().get(MetadataScope.USER));
        // Make sure system metadata is returned, we cannot check for exact match since we haven't set it
        Metadata systemMetadata = result.getMetadata().get(MetadataScope.SYSTEM);
        Assert.assertNotNull(systemMetadata);
        Assert.assertFalse(systemMetadata.getProperties().isEmpty());
        Assert.assertFalse(systemMetadata.getTags().isEmpty());
    }
}

Example 5 with MetadataSearchResultRecord

use of co.cask.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method assertDataEntitySearch.

private void assertDataEntitySearch() throws Exception {
    DatasetId datasetInstance = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME);
    DatasetId datasetInstance2 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME2);
    DatasetId datasetInstance3 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME3);
    DatasetId datasetInstance4 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME4);
    DatasetId datasetInstance5 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME5);
    DatasetId datasetInstance6 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME6);
    DatasetId datasetInstance7 = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME7);
    DatasetId dsWithSchema = NamespaceId.DEFAULT.dataset(AllProgramsApp.DS_WITH_SCHEMA_NAME);
    StreamId streamId = NamespaceId.DEFAULT.stream(AllProgramsApp.STREAM_NAME);
    StreamViewId view = streamId.view("view");
    Set<MetadataSearchResultRecord> expected = ImmutableSet.of(new MetadataSearchResultRecord(streamId), new MetadataSearchResultRecord(mystream));
    Set<MetadataSearchResultRecord> expectedWithView = ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expected).add(new MetadataSearchResultRecord(myview)).build();
    // schema search with fieldname
    Set<MetadataSearchResultRecord> metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "body");
    Assert.assertEquals(expectedWithView, metadataSearchResultRecords);
    // schema search with fieldname and fieldtype
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "body:" + Schema.Type.STRING.toString());
    Assert.assertEquals(expected, metadataSearchResultRecords);
    // schema search for partial fieldname
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "bo*");
    Assert.assertEquals(expectedWithView, metadataSearchResultRecords);
    // schema search with fieldname and all/partial fieldtype
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "body:STR*");
    Assert.assertEquals(expected, metadataSearchResultRecords);
    // schema search for a field with the given fieldname:fieldtype
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "body:STRING+field1:STRING");
    Assert.assertEquals(ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expected).add(new MetadataSearchResultRecord(dsWithSchema)).build(), metadataSearchResultRecords);
    // create a view
    Schema viewSchema = Schema.recordOf("record", Schema.Field.of("viewBody", Schema.nullableOf(Schema.of(Schema.Type.BYTES))));
    streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("format", viewSchema)));
    // search all entities that have a defined schema
    // add a user property with "schema" as key
    Map<String, String> datasetProperties = ImmutableMap.of("schema", "schemaValue");
    addProperties(datasetInstance, datasetProperties);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "schema:*");
    Assert.assertEquals(ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expectedWithView).add(new MetadataSearchResultRecord(datasetInstance)).add(new MetadataSearchResultRecord(dsWithSchema)).add(new MetadataSearchResultRecord(view)).build(), metadataSearchResultRecords);
    // search dataset
    ImmutableSet<MetadataSearchResultRecord> expectedKvTables = ImmutableSet.of(new MetadataSearchResultRecord(datasetInstance), new MetadataSearchResultRecord(datasetInstance2), new MetadataSearchResultRecord(datasetInstance3), new MetadataSearchResultRecord(myds));
    ImmutableSet<MetadataSearchResultRecord> expectedExplorableDatasets = ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expectedKvTables).add(new MetadataSearchResultRecord(datasetInstance4)).add(new MetadataSearchResultRecord(datasetInstance5)).add(new MetadataSearchResultRecord(dsWithSchema)).build();
    ImmutableSet<MetadataSearchResultRecord> expectedAllDatasets = ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expectedExplorableDatasets).add(new MetadataSearchResultRecord(datasetInstance6)).add(new MetadataSearchResultRecord(datasetInstance7)).build();
    ImmutableSet<MetadataSearchResultRecord> expectedExplorables = ImmutableSet.<MetadataSearchResultRecord>builder().addAll(expectedExplorableDatasets).add(new MetadataSearchResultRecord(streamId)).add(new MetadataSearchResultRecord(mystream)).build();
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "explore");
    Assert.assertEquals(expectedExplorables, metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, KeyValueTable.class.getName());
    Assert.assertEquals(expectedKvTables, metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "type:*");
    Assert.assertEquals(expectedAllDatasets, metadataSearchResultRecords);
    // search using ttl
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "ttl:*");
    Assert.assertEquals(expected, metadataSearchResultRecords);
    // search using names
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.STREAM_NAME);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(streamId), new MetadataSearchResultRecord(view)), metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.STREAM_NAME, EntityTypeSimpleName.STREAM);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(streamId)), metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.STREAM_NAME, EntityTypeSimpleName.VIEW);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(view)), metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, "view", EntityTypeSimpleName.VIEW);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(view)), metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.DATASET_NAME);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(datasetInstance)), metadataSearchResultRecords);
    metadataSearchResultRecords = searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.DS_WITH_SCHEMA_NAME);
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(dsWithSchema)), metadataSearchResultRecords);
}