Examples with MetadataSearchResultRecord co.cask.cdap.proto.metadata.MetadataSearchResultRecord used on opensource projects

Example 16 with MetadataSearchResultRecord

use of co.cask.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method assertAppSearch.

private void assertAppSearch(ApplicationId app, ArtifactId artifact) throws Exception {
    // using app name
    ImmutableSet<MetadataSearchResultRecord> expected = ImmutableSet.of(new MetadataSearchResultRecord(app));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.NAME));
    // using artifact name: both app and artifact should match
    Assert.assertEquals(ImmutableSet.of(new MetadataSearchResultRecord(app), new MetadataSearchResultRecord(artifact)), searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.class.getSimpleName()));
    // using program names
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.NoOpFlow.NAME, EntityTypeSimpleName.APP));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.NoOpMR.NAME, EntityTypeSimpleName.APP));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.NoOpService.NAME, EntityTypeSimpleName.APP));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.NoOpSpark.NAME, EntityTypeSimpleName.APP));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.NoOpWorker.NAME, EntityTypeSimpleName.APP));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.NoOpWorkflow.NAME, EntityTypeSimpleName.APP));
    // using program types
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, ProgramType.FLOW.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + "*", EntityTypeSimpleName.APP));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, ProgramType.MAPREDUCE.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + "*", EntityTypeSimpleName.APP));
    Assert.assertEquals(ImmutableSet.builder().addAll(expected).add(new MetadataSearchResultRecord(application)).build(), searchMetadata(NamespaceId.DEFAULT, ProgramType.SERVICE.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + "*", EntityTypeSimpleName.APP));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, ProgramType.SPARK.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + "*", EntityTypeSimpleName.APP));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, ProgramType.WORKER.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + "*", EntityTypeSimpleName.APP));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, ProgramType.WORKFLOW.getPrettyName() + MetadataDataset.KEYVALUE_SEPARATOR + "*", EntityTypeSimpleName.APP));
    // using schedule
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, AllProgramsApp.SCHEDULE_NAME));
    Assert.assertEquals(expected, searchMetadata(NamespaceId.DEFAULT, "EveryMinute"));
}

Example 17 with MetadataSearchResultRecord

use of co.cask.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testTags.

@Test
public void testTags() throws Exception {
    // should fail because we haven't provided any metadata in the request
    addTags(application, null, BadRequestException.class);
    Set<String> appTags = ImmutableSet.of("aTag", "aT", "Wow-WOW1", "WOW_WOW2");
    addTags(application, appTags);
    // should fail because we haven't provided any metadata in the request
    addTags(pingService, null, BadRequestException.class);
    Set<String> serviceTags = ImmutableSet.of("sTag", "sT");
    addTags(pingService, serviceTags);
    addTags(myds, null, BadRequestException.class);
    Set<String> datasetTags = ImmutableSet.of("dTag", "dT");
    addTags(myds, datasetTags);
    addTags(mystream, null, BadRequestException.class);
    Set<String> streamTags = ImmutableSet.of("stTag", "stT", "Wow-WOW1", "WOW_WOW2");
    addTags(mystream, streamTags);
    addTags(myview, null, BadRequestException.class);
    Set<String> viewTags = ImmutableSet.of("viewTag", "viewT");
    addTags(myview, viewTags);
    Set<String> artifactTags = ImmutableSet.of("rTag", "rT");
    addTags(artifactId, artifactTags);
    // retrieve tags and verify
    Set<String> tags = getTags(application, MetadataScope.USER);
    Assert.assertTrue(tags.containsAll(appTags));
    Assert.assertTrue(appTags.containsAll(tags));
    tags = getTags(pingService, MetadataScope.USER);
    Assert.assertTrue(tags.containsAll(serviceTags));
    Assert.assertTrue(serviceTags.containsAll(tags));
    tags = getTags(myds, MetadataScope.USER);
    Assert.assertTrue(tags.containsAll(datasetTags));
    Assert.assertTrue(datasetTags.containsAll(tags));
    tags = getTags(mystream, MetadataScope.USER);
    Assert.assertTrue(tags.containsAll(streamTags));
    Assert.assertTrue(streamTags.containsAll(tags));
    tags = getTags(myview, MetadataScope.USER);
    Assert.assertTrue(tags.containsAll(viewTags));
    Assert.assertTrue(viewTags.containsAll(tags));
    tags = getTags(artifactId, MetadataScope.USER);
    Assert.assertTrue(tags.containsAll(artifactTags));
    Assert.assertTrue(artifactTags.containsAll(tags));
    // test search for stream
    Set<MetadataSearchResultRecord> searchTags = searchMetadata(NamespaceId.DEFAULT, "stT", EntityTypeSimpleName.STREAM);
    Set<MetadataSearchResultRecord> expected = ImmutableSet.of(new MetadataSearchResultRecord(mystream));
    Assert.assertEquals(expected, searchTags);
    searchTags = searchMetadata(NamespaceId.DEFAULT, "Wow", EntityTypeSimpleName.STREAM);
    expected = ImmutableSet.of(new MetadataSearchResultRecord(mystream));
    Assert.assertEquals(expected, searchTags);
    // test search for view with lowercase tag when metadata was stored in mixed case
    searchTags = searchMetadata(NamespaceId.DEFAULT, "viewtag", EntityTypeSimpleName.VIEW);
    expected = ImmutableSet.of(new MetadataSearchResultRecord(myview));
    Assert.assertEquals(expected, searchTags);
    // test prefix search, should match stream and application
    searchTags = searchMetadata(NamespaceId.DEFAULT, "Wow*", EntityTypeSimpleName.ALL);
    expected = ImmutableSet.of(new MetadataSearchResultRecord(application), new MetadataSearchResultRecord(mystream));
    Assert.assertEquals(expected, searchTags);
    // search without any target param
    searchTags = searchMetadata(NamespaceId.DEFAULT, "Wow*");
    Assert.assertEquals(expected, searchTags);
    // search non-existent tags should return empty set
    searchTags = searchMetadata(NamespaceId.DEFAULT, "NullKey");
    Assert.assertEquals(ImmutableSet.<MetadataSearchResultRecord>of(), searchTags);
    // test removal
    removeTag(application, "aTag");
    Assert.assertEquals(ImmutableSet.of("aT", "Wow-WOW1", "WOW_WOW2"), getTags(application, MetadataScope.USER));
    removeTags(pingService);
    Assert.assertTrue(getTags(pingService, MetadataScope.USER).isEmpty());
    removeTags(pingService);
    Assert.assertTrue(getTags(pingService, MetadataScope.USER).isEmpty());
    removeTag(myds, "dT");
    Assert.assertEquals(ImmutableSet.of("dTag"), getTags(myds, MetadataScope.USER));
    removeTag(mystream, "stT");
    removeTag(mystream, "stTag");
    removeTag(mystream, "Wow-WOW1");
    removeTag(mystream, "WOW_WOW2");
    removeTag(myview, "viewT");
    removeTag(myview, "viewTag");
    Assert.assertTrue(getTags(mystream, MetadataScope.USER).isEmpty());
    removeTag(artifactId, "rTag");
    removeTag(artifactId, "rT");
    Assert.assertTrue(getTags(artifactId, MetadataScope.USER).isEmpty());
    // cleanup
    removeTags(application);
    removeTags(pingService);
    removeTags(myds);
    removeTags(mystream);
    removeTags(myview);
    removeTags(artifactId);
    Assert.assertTrue(getTags(application, MetadataScope.USER).isEmpty());
    Assert.assertTrue(getTags(pingService, MetadataScope.USER).isEmpty());
    Assert.assertTrue(getTags(myds, MetadataScope.USER).isEmpty());
    Assert.assertTrue(getTags(mystream, MetadataScope.USER).isEmpty());
    Assert.assertTrue(getTags(artifactId, MetadataScope.USER).isEmpty());
    // non-existing namespace
    addTags(nonExistingApp, appTags, NotFoundException.class);
    addTags(nonExistingService, serviceTags, NotFoundException.class);
    addTags(nonExistingDataset, datasetTags, NotFoundException.class);
    addTags(nonExistingStream, streamTags, NotFoundException.class);
    addTags(nonExistingView, streamTags, NotFoundException.class);
    addTags(nonExistingArtifact, artifactTags, NotFoundException.class);
}

Example 18 with MetadataSearchResultRecord

use of co.cask.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataStoreTest method testSearchWeight.

@Test
public void testSearchWeight() throws Exception {
    ProgramId flow1 = new ProgramId("ns1", "app1", ProgramType.FLOW, "flow1");
    StreamId stream1 = new StreamId("ns1", "s1");
    DatasetId dataset1 = new DatasetId("ns1", "ds1");
    // Add metadata
    String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    Map<String, String> flowUserProps = ImmutableMap.of("key1", "value1", "key2", "value2", "multiword", multiWordValue);
    Map<String, String> flowSysProps = ImmutableMap.of("sysKey1", "sysValue1");
    Set<String> flowUserTags = ImmutableSet.of("tag1", "tag2");
    Set<String> streamUserTags = ImmutableSet.of("tag3", "tag4");
    Set<String> flowSysTags = ImmutableSet.of("sysTag1");
    store.setProperties(MetadataScope.USER, flow1, flowUserProps);
    store.setProperties(MetadataScope.SYSTEM, flow1, flowSysProps);
    store.addTags(MetadataScope.USER, flow1, flowUserTags.toArray(new String[flowUserTags.size()]));
    store.addTags(MetadataScope.SYSTEM, flow1, flowSysTags.toArray(new String[flowSysTags.size()]));
    store.addTags(MetadataScope.USER, stream1, streamUserTags.toArray(new String[streamUserTags.size()]));
    store.removeTags(MetadataScope.USER, stream1, streamUserTags.toArray(new String[streamUserTags.size()]));
    store.setProperties(MetadataScope.USER, stream1, flowUserProps);
    store.removeProperties(MetadataScope.USER, stream1, "key1", "key2", "multiword");
    Map<String, String> streamUserProps = ImmutableMap.of("sKey1", "sValue1 sValue2", "Key1", "Value1");
    store.setProperties(MetadataScope.USER, stream1, streamUserProps);
    Map<String, String> datasetUserProps = ImmutableMap.of("sKey1", "sValuee1 sValuee2");
    store.setProperties(MetadataScope.USER, dataset1, datasetUserProps);
    // Test score and metadata match
    MetadataSearchResponse response = search("ns1", "value1 multiword:av2");
    Assert.assertEquals(2, response.getTotal());
    List<MetadataSearchResultRecord> actual = Lists.newArrayList(response.getResults());
    Map<MetadataScope, Metadata> expectedFlowMetadata = ImmutableMap.of(MetadataScope.USER, new Metadata(flowUserProps, flowUserTags), MetadataScope.SYSTEM, new Metadata(flowSysProps, flowSysTags));
    Map<MetadataScope, Metadata> expectedStreamMetadata = ImmutableMap.of(MetadataScope.USER, new Metadata(streamUserProps, Collections.<String>emptySet()));
    Map<MetadataScope, Metadata> expectedDatasetMetadata = ImmutableMap.of(MetadataScope.USER, new Metadata(datasetUserProps, Collections.<String>emptySet()));
    List<MetadataSearchResultRecord> expected = Lists.newArrayList(new MetadataSearchResultRecord(flow1, expectedFlowMetadata), new MetadataSearchResultRecord(stream1, expectedStreamMetadata));
    Assert.assertEquals(expected, actual);
    response = search("ns1", "value1 sValue*");
    Assert.assertEquals(3, response.getTotal());
    actual = Lists.newArrayList(response.getResults());
    expected = Lists.newArrayList(new MetadataSearchResultRecord(stream1, expectedStreamMetadata), new MetadataSearchResultRecord(dataset1, expectedDatasetMetadata), new MetadataSearchResultRecord(flow1, expectedFlowMetadata));
    Assert.assertEquals(expected, actual);
    response = search("ns1", "*");
    Assert.assertEquals(3, response.getTotal());
    actual = Lists.newArrayList(response.getResults());
    Assert.assertTrue(actual.containsAll(expected));
}

Example 19 with MetadataSearchResultRecord

use of co.cask.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataStoreTest method testSearchPagination.

// Tests pagination for search results queries that do not have indexes stored in sorted order
// The pagination for these queries is done in DefaultMetadataStore, so adding this test here.
@Test
public void testSearchPagination() throws BadRequestException {
    NamespaceId ns = new NamespaceId("ns");
    ProgramId flow = ns.app("app").flow("flow");
    StreamId stream = ns.stream("stream");
    DatasetId dataset = ns.dataset("dataset");
    // add a dummy entity which starts with _ to test that it doesnt show up see: CDAP-7910
    DatasetId trackerDataset = ns.dataset("_auditLog");
    store.addTags(MetadataScope.USER, flow, "tag", "tag1");
    store.addTags(MetadataScope.USER, stream, "tag2", "tag3 tag4");
    store.addTags(MetadataScope.USER, dataset, "tag5 tag6", "tag7 tag8");
    // add bunch of tags to ensure higher weight to this entity in search result
    store.addTags(MetadataScope.USER, trackerDataset, "tag9", "tag10", "tag11", "tag12", "tag13");
    MetadataSearchResultRecord flowSearchResult = new MetadataSearchResultRecord(flow);
    MetadataSearchResultRecord streamSearchResult = new MetadataSearchResultRecord(stream);
    MetadataSearchResultRecord datasetSearchResult = new MetadataSearchResultRecord(dataset);
    MetadataSearchResultRecord trackerDatasetSearchResult = new MetadataSearchResultRecord(trackerDataset);
    // relevance order for searchQuery "tag*" is trackerDataset, dataset, stream, flow
    // (this depends on how many tags got matched with the search query)
    // trackerDataset entity should not be part
    MetadataSearchResponse response = search(ns.getNamespace(), "tag*", 0, Integer.MAX_VALUE, 1);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.of(datasetSearchResult, streamSearchResult, flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    // trackerDataset entity should be be part since showHidden is true
    response = search(ns.getNamespace(), "tag*", 0, Integer.MAX_VALUE, 1, true);
    Assert.assertEquals(4, response.getTotal());
    Assert.assertEquals(ImmutableList.of(trackerDatasetSearchResult, datasetSearchResult, streamSearchResult, flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    response = search(ns.getNamespace(), "tag*", 0, 2, 1);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.of(datasetSearchResult, streamSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    // skipping trackerDataset should not affect the offset
    response = search(ns.getNamespace(), "tag*", 1, 2, 1);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.of(streamSearchResult, flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    // if showHidden is true trackerDataset should affect the offset
    response = search(ns.getNamespace(), "tag*", 1, 3, 1, true);
    Assert.assertEquals(4, response.getTotal());
    Assert.assertEquals(ImmutableList.of(datasetSearchResult, streamSearchResult, flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    response = search(ns.getNamespace(), "tag*", 2, 2, 1);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.of(flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    response = search(ns.getNamespace(), "tag*", 4, 2, 1);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.<MetadataSearchResultRecord>of(), ImmutableList.copyOf(stripMetadata(response.getResults())));
    // ensure overflow bug is squashed (JIRA: CDAP-8133)
    response = search(ns.getNamespace(), "tag*", 1, Integer.MAX_VALUE, 0);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.of(streamSearchResult, flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
}