Examples with MetadataSearchResponse co.cask.cdap.proto.metadata.MetadataSearchResponse used on opensource projects

Example 6 with MetadataSearchResponse

use of co.cask.cdap.proto.metadata.MetadataSearchResponse in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchResultPagination.

@Test
public void testSearchResultPagination() throws Exception {
    NamespaceId namespace = new NamespaceId("pagination");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
    StreamId stream = namespace.stream("text");
    DatasetId dataset = namespace.dataset("mydataset");
    StreamViewId view = stream.view("view");
    DatasetId trackerDataset = namespace.dataset("_auditLog");
    // create entities so system metadata is annotated
    streamClient.create(stream);
    streamViewClient.createOrUpdate(view, new ViewSpecification(new FormatSpecification("csv", null, null)));
    datasetClient.create(dataset, new DatasetInstanceConfiguration(Table.class.getName(), Collections.<String, String>emptyMap()));
    datasetClient.create(trackerDataset, new DatasetInstanceConfiguration(Table.class.getName(), Collections.<String, String>emptyMap()));
    // search with showHidden to true
    EnumSet<EntityTypeSimpleName> targets = EnumSet.allOf(EntityTypeSimpleName.class);
    String sort = AbstractSystemMetadataWriter.ENTITY_NAME_KEY + " asc";
    // search to get all the above entities offset 0, limit interger max  and cursors 0
    MetadataSearchResponse searchResponse = searchMetadata(namespace, "*", targets, sort, 0, Integer.MAX_VALUE, 0, null, true);
    List<MetadataSearchResultRecord> expectedResults = ImmutableList.of(new MetadataSearchResultRecord(trackerDataset), new MetadataSearchResultRecord(dataset), new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(view));
    List<String> expectedCursors = ImmutableList.of();
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertEquals(expectedCursors, searchResponse.getCursors());
    // no offset, limit 1, no cursors
    searchResponse = searchMetadata(namespace, "*", targets, sort, 0, 1, 0, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(dataset));
    expectedCursors = ImmutableList.of();
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertEquals(expectedCursors, searchResponse.getCursors());
    // no offset, limit 1, 2 cursors, should return 1st result, with 2 cursors
    searchResponse = searchMetadata(namespace, "*", targets, sort, 0, 1, 2, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(dataset));
    expectedCursors = ImmutableList.of(stream.getEntityName(), view.getEntityName());
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertEquals(expectedCursors, searchResponse.getCursors());
    // offset 1, limit 1, 2 cursors, should return 2nd result, with only 1 cursor since we don't have enough data
    searchResponse = searchMetadata(namespace, "*", targets, sort, 1, 1, 2, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(stream));
    expectedCursors = ImmutableList.of(view.getEntityName());
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertEquals(expectedCursors, searchResponse.getCursors());
    // offset 2, limit 1, 2 cursors, should return 3rd result, with 0 cursors since we don't have enough data
    searchResponse = searchMetadata(namespace, "*", targets, sort, 2, 1, 2, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(view));
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertTrue(searchResponse.getCursors().isEmpty());
    // offset 3, limit 1, 2 cursors, should 0 results, with 0 cursors since we don't have enough data
    searchResponse = searchMetadata(namespace, "*", targets, sort, 3, 1, 2, null);
    Assert.assertTrue(searchResponse.getResults().isEmpty());
    Assert.assertTrue(searchResponse.getCursors().isEmpty());
    // no offset, no limit, should return everything
    searchResponse = searchMetadata(namespace, "*", targets, sort, 0, Integer.MAX_VALUE, 4, null);
    expectedResults = ImmutableList.of(new MetadataSearchResultRecord(dataset), new MetadataSearchResultRecord(stream), new MetadataSearchResultRecord(view));
    Assert.assertEquals(expectedResults, new ArrayList<>(searchResponse.getResults()));
    Assert.assertTrue(searchResponse.getCursors().isEmpty());
    // cleanup
    namespaceClient.delete(namespace);
}

Example 7 with MetadataSearchResponse

use of co.cask.cdap.proto.metadata.MetadataSearchResponse in project cdap by caskdata.

the class MetadataHttpHandler method searchMetadata.

@GET
@Path("/namespaces/{namespace-id}/metadata/search")
public void searchMetadata(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @QueryParam("query") String searchQuery, @QueryParam("target") List<String> targets, @QueryParam("sort") @DefaultValue("") String sort, @QueryParam("offset") @DefaultValue("0") int offset, // 2147483647 is Integer.MAX_VALUE
@QueryParam("limit") @DefaultValue("2147483647") int limit, @QueryParam("numCursors") @DefaultValue("0") int numCursors, @QueryParam("cursor") @DefaultValue("") String cursor, @QueryParam("showHidden") @DefaultValue("false") boolean showHidden, @Nullable @QueryParam("entityScope") String entityScope) throws Exception {
    if (searchQuery == null || searchQuery.isEmpty()) {
        throw new BadRequestException("query is not specified");
    }
    Set<EntityTypeSimpleName> types = Collections.emptySet();
    if (targets != null) {
        types = ImmutableSet.copyOf(Iterables.transform(targets, STRING_TO_TARGET_TYPE));
    }
    SortInfo sortInfo = SortInfo.of(URLDecoder.decode(sort, "UTF-8"));
    if (SortInfo.DEFAULT.equals(sortInfo)) {
        if (!(cursor.isEmpty()) || 0 != numCursors) {
            throw new BadRequestException("Cursors are not supported when sort info is not specified.");
        }
    }
    try {
        MetadataSearchResponse response = metadataAdmin.search(namespaceId, URLDecoder.decode(searchQuery, "UTF-8"), types, sortInfo, offset, limit, numCursors, cursor, showHidden, validateEntityScope(entityScope));
        responder.sendJson(HttpResponseStatus.OK, response, MetadataSearchResponse.class, GSON);
    } catch (Exception e) {
        // if MetadataDataset throws an exception, it gets wrapped
        if (Throwables.getRootCause(e) instanceof BadRequestException) {
            throw new BadRequestException(e.getMessage(), e);
        }
        throw e;
    }
}

Example 8 with MetadataSearchResponse

use of co.cask.cdap.proto.metadata.MetadataSearchResponse in project cdap by caskdata.

the class MetadataStoreTest method testSearchWeight.

@Test
public void testSearchWeight() throws Exception {
    ProgramId flow1 = new ProgramId("ns1", "app1", ProgramType.FLOW, "flow1");
    StreamId stream1 = new StreamId("ns1", "s1");
    DatasetId dataset1 = new DatasetId("ns1", "ds1");
    // Add metadata
    String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    Map<String, String> flowUserProps = ImmutableMap.of("key1", "value1", "key2", "value2", "multiword", multiWordValue);
    Map<String, String> flowSysProps = ImmutableMap.of("sysKey1", "sysValue1");
    Set<String> flowUserTags = ImmutableSet.of("tag1", "tag2");
    Set<String> streamUserTags = ImmutableSet.of("tag3", "tag4");
    Set<String> flowSysTags = ImmutableSet.of("sysTag1");
    store.setProperties(MetadataScope.USER, flow1, flowUserProps);
    store.setProperties(MetadataScope.SYSTEM, flow1, flowSysProps);
    store.addTags(MetadataScope.USER, flow1, flowUserTags.toArray(new String[flowUserTags.size()]));
    store.addTags(MetadataScope.SYSTEM, flow1, flowSysTags.toArray(new String[flowSysTags.size()]));
    store.addTags(MetadataScope.USER, stream1, streamUserTags.toArray(new String[streamUserTags.size()]));
    store.removeTags(MetadataScope.USER, stream1, streamUserTags.toArray(new String[streamUserTags.size()]));
    store.setProperties(MetadataScope.USER, stream1, flowUserProps);
    store.removeProperties(MetadataScope.USER, stream1, "key1", "key2", "multiword");
    Map<String, String> streamUserProps = ImmutableMap.of("sKey1", "sValue1 sValue2", "Key1", "Value1");
    store.setProperties(MetadataScope.USER, stream1, streamUserProps);
    Map<String, String> datasetUserProps = ImmutableMap.of("sKey1", "sValuee1 sValuee2");
    store.setProperties(MetadataScope.USER, dataset1, datasetUserProps);
    // Test score and metadata match
    MetadataSearchResponse response = search("ns1", "value1 multiword:av2");
    Assert.assertEquals(2, response.getTotal());
    List<MetadataSearchResultRecord> actual = Lists.newArrayList(response.getResults());
    Map<MetadataScope, Metadata> expectedFlowMetadata = ImmutableMap.of(MetadataScope.USER, new Metadata(flowUserProps, flowUserTags), MetadataScope.SYSTEM, new Metadata(flowSysProps, flowSysTags));
    Map<MetadataScope, Metadata> expectedStreamMetadata = ImmutableMap.of(MetadataScope.USER, new Metadata(streamUserProps, Collections.<String>emptySet()));
    Map<MetadataScope, Metadata> expectedDatasetMetadata = ImmutableMap.of(MetadataScope.USER, new Metadata(datasetUserProps, Collections.<String>emptySet()));
    List<MetadataSearchResultRecord> expected = Lists.newArrayList(new MetadataSearchResultRecord(flow1, expectedFlowMetadata), new MetadataSearchResultRecord(stream1, expectedStreamMetadata));
    Assert.assertEquals(expected, actual);
    response = search("ns1", "value1 sValue*");
    Assert.assertEquals(3, response.getTotal());
    actual = Lists.newArrayList(response.getResults());
    expected = Lists.newArrayList(new MetadataSearchResultRecord(stream1, expectedStreamMetadata), new MetadataSearchResultRecord(dataset1, expectedDatasetMetadata), new MetadataSearchResultRecord(flow1, expectedFlowMetadata));
    Assert.assertEquals(expected, actual);
    response = search("ns1", "*");
    Assert.assertEquals(3, response.getTotal());
    actual = Lists.newArrayList(response.getResults());
    Assert.assertTrue(actual.containsAll(expected));
}

Example 9 with MetadataSearchResponse

use of co.cask.cdap.proto.metadata.MetadataSearchResponse in project cdap by caskdata.

the class MetadataStoreTest method testSearchPagination.

// Tests pagination for search results queries that do not have indexes stored in sorted order
// The pagination for these queries is done in DefaultMetadataStore, so adding this test here.
@Test
public void testSearchPagination() throws BadRequestException {
    NamespaceId ns = new NamespaceId("ns");
    ProgramId flow = ns.app("app").flow("flow");
    StreamId stream = ns.stream("stream");
    DatasetId dataset = ns.dataset("dataset");
    // add a dummy entity which starts with _ to test that it doesnt show up see: CDAP-7910
    DatasetId trackerDataset = ns.dataset("_auditLog");
    store.addTags(MetadataScope.USER, flow, "tag", "tag1");
    store.addTags(MetadataScope.USER, stream, "tag2", "tag3 tag4");
    store.addTags(MetadataScope.USER, dataset, "tag5 tag6", "tag7 tag8");
    // add bunch of tags to ensure higher weight to this entity in search result
    store.addTags(MetadataScope.USER, trackerDataset, "tag9", "tag10", "tag11", "tag12", "tag13");
    MetadataSearchResultRecord flowSearchResult = new MetadataSearchResultRecord(flow);
    MetadataSearchResultRecord streamSearchResult = new MetadataSearchResultRecord(stream);
    MetadataSearchResultRecord datasetSearchResult = new MetadataSearchResultRecord(dataset);
    MetadataSearchResultRecord trackerDatasetSearchResult = new MetadataSearchResultRecord(trackerDataset);
    // relevance order for searchQuery "tag*" is trackerDataset, dataset, stream, flow
    // (this depends on how many tags got matched with the search query)
    // trackerDataset entity should not be part
    MetadataSearchResponse response = search(ns.getNamespace(), "tag*", 0, Integer.MAX_VALUE, 1);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.of(datasetSearchResult, streamSearchResult, flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    // trackerDataset entity should be be part since showHidden is true
    response = search(ns.getNamespace(), "tag*", 0, Integer.MAX_VALUE, 1, true);
    Assert.assertEquals(4, response.getTotal());
    Assert.assertEquals(ImmutableList.of(trackerDatasetSearchResult, datasetSearchResult, streamSearchResult, flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    response = search(ns.getNamespace(), "tag*", 0, 2, 1);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.of(datasetSearchResult, streamSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    // skipping trackerDataset should not affect the offset
    response = search(ns.getNamespace(), "tag*", 1, 2, 1);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.of(streamSearchResult, flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    // if showHidden is true trackerDataset should affect the offset
    response = search(ns.getNamespace(), "tag*", 1, 3, 1, true);
    Assert.assertEquals(4, response.getTotal());
    Assert.assertEquals(ImmutableList.of(datasetSearchResult, streamSearchResult, flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    response = search(ns.getNamespace(), "tag*", 2, 2, 1);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.of(flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
    response = search(ns.getNamespace(), "tag*", 4, 2, 1);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.<MetadataSearchResultRecord>of(), ImmutableList.copyOf(stripMetadata(response.getResults())));
    // ensure overflow bug is squashed (JIRA: CDAP-8133)
    response = search(ns.getNamespace(), "tag*", 1, Integer.MAX_VALUE, 0);
    Assert.assertEquals(3, response.getTotal());
    Assert.assertEquals(ImmutableList.of(streamSearchResult, flowSearchResult), ImmutableList.copyOf(stripMetadata(response.getResults())));
}