use of io.cdap.cdap.spi.metadata.SearchResponse in project cdap by caskdata.
the class DatasetMetadataStorage method search.
@Override
public SearchResponse search(SearchRequest request) {
Cursor cursor = request.getCursor() != null && !request.getCursor().isEmpty() ? Cursor.fromString(request.getCursor()) : null;
Set<String> namespaces = cursor == null ? request.getNamespaces() : cursor.getNamespaces();
ImmutablePair<NamespaceId, Set<EntityScope>> namespaceAndScopes = determineNamespaceAndScopes(namespaces);
CursorAndOffsetInfo cursorOffsetAndLimits = determineCursorOffsetAndLimits(request, cursor);
String query = cursor != null ? cursor.getQuery() : request.getQuery() == null || request.getQuery().isEmpty() ? "*" : request.getQuery();
Set<String> types = cursor != null ? cursor.getTypes() : request.getTypes();
types = types == null ? Collections.emptySet() : types;
Sorting sorting = cursor == null ? request.getSorting() : cursor.getSorting() == null ? null : Sorting.of(cursor.getSorting());
SortInfo sortInfo = sorting == null ? SortInfo.DEFAULT : new SortInfo(sorting.getKey(), SortInfo.SortOrder.valueOf(sorting.getOrder().name()));
boolean showHidden = cursor != null ? cursor.isShowHidden() : request.isShowHidden();
MetadataScope scope = cursor != null ? cursor.getScope() : request.getScope();
MetadataSearchResponse response = search(new io.cdap.cdap.data2.metadata.dataset.SearchRequest(namespaceAndScopes.getFirst(), query, types, sortInfo, cursorOffsetAndLimits.getOffsetToRequest(), cursorOffsetAndLimits.getLimitToRequest(), request.isCursorRequested() ? 1 : 0, cursorOffsetAndLimits.getCursor(), showHidden, namespaceAndScopes.getSecond()), scope);
// translate results back and limit them to at most what was requested (see above where we add 1)
int limitToRespond = cursorOffsetAndLimits.getLimitToRespond();
int offsetToRespond = cursorOffsetAndLimits.getOffsetToRespond();
List<MetadataRecord> results = response.getResults().stream().limit(limitToRespond).map(record -> {
Metadata metadata = Metadata.EMPTY;
for (Map.Entry<MetadataScope, io.cdap.cdap.api.metadata.Metadata> entry : record.getMetadata().entrySet()) {
Metadata toAdd = new Metadata(entry.getKey(), entry.getValue().getTags(), entry.getValue().getProperties());
metadata = mergeDisjointMetadata(metadata, toAdd);
}
return new MetadataRecord(record.getMetadataEntity(), metadata);
}).collect(Collectors.toList());
Cursor newCursor = null;
if (response.getCursors() != null && !response.getCursors().isEmpty()) {
String actualCursor = response.getCursors().get(0);
if (cursor != null) {
// the new cursor's offset is the previous cursor's offset plus the number of results
newCursor = new Cursor(cursor, cursor.getOffset() + results.size(), actualCursor);
} else {
newCursor = new Cursor(offsetToRespond + results.size(), limitToRespond, showHidden, scope, namespaces, types, sorting == null ? null : sorting.toString(), actualCursor, query);
}
}
// adjust the total results by the difference of requested offset and the true offset that we respond back
int totalResults = offsetToRespond - cursorOffsetAndLimits.getOffsetToRequest() + response.getTotal();
return new SearchResponse(request, newCursor == null ? null : newCursor.toString(), offsetToRespond, limitToRespond, totalResults, results);
}
use of io.cdap.cdap.spi.metadata.SearchResponse in project cdap by caskdata.
the class ElasticsearchMetadataStorageTest method testScrollTimeout.
@Test
public void testScrollTimeout() throws IOException, InterruptedException {
MetadataStorage mds = getMetadataStorage();
MutationOptions options = MutationOptions.builder().setAsynchronous(false).build();
List<MetadataRecord> records = IntStream.range(0, 20).boxed().map(i -> new MetadataRecord(MetadataEntity.ofDataset("ns" + i, "ds" + i), new Metadata(MetadataScope.USER, tags("tag", "t" + i), props("p", "v" + i)))).collect(Collectors.toList());
mds.batch(records.stream().map(r -> new Update(r.getEntity(), r.getMetadata())).collect(Collectors.toList()), options);
SearchRequest request = SearchRequest.of("t*").setCursorRequested(true).setLimit(5).build();
SearchResponse response = mds.search(request);
Assert.assertEquals(5, response.getResults().size());
Assert.assertNotNull(response.getCursor());
SearchRequest request2 = SearchRequest.of("t*").setCursor(response.getCursor()).build();
SearchResponse response2 = mds.search(request2);
Assert.assertEquals(5, response2.getResults().size());
// it works despite a cursor and an offset (which is equal to the cursor's)
SearchRequest request2a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(5).build();
SearchResponse response2a = mds.search(request2a);
Assert.assertEquals(5, response2a.getOffset());
Assert.assertEquals(5, response2a.getLimit());
Assert.assertEquals(response2.getResults(), response2a.getResults());
// it works despite a cursor and an offset (which is different from the cursor's)
SearchRequest request2b = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
SearchResponse response2b = mds.search(request2b);
Assert.assertEquals(5, response2b.getOffset());
Assert.assertEquals(5, response2b.getLimit());
Assert.assertEquals(response2.getResults(), response2b.getResults());
// sleep 1 sec longer than the configured scroll timeout to invalidate cursor
TimeUnit.SECONDS.sleep(3);
SearchResponse response3 = mds.search(request2);
Assert.assertEquals(response2.getResults(), response3.getResults());
Assert.assertEquals(response2.getCursor(), response3.getCursor());
// it works despite an expired cursor and an offset (which is different from the cursor's)
SearchRequest request3a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
SearchResponse response3a = mds.search(request3a);
Assert.assertEquals(5, response3a.getOffset());
Assert.assertEquals(5, response3a.getLimit());
Assert.assertEquals(response2.getResults(), response3a.getResults());
// create a nonsense cursor and search with that
Cursor cursor = Cursor.fromString(response.getCursor());
cursor = new Cursor(cursor, cursor.getOffset(), "nosuchcursor");
SearchRequest request4 = SearchRequest.of("t*").setCursor(cursor.toString()).build();
SearchResponse response4 = mds.search(request4);
// compare only the results, not the entire response (the cursor is different)
Assert.assertEquals(response2.getResults(), response4.getResults());
// clean up
mds.batch(records.stream().map(MetadataRecord::getEntity).map(Drop::new).collect(Collectors.toList()), options);
}
use of io.cdap.cdap.spi.metadata.SearchResponse in project cdap by caskdata.
the class DataPipelineConnectionTest method assertMetadataSearch.
private void assertMetadataSearch(Set<MetadataEntity> appsRelated, String query) throws Exception {
SearchResponse search = getMetadataAdmin().search(SearchRequest.of(query).build());
Set<MetadataEntity> actual = search.getResults().stream().map(record -> record.getEntity()).collect(Collectors.toSet());
Assert.assertEquals(appsRelated, actual);
}
use of io.cdap.cdap.spi.metadata.SearchResponse in project cdap by caskdata.
the class DatasetMetadataStorageTest method testCrossNamespacePagination.
// this test is specific to teh DatasetMetadataStorage, because of the specific way it tests pagination:
// it requests offsets that are not a multiple of the page size, which is not supported in all implementations.
@Test
public void testCrossNamespacePagination() throws IOException {
MetadataStorage mds = getMetadataStorage();
NamespaceId ns1Id = new NamespaceId("ns1");
NamespaceId ns2Id = new NamespaceId("ns2");
MetadataEntity ns1app1 = ns1Id.app("a1").toMetadataEntity();
MetadataEntity ns1app2 = ns1Id.app("a2").toMetadataEntity();
MetadataEntity ns1app3 = ns1Id.app("a3").toMetadataEntity();
MetadataEntity ns2app1 = ns2Id.app("a1").toMetadataEntity();
MetadataEntity ns2app2 = ns2Id.app("a2").toMetadataEntity();
mds.batch(ImmutableList.of(new Update(ns1app1, new Metadata(USER, tags("v1"))), new Update(ns1app2, new Metadata(USER, tags("v1"))), new Update(ns1app3, new Metadata(USER, tags("v1"))), new Update(ns2app1, new Metadata(USER, tags("v1"))), new Update(ns2app2, new Metadata(USER, tags("v1")))), MutationOptions.DEFAULT);
MetadataRecord record11 = new MetadataRecord(ns1app1, new Metadata(USER, tags("v1")));
MetadataRecord record12 = new MetadataRecord(ns1app2, new Metadata(USER, tags("v1")));
MetadataRecord record13 = new MetadataRecord(ns1app3, new Metadata(USER, tags("v1")));
MetadataRecord record21 = new MetadataRecord(ns2app1, new Metadata(USER, tags("v1")));
MetadataRecord record22 = new MetadataRecord(ns2app2, new Metadata(USER, tags("v1")));
SearchResponse response = assertResults(mds, SearchRequest.of("*").setLimit(Integer.MAX_VALUE).setCursorRequested(true).build(), record11, record12, record13, record21, record22);
// iterate over results to find the order in which they are returned
Iterator<MetadataRecord> resultIter = response.getResults().iterator();
MetadataRecord[] results = { resultIter.next(), resultIter.next(), resultIter.next(), resultIter.next(), resultIter.next() };
// get 4 results (guaranteed to have at least one from each namespace), offset 1
assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(1).setLimit(4).build(), results[1], results[2], results[3], results[4]);
// get the first four
assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(0).setLimit(4).build(), results[0], results[1], results[2], results[3]);
// get middle 3
assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(1).setLimit(3).build(), results[1], results[2], results[3], results[3]);
// clean up
mds.batch(ImmutableList.of(new Drop(ns1app1), new Drop(ns1app2), new Drop(ns1app3), new Drop(ns2app1), new Drop(ns2app2)), MutationOptions.DEFAULT);
}
use of io.cdap.cdap.spi.metadata.SearchResponse in project cdap by caskdata.
the class MetadataHttpHandler method searchMetadata.
@GET
@Path("/metadata/search")
public void searchMetadata(HttpRequest request, HttpResponder responder, @Nullable @QueryParam("namespaces") List<String> namespaces, @Nullable @QueryParam("scope") String scope, @Nullable @QueryParam("query") String searchQuery, @Nullable @QueryParam("target") List<String> targets, @Nullable @QueryParam("sort") String sort, @QueryParam("offset") @DefaultValue("0") int offset, // 2147483647 is Integer.MAX_VALUE
@QueryParam("limit") @DefaultValue("2147483647") int limit, @Nullable @QueryParam("numCursors") Integer numCursors, @QueryParam("cursorRequested") @DefaultValue("false") boolean cursorRequested, @Nullable @QueryParam("cursor") String cursor, @QueryParam("showHidden") @DefaultValue("false") boolean showHidden, @Nullable @QueryParam("entityScope") String entityScope, @Nullable @QueryParam("responseFormat") @DefaultValue("v5") String responseFormat) throws Exception {
SearchRequest searchRequest = getValidatedSearchRequest(scope, namespaces, searchQuery, targets, sort, offset, limit, numCursors, cursorRequested, cursor, showHidden, entityScope);
SearchResponse response = metadataAdmin.search(searchRequest);
responder.sendJson(HttpResponseStatus.OK, GSON.toJson("v5".equals(responseFormat) ? MetadataCompatibility.toV5Response(response, entityScope) : response));
}
Aggregations