Examples with Cursor io.cdap.cdap.common.metadata.Cursor used on opensource projects

Example 1 with Cursor

use of io.cdap.cdap.common.metadata.Cursor in project cdap by caskdata.

the class DatasetMetadataStorage method search.

@Override
public SearchResponse search(SearchRequest request) {
    Cursor cursor = request.getCursor() != null && !request.getCursor().isEmpty() ? Cursor.fromString(request.getCursor()) : null;
    Set<String> namespaces = cursor == null ? request.getNamespaces() : cursor.getNamespaces();
    ImmutablePair<NamespaceId, Set<EntityScope>> namespaceAndScopes = determineNamespaceAndScopes(namespaces);
    CursorAndOffsetInfo cursorOffsetAndLimits = determineCursorOffsetAndLimits(request, cursor);
    String query = cursor != null ? cursor.getQuery() : request.getQuery() == null || request.getQuery().isEmpty() ? "*" : request.getQuery();
    Set<String> types = cursor != null ? cursor.getTypes() : request.getTypes();
    types = types == null ? Collections.emptySet() : types;
    Sorting sorting = cursor == null ? request.getSorting() : cursor.getSorting() == null ? null : Sorting.of(cursor.getSorting());
    SortInfo sortInfo = sorting == null ? SortInfo.DEFAULT : new SortInfo(sorting.getKey(), SortInfo.SortOrder.valueOf(sorting.getOrder().name()));
    boolean showHidden = cursor != null ? cursor.isShowHidden() : request.isShowHidden();
    MetadataScope scope = cursor != null ? cursor.getScope() : request.getScope();
    MetadataSearchResponse response = search(new io.cdap.cdap.data2.metadata.dataset.SearchRequest(namespaceAndScopes.getFirst(), query, types, sortInfo, cursorOffsetAndLimits.getOffsetToRequest(), cursorOffsetAndLimits.getLimitToRequest(), request.isCursorRequested() ? 1 : 0, cursorOffsetAndLimits.getCursor(), showHidden, namespaceAndScopes.getSecond()), scope);
    // translate results back and limit them to at most what was requested (see above where we add 1)
    int limitToRespond = cursorOffsetAndLimits.getLimitToRespond();
    int offsetToRespond = cursorOffsetAndLimits.getOffsetToRespond();
    List<MetadataRecord> results = response.getResults().stream().limit(limitToRespond).map(record -> {
        Metadata metadata = Metadata.EMPTY;
        for (Map.Entry<MetadataScope, io.cdap.cdap.api.metadata.Metadata> entry : record.getMetadata().entrySet()) {
            Metadata toAdd = new Metadata(entry.getKey(), entry.getValue().getTags(), entry.getValue().getProperties());
            metadata = mergeDisjointMetadata(metadata, toAdd);
        }
        return new MetadataRecord(record.getMetadataEntity(), metadata);
    }).collect(Collectors.toList());
    Cursor newCursor = null;
    if (response.getCursors() != null && !response.getCursors().isEmpty()) {
        String actualCursor = response.getCursors().get(0);
        if (cursor != null) {
            // the new cursor's offset is the previous cursor's offset plus the number of results
            newCursor = new Cursor(cursor, cursor.getOffset() + results.size(), actualCursor);
        } else {
            newCursor = new Cursor(offsetToRespond + results.size(), limitToRespond, showHidden, scope, namespaces, types, sorting == null ? null : sorting.toString(), actualCursor, query);
        }
    }
    // adjust the total results by the difference of requested offset and the true offset that we respond back
    int totalResults = offsetToRespond - cursorOffsetAndLimits.getOffsetToRequest() + response.getTotal();
    return new SearchResponse(request, newCursor == null ? null : newCursor.toString(), offsetToRespond, limitToRespond, totalResults, results);
}

Example 2 with Cursor

use of io.cdap.cdap.common.metadata.Cursor in project cdap by caskdata.

the class ElasticsearchMetadataStorage method computeCursor.

/**
 * Generates the cursor to return as part of a search response:
 * <ul><li>
 *   If the Elasticsearch response does not have a scroll id, returns null;
 * </li><li>
 *   If the number of results in this response exceeds the total number of results,
 *   cancels the scroll returned by Elasticsearch, then returns null.
 * </li><li>
 *   Otherwise returns a cursor representing the Elasticsearch scroll id,
 *   offset and page size for a subsequent search.
 * </li></ul>
 */
@Nullable
private String computeCursor(SearchResponse searchResponse, Cursor cursor) {
    if (searchResponse.getScrollId() != null) {
        SearchHits hits = searchResponse.getHits();
        int newOffset = cursor.getOffset() + hits.getHits().length;
        if (newOffset < hits.getTotalHits()) {
            return new Cursor(cursor, newOffset, searchResponse.getScrollId()).toString();
        }
        // we have seen all results, there are no more to fetch: clear the scroll
        cancelSroll(searchResponse.getScrollId());
    }
    return null;
}

Example 3 with Cursor

use of io.cdap.cdap.common.metadata.Cursor in project cdap by caskdata.

the class ElasticsearchMetadataStorage method doScroll.

/**
 * Perform a search that continues a previous search using a cursor. Such a search
 * is always relative to the offset of the cursor (with no additional offset allowed)
 * and uses the the same page size as the original search.
 *
 * If the cursor provided in the request had expired, this performs a new search,
 * beginning at the offset given by the cursor.
 *
 * @return the search response containing the next page of results.
 */
private io.cdap.cdap.spi.metadata.SearchResponse doScroll(SearchRequest request) throws IOException {
    Cursor cursor = Cursor.fromString(request.getCursor());
    SearchScrollRequest scrollRequest = new SearchScrollRequest(cursor.getActualCursor());
    if (request.isCursorRequested()) {
        scrollRequest.scroll(scrollTimeout);
    }
    SearchResponse searchResponse;
    RestHighLevelClient client = getClient();
    try {
        searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
    } catch (ElasticsearchStatusException e) {
        // scroll invalid or timed out
        return doSearch(createRequestFromCursor(request, cursor));
    }
    if (searchResponse.isTimedOut()) {
        // scroll had expired, we have to search again
        return doSearch(createRequestFromCursor(request, cursor));
    }
    return createSearchResponse(request, searchResponse, computeCursor(searchResponse, cursor), cursor.getOffset(), cursor.getLimit());
}

Example 4 with Cursor

use of io.cdap.cdap.common.metadata.Cursor in project cdap by caskdata.

the class ElasticsearchMetadataStorage method computeCursor.

@Nullable
private String computeCursor(SearchResponse searchResponse, SearchRequest request) {
    if (searchResponse.getScrollId() != null) {
        SearchHits hits = searchResponse.getHits();
        int newOffset = request.getOffset() + hits.getHits().length;
        if (newOffset < hits.getTotalHits()) {
            return new Cursor(newOffset, request.getLimit(), request.isShowHidden(), request.getScope(), request.getNamespaces(), request.getTypes(), request.getSorting() == null ? null : request.getSorting().toString(), searchResponse.getScrollId(), request.getQuery()).toString();
        }
        // we have seen all results, there are no more to fetch: clear the scroll
        cancelSroll(searchResponse.getScrollId());
    }
    return null;
}

Example 5 with Cursor

use of io.cdap.cdap.common.metadata.Cursor in project cdap by caskdata.

the class ElasticsearchMetadataStorageTest method testScrollTimeout.

@Test
public void testScrollTimeout() throws IOException, InterruptedException {
    MetadataStorage mds = getMetadataStorage();
    MutationOptions options = MutationOptions.builder().setAsynchronous(false).build();
    List<MetadataRecord> records = IntStream.range(0, 20).boxed().map(i -> new MetadataRecord(MetadataEntity.ofDataset("ns" + i, "ds" + i), new Metadata(MetadataScope.USER, tags("tag", "t" + i), props("p", "v" + i)))).collect(Collectors.toList());
    mds.batch(records.stream().map(r -> new Update(r.getEntity(), r.getMetadata())).collect(Collectors.toList()), options);
    SearchRequest request = SearchRequest.of("t*").setCursorRequested(true).setLimit(5).build();
    SearchResponse response = mds.search(request);
    Assert.assertEquals(5, response.getResults().size());
    Assert.assertNotNull(response.getCursor());
    SearchRequest request2 = SearchRequest.of("t*").setCursor(response.getCursor()).build();
    SearchResponse response2 = mds.search(request2);
    Assert.assertEquals(5, response2.getResults().size());
    // it works despite a cursor and an offset (which is equal to the cursor's)
    SearchRequest request2a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(5).build();
    SearchResponse response2a = mds.search(request2a);
    Assert.assertEquals(5, response2a.getOffset());
    Assert.assertEquals(5, response2a.getLimit());
    Assert.assertEquals(response2.getResults(), response2a.getResults());
    // it works despite a cursor and an offset (which is different from the cursor's)
    SearchRequest request2b = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response2b = mds.search(request2b);
    Assert.assertEquals(5, response2b.getOffset());
    Assert.assertEquals(5, response2b.getLimit());
    Assert.assertEquals(response2.getResults(), response2b.getResults());
    // sleep 1 sec longer than the configured scroll timeout to invalidate cursor
    TimeUnit.SECONDS.sleep(3);
    SearchResponse response3 = mds.search(request2);
    Assert.assertEquals(response2.getResults(), response3.getResults());
    Assert.assertEquals(response2.getCursor(), response3.getCursor());
    // it works despite an expired cursor and an offset (which is different from the cursor's)
    SearchRequest request3a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response3a = mds.search(request3a);
    Assert.assertEquals(5, response3a.getOffset());
    Assert.assertEquals(5, response3a.getLimit());
    Assert.assertEquals(response2.getResults(), response3a.getResults());
    // create a nonsense cursor and search with that
    Cursor cursor = Cursor.fromString(response.getCursor());
    cursor = new Cursor(cursor, cursor.getOffset(), "nosuchcursor");
    SearchRequest request4 = SearchRequest.of("t*").setCursor(cursor.toString()).build();
    SearchResponse response4 = mds.search(request4);
    // compare only the results, not the entire response (the cursor is different)
    Assert.assertEquals(response2.getResults(), response4.getResults());
    // clean up
    mds.batch(records.stream().map(MetadataRecord::getEntity).map(Drop::new).collect(Collectors.toList()), options);
}