Examples with SearchRequest io.cdap.cdap.spi.metadata.SearchRequest used on opensource projects

Example 1 with SearchRequest

use of io.cdap.cdap.spi.metadata.SearchRequest in project cdap by caskdata.

the class DatasetMetadataStorage method search.

@Override
public SearchResponse search(SearchRequest request) {
    Cursor cursor = request.getCursor() != null && !request.getCursor().isEmpty() ? Cursor.fromString(request.getCursor()) : null;
    Set<String> namespaces = cursor == null ? request.getNamespaces() : cursor.getNamespaces();
    ImmutablePair<NamespaceId, Set<EntityScope>> namespaceAndScopes = determineNamespaceAndScopes(namespaces);
    CursorAndOffsetInfo cursorOffsetAndLimits = determineCursorOffsetAndLimits(request, cursor);
    String query = cursor != null ? cursor.getQuery() : request.getQuery() == null || request.getQuery().isEmpty() ? "*" : request.getQuery();
    Set<String> types = cursor != null ? cursor.getTypes() : request.getTypes();
    types = types == null ? Collections.emptySet() : types;
    Sorting sorting = cursor == null ? request.getSorting() : cursor.getSorting() == null ? null : Sorting.of(cursor.getSorting());
    SortInfo sortInfo = sorting == null ? SortInfo.DEFAULT : new SortInfo(sorting.getKey(), SortInfo.SortOrder.valueOf(sorting.getOrder().name()));
    boolean showHidden = cursor != null ? cursor.isShowHidden() : request.isShowHidden();
    MetadataScope scope = cursor != null ? cursor.getScope() : request.getScope();
    MetadataSearchResponse response = search(new io.cdap.cdap.data2.metadata.dataset.SearchRequest(namespaceAndScopes.getFirst(), query, types, sortInfo, cursorOffsetAndLimits.getOffsetToRequest(), cursorOffsetAndLimits.getLimitToRequest(), request.isCursorRequested() ? 1 : 0, cursorOffsetAndLimits.getCursor(), showHidden, namespaceAndScopes.getSecond()), scope);
    // translate results back and limit them to at most what was requested (see above where we add 1)
    int limitToRespond = cursorOffsetAndLimits.getLimitToRespond();
    int offsetToRespond = cursorOffsetAndLimits.getOffsetToRespond();
    List<MetadataRecord> results = response.getResults().stream().limit(limitToRespond).map(record -> {
        Metadata metadata = Metadata.EMPTY;
        for (Map.Entry<MetadataScope, io.cdap.cdap.api.metadata.Metadata> entry : record.getMetadata().entrySet()) {
            Metadata toAdd = new Metadata(entry.getKey(), entry.getValue().getTags(), entry.getValue().getProperties());
            metadata = mergeDisjointMetadata(metadata, toAdd);
        }
        return new MetadataRecord(record.getMetadataEntity(), metadata);
    }).collect(Collectors.toList());
    Cursor newCursor = null;
    if (response.getCursors() != null && !response.getCursors().isEmpty()) {
        String actualCursor = response.getCursors().get(0);
        if (cursor != null) {
            // the new cursor's offset is the previous cursor's offset plus the number of results
            newCursor = new Cursor(cursor, cursor.getOffset() + results.size(), actualCursor);
        } else {
            newCursor = new Cursor(offsetToRespond + results.size(), limitToRespond, showHidden, scope, namespaces, types, sorting == null ? null : sorting.toString(), actualCursor, query);
        }
    }
    // adjust the total results by the difference of requested offset and the true offset that we respond back
    int totalResults = offsetToRespond - cursorOffsetAndLimits.getOffsetToRequest() + response.getTotal();
    return new SearchResponse(request, newCursor == null ? null : newCursor.toString(), offsetToRespond, limitToRespond, totalResults, results);
}

Example 2 with SearchRequest

use of io.cdap.cdap.spi.metadata.SearchRequest in project cdap by caskdata.

the class MetadataAdminAuthorizationTest method testSearch.

@Test
public void testSearch() throws Exception {
    SecurityRequestContext.setUserId(ALICE.getName());
    ApplicationId applicationId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    // grant all the privileges needed to deploy the app
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT), ALICE, Collections.singleton(StandardPermission.GET));
    accessController.grant(Authorizable.fromEntityId(applicationId), ALICE, Collections.singleton(StandardPermission.CREATE));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.artifact(AllProgramsApp.class.getSimpleName(), "1.0-SNAPSHOT")), ALICE, Collections.singleton(StandardPermission.CREATE));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME)), ALICE, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME2)), ALICE, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME3)), ALICE, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.dataset(AllProgramsApp.DS_WITH_SCHEMA_NAME)), ALICE, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.datasetType(KeyValueTable.class.getName())), ALICE, Collections.singleton(StandardPermission.UPDATE));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.datasetType(KeyValueTable.class.getName())), ALICE, Collections.singleton(StandardPermission.UPDATE));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.datasetType(ObjectMappedTable.class.getName())), ALICE, Collections.singleton(StandardPermission.UPDATE));
    // no auto grant now, need to have privileges on the program to be able to see the programs
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.SERVICE, AllProgramsApp.NoOpService.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.WORKER, AllProgramsApp.NoOpWorker.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.SPARK, AllProgramsApp.NoOpSpark.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.MAPREDUCE, AllProgramsApp.NoOpMR.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.MAPREDUCE, AllProgramsApp.NoOpMR2.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.WORKFLOW, AllProgramsApp.NoOpWorkflow.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    AppFabricTestHelper.deployApplication(Id.Namespace.DEFAULT, AllProgramsApp.class, "{}", cConf);
    SearchRequest searchRequest = SearchRequest.of("*").addSystemNamespace().addNamespace(NamespaceId.DEFAULT.getNamespace()).addType(MetadataEntity.NAMESPACE).addType(MetadataEntity.ARTIFACT).addType(MetadataEntity.APPLICATION).addType(MetadataEntity.PROGRAM).addType(MetadataEntity.DATASET).setOffset(0).setLimit(Integer.MAX_VALUE).build();
    Tasks.waitFor(false, () -> metadataAdmin.search(searchRequest).getResults().isEmpty(), 5, TimeUnit.SECONDS);
    SecurityRequestContext.setUserId("bob");
    Assert.assertTrue(metadataAdmin.search(searchRequest).getResults().isEmpty());
}

Example 3 with SearchRequest

use of io.cdap.cdap.spi.metadata.SearchRequest in project cdap by caskdata.

the class ElasticsearchMetadataStorage method doSearch.

/**
 * Perform a search that does continue a previous search using a cursor.
 *
 * @param request the search request
 */
private io.cdap.cdap.spi.metadata.SearchResponse doSearch(SearchRequest request) throws IOException {
    org.elasticsearch.action.search.SearchRequest searchRequest = new org.elasticsearch.action.search.SearchRequest(indexName);
    SearchSourceBuilder searchSource = createSearchSource(request);
    searchRequest.source(searchSource);
    // only request a scroll if the offset is 0. Elastic will throw otherwise
    if (request.isCursorRequested() && searchSource.from() == 0) {
        searchRequest.scroll(scrollTimeout);
    }
    RestHighLevelClient client = getClient();
    LOG.debug("Executing search request {}", searchRequest);
    SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
    return createSearchResponse(request, searchResponse, computeCursor(searchResponse, request), request.getOffset(), request.getLimit());
}

Example 4 with SearchRequest

use of io.cdap.cdap.spi.metadata.SearchRequest in project cdap by caskdata.

the class ElasticsearchMetadataStorageTest method testScrollTimeout.

@Test
public void testScrollTimeout() throws IOException, InterruptedException {
    MetadataStorage mds = getMetadataStorage();
    MutationOptions options = MutationOptions.builder().setAsynchronous(false).build();
    List<MetadataRecord> records = IntStream.range(0, 20).boxed().map(i -> new MetadataRecord(MetadataEntity.ofDataset("ns" + i, "ds" + i), new Metadata(MetadataScope.USER, tags("tag", "t" + i), props("p", "v" + i)))).collect(Collectors.toList());
    mds.batch(records.stream().map(r -> new Update(r.getEntity(), r.getMetadata())).collect(Collectors.toList()), options);
    SearchRequest request = SearchRequest.of("t*").setCursorRequested(true).setLimit(5).build();
    SearchResponse response = mds.search(request);
    Assert.assertEquals(5, response.getResults().size());
    Assert.assertNotNull(response.getCursor());
    SearchRequest request2 = SearchRequest.of("t*").setCursor(response.getCursor()).build();
    SearchResponse response2 = mds.search(request2);
    Assert.assertEquals(5, response2.getResults().size());
    // it works despite a cursor and an offset (which is equal to the cursor's)
    SearchRequest request2a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(5).build();
    SearchResponse response2a = mds.search(request2a);
    Assert.assertEquals(5, response2a.getOffset());
    Assert.assertEquals(5, response2a.getLimit());
    Assert.assertEquals(response2.getResults(), response2a.getResults());
    // it works despite a cursor and an offset (which is different from the cursor's)
    SearchRequest request2b = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response2b = mds.search(request2b);
    Assert.assertEquals(5, response2b.getOffset());
    Assert.assertEquals(5, response2b.getLimit());
    Assert.assertEquals(response2.getResults(), response2b.getResults());
    // sleep 1 sec longer than the configured scroll timeout to invalidate cursor
    TimeUnit.SECONDS.sleep(3);
    SearchResponse response3 = mds.search(request2);
    Assert.assertEquals(response2.getResults(), response3.getResults());
    Assert.assertEquals(response2.getCursor(), response3.getCursor());
    // it works despite an expired cursor and an offset (which is different from the cursor's)
    SearchRequest request3a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response3a = mds.search(request3a);
    Assert.assertEquals(5, response3a.getOffset());
    Assert.assertEquals(5, response3a.getLimit());
    Assert.assertEquals(response2.getResults(), response3a.getResults());
    // create a nonsense cursor and search with that
    Cursor cursor = Cursor.fromString(response.getCursor());
    cursor = new Cursor(cursor, cursor.getOffset(), "nosuchcursor");
    SearchRequest request4 = SearchRequest.of("t*").setCursor(cursor.toString()).build();
    SearchResponse response4 = mds.search(request4);
    // compare only the results, not the entire response (the cursor is different)
    Assert.assertEquals(response2.getResults(), response4.getResults());
    // clean up
    mds.batch(records.stream().map(MetadataRecord::getEntity).map(Drop::new).collect(Collectors.toList()), options);
}

Example 5 with SearchRequest

use of io.cdap.cdap.spi.metadata.SearchRequest in project cdap by caskdata.

the class DatasetMetadataStorageTest method testCursorsOffsetsAndLimits.

private void testCursorsOffsetsAndLimits(Cursor cursor, boolean cursorRequested, int offsetRequested, int limitRequested, String expectedCursor, int expectedOffsetToRequest, int expectedOffsetToRespond, int expectedLimitToRequest, int expectedLimitToRespond) {
    SearchRequest request = SearchRequest.of("*").setCursor(cursor == null ? null : cursor.toString()).setCursorRequested(cursorRequested).setOffset(offsetRequested).setLimit(limitRequested).build();
    DatasetMetadataStorage.CursorAndOffsetInfo info = DatasetMetadataStorage.determineCursorOffsetAndLimits(request, cursor);
    Assert.assertEquals(expectedCursor, info.getCursor());
    Assert.assertEquals(expectedOffsetToRequest, info.getOffsetToRequest());
    Assert.assertEquals(expectedOffsetToRespond, info.getOffsetToRespond());
    Assert.assertEquals(expectedLimitToRequest, info.getLimitToRequest());
    Assert.assertEquals(expectedLimitToRespond, info.getLimitToRespond());
}