Examples with SearchRequest io.cdap.cdap.spi.metadata.SearchRequest used on opensource projects

Example 6 with SearchRequest

use of io.cdap.cdap.spi.metadata.SearchRequest in project cdap by cdapio.

the class DatasetMetadataStorageTest method testCursorsOffsetsAndLimits.

private void testCursorsOffsetsAndLimits(Cursor cursor, boolean cursorRequested, int offsetRequested, int limitRequested, String expectedCursor, int expectedOffsetToRequest, int expectedOffsetToRespond, int expectedLimitToRequest, int expectedLimitToRespond) {
    SearchRequest request = SearchRequest.of("*").setCursor(cursor == null ? null : cursor.toString()).setCursorRequested(cursorRequested).setOffset(offsetRequested).setLimit(limitRequested).build();
    DatasetMetadataStorage.CursorAndOffsetInfo info = DatasetMetadataStorage.determineCursorOffsetAndLimits(request, cursor);
    Assert.assertEquals(expectedCursor, info.getCursor());
    Assert.assertEquals(expectedOffsetToRequest, info.getOffsetToRequest());
    Assert.assertEquals(expectedOffsetToRespond, info.getOffsetToRespond());
    Assert.assertEquals(expectedLimitToRequest, info.getLimitToRequest());
    Assert.assertEquals(expectedLimitToRespond, info.getLimitToRespond());
}

Example 7 with SearchRequest

use of io.cdap.cdap.spi.metadata.SearchRequest in project cdap by cdapio.

the class MetadataAdminAuthorizationTest method testSearch.

@Test
public void testSearch() throws Exception {
    SecurityRequestContext.setUserId(ALICE.getName());
    ApplicationId applicationId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    // grant all the privileges needed to deploy the app
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT), ALICE, Collections.singleton(StandardPermission.GET));
    accessController.grant(Authorizable.fromEntityId(applicationId), ALICE, Collections.singleton(StandardPermission.CREATE));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.artifact(AllProgramsApp.class.getSimpleName(), "1.0-SNAPSHOT")), ALICE, Collections.singleton(StandardPermission.CREATE));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME)), ALICE, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME2)), ALICE, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME3)), ALICE, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.dataset(AllProgramsApp.DS_WITH_SCHEMA_NAME)), ALICE, EnumSet.of(StandardPermission.CREATE, StandardPermission.GET));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.datasetType(KeyValueTable.class.getName())), ALICE, Collections.singleton(StandardPermission.UPDATE));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.datasetType(KeyValueTable.class.getName())), ALICE, Collections.singleton(StandardPermission.UPDATE));
    accessController.grant(Authorizable.fromEntityId(NamespaceId.DEFAULT.datasetType(ObjectMappedTable.class.getName())), ALICE, Collections.singleton(StandardPermission.UPDATE));
    // no auto grant now, need to have privileges on the program to be able to see the programs
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.SERVICE, AllProgramsApp.NoOpService.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.WORKER, AllProgramsApp.NoOpWorker.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.SPARK, AllProgramsApp.NoOpSpark.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.MAPREDUCE, AllProgramsApp.NoOpMR.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.MAPREDUCE, AllProgramsApp.NoOpMR2.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    accessController.grant(Authorizable.fromEntityId(applicationId.program(ProgramType.WORKFLOW, AllProgramsApp.NoOpWorkflow.NAME)), ALICE, Collections.singleton(ApplicationPermission.EXECUTE));
    AppFabricTestHelper.deployApplication(Id.Namespace.DEFAULT, AllProgramsApp.class, "{}", cConf);
    SearchRequest searchRequest = SearchRequest.of("*").addSystemNamespace().addNamespace(NamespaceId.DEFAULT.getNamespace()).addType(MetadataEntity.NAMESPACE).addType(MetadataEntity.ARTIFACT).addType(MetadataEntity.APPLICATION).addType(MetadataEntity.PROGRAM).addType(MetadataEntity.DATASET).setOffset(0).setLimit(Integer.MAX_VALUE).build();
    Tasks.waitFor(false, () -> metadataAdmin.search(searchRequest).getResults().isEmpty(), 5, TimeUnit.SECONDS);
    SecurityRequestContext.setUserId("bob");
    Assert.assertTrue(metadataAdmin.search(searchRequest).getResults().isEmpty());
}

Example 8 with SearchRequest

use of io.cdap.cdap.spi.metadata.SearchRequest in project cdap by cdapio.

the class DatasetMetadataStorage method search.

@Override
public SearchResponse search(SearchRequest request) {
    Cursor cursor = request.getCursor() != null && !request.getCursor().isEmpty() ? Cursor.fromString(request.getCursor()) : null;
    Set<String> namespaces = cursor == null ? request.getNamespaces() : cursor.getNamespaces();
    ImmutablePair<NamespaceId, Set<EntityScope>> namespaceAndScopes = determineNamespaceAndScopes(namespaces);
    CursorAndOffsetInfo cursorOffsetAndLimits = determineCursorOffsetAndLimits(request, cursor);
    String query = cursor != null ? cursor.getQuery() : request.getQuery() == null || request.getQuery().isEmpty() ? "*" : request.getQuery();
    Set<String> types = cursor != null ? cursor.getTypes() : request.getTypes();
    types = types == null ? Collections.emptySet() : types;
    Sorting sorting = cursor == null ? request.getSorting() : cursor.getSorting() == null ? null : Sorting.of(cursor.getSorting());
    SortInfo sortInfo = sorting == null ? SortInfo.DEFAULT : new SortInfo(sorting.getKey(), SortInfo.SortOrder.valueOf(sorting.getOrder().name()));
    boolean showHidden = cursor != null ? cursor.isShowHidden() : request.isShowHidden();
    MetadataScope scope = cursor != null ? cursor.getScope() : request.getScope();
    MetadataSearchResponse response = search(new io.cdap.cdap.data2.metadata.dataset.SearchRequest(namespaceAndScopes.getFirst(), query, types, sortInfo, cursorOffsetAndLimits.getOffsetToRequest(), cursorOffsetAndLimits.getLimitToRequest(), request.isCursorRequested() ? 1 : 0, cursorOffsetAndLimits.getCursor(), showHidden, namespaceAndScopes.getSecond()), scope);
    // translate results back and limit them to at most what was requested (see above where we add 1)
    int limitToRespond = cursorOffsetAndLimits.getLimitToRespond();
    int offsetToRespond = cursorOffsetAndLimits.getOffsetToRespond();
    List<MetadataRecord> results = response.getResults().stream().limit(limitToRespond).map(record -> {
        Metadata metadata = Metadata.EMPTY;
        for (Map.Entry<MetadataScope, io.cdap.cdap.api.metadata.Metadata> entry : record.getMetadata().entrySet()) {
            Metadata toAdd = new Metadata(entry.getKey(), entry.getValue().getTags(), entry.getValue().getProperties());
            metadata = mergeDisjointMetadata(metadata, toAdd);
        }
        return new MetadataRecord(record.getMetadataEntity(), metadata);
    }).collect(Collectors.toList());
    Cursor newCursor = null;
    if (response.getCursors() != null && !response.getCursors().isEmpty()) {
        String actualCursor = response.getCursors().get(0);
        if (cursor != null) {
            // the new cursor's offset is the previous cursor's offset plus the number of results
            newCursor = new Cursor(cursor, cursor.getOffset() + results.size(), actualCursor);
        } else {
            newCursor = new Cursor(offsetToRespond + results.size(), limitToRespond, showHidden, scope, namespaces, types, sorting == null ? null : sorting.toString(), actualCursor, query);
        }
    }
    // adjust the total results by the difference of requested offset and the true offset that we respond back
    int totalResults = offsetToRespond - cursorOffsetAndLimits.getOffsetToRequest() + response.getTotal();
    return new SearchResponse(request, newCursor == null ? null : newCursor.toString(), offsetToRespond, limitToRespond, totalResults, results);
}

Example 9 with SearchRequest

use of io.cdap.cdap.spi.metadata.SearchRequest in project cdap by cdapio.

the class MetadataHttpHandler method getValidatedSearchRequest.

// TODO (CDAP-14946): Find a better way to determine allowed combinations of search parameters
private SearchRequest getValidatedSearchRequest(@Nullable String scope, @Nullable List<String> namespaces, @Nullable String searchQuery, @Nullable List<String> targets, @Nullable String sort, int offset, int limit, @Nullable Integer numCursors, boolean cursorRequested, @Nullable String cursor, boolean showHidden, @Nullable String entityScope) throws BadRequestException {
    try {
        SearchRequest.Builder builder = SearchRequest.of(searchQuery == null ? "*" : searchQuery);
        if (scope != null) {
            builder.setScope(validateScope(scope));
        }
        if (EntityScope.SYSTEM == validateEntityScope(entityScope)) {
            builder.addNamespace(entityScope.toLowerCase());
        } else if (namespaces != null) {
            for (String namespace : namespaces) {
                builder.addNamespace(namespace);
            }
        }
        if (targets != null) {
            targets.forEach(builder::addType);
        }
        if (sort != null) {
            Sorting sorting;
            try {
                sorting = Sorting.of(URLDecoder.decode(sort, StandardCharsets.UTF_8.name()));
            } catch (UnsupportedEncodingException e) {
                // this cannot happen because UTF_8 is always supported
                throw new IllegalStateException(e);
            }
            if (!MetadataConstants.ENTITY_NAME_KEY.equalsIgnoreCase(sorting.getKey()) && !MetadataConstants.CREATION_TIME_KEY.equalsIgnoreCase(sorting.getKey())) {
                throw new IllegalArgumentException("Sorting is only supported on fields: " + MetadataConstants.ENTITY_NAME_KEY + ", " + MetadataConstants.CREATION_TIME_KEY);
            }
            builder.setSorting(sorting);
        }
        builder.setOffset(offset);
        builder.setLimit(limit);
        if (cursorRequested || (numCursors != null && numCursors > 0)) {
            if (sort == null) {
                throw new IllegalArgumentException("Specify a sort order when requesting a cursor");
            }
            builder.setCursorRequested(true);
        }
        if (cursor != null) {
            if (sort == null) {
                throw new IllegalArgumentException("Specify a sort order when passing in a cursor");
            }
            builder.setCursor(cursor);
        }
        builder.setShowHidden(showHidden);
        SearchRequest request = builder.build();
        LOG.trace("Received search request {}", request);
        return request;
    } catch (IllegalArgumentException e) {
        throw new BadRequestException(e.getMessage(), e);
    }
}

Example 10 with SearchRequest

use of io.cdap.cdap.spi.metadata.SearchRequest in project cdap by cdapio.

the class MetadataHttpHandler method searchMetadata.

@GET
@Path("/metadata/search")
public void searchMetadata(HttpRequest request, HttpResponder responder, @Nullable @QueryParam("namespaces") List<String> namespaces, @Nullable @QueryParam("scope") String scope, @Nullable @QueryParam("query") String searchQuery, @Nullable @QueryParam("target") List<String> targets, @Nullable @QueryParam("sort") String sort, @QueryParam("offset") @DefaultValue("0") int offset, // 2147483647 is Integer.MAX_VALUE
@QueryParam("limit") @DefaultValue("2147483647") int limit, @Nullable @QueryParam("numCursors") Integer numCursors, @QueryParam("cursorRequested") @DefaultValue("false") boolean cursorRequested, @Nullable @QueryParam("cursor") String cursor, @QueryParam("showHidden") @DefaultValue("false") boolean showHidden, @Nullable @QueryParam("entityScope") String entityScope, @Nullable @QueryParam("responseFormat") @DefaultValue("v5") String responseFormat) throws Exception {
    SearchRequest searchRequest = getValidatedSearchRequest(scope, namespaces, searchQuery, targets, sort, offset, limit, numCursors, cursorRequested, cursor, showHidden, entityScope);
    SearchResponse response = metadataAdmin.search(searchRequest);
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson("v5".equals(responseFormat) ? MetadataCompatibility.toV5Response(response, entityScope) : response));
}