Examples with MetadataSearchResponse io.cdap.cdap.proto.metadata.MetadataSearchResponse used on opensource projects

Example 1 with MetadataSearchResponse

use of io.cdap.cdap.proto.metadata.MetadataSearchResponse in project cdap by caskdata.

the class DatasetMetadataStorage method search.

@Override
public SearchResponse search(SearchRequest request) {
    Cursor cursor = request.getCursor() != null && !request.getCursor().isEmpty() ? Cursor.fromString(request.getCursor()) : null;
    Set<String> namespaces = cursor == null ? request.getNamespaces() : cursor.getNamespaces();
    ImmutablePair<NamespaceId, Set<EntityScope>> namespaceAndScopes = determineNamespaceAndScopes(namespaces);
    CursorAndOffsetInfo cursorOffsetAndLimits = determineCursorOffsetAndLimits(request, cursor);
    String query = cursor != null ? cursor.getQuery() : request.getQuery() == null || request.getQuery().isEmpty() ? "*" : request.getQuery();
    Set<String> types = cursor != null ? cursor.getTypes() : request.getTypes();
    types = types == null ? Collections.emptySet() : types;
    Sorting sorting = cursor == null ? request.getSorting() : cursor.getSorting() == null ? null : Sorting.of(cursor.getSorting());
    SortInfo sortInfo = sorting == null ? SortInfo.DEFAULT : new SortInfo(sorting.getKey(), SortInfo.SortOrder.valueOf(sorting.getOrder().name()));
    boolean showHidden = cursor != null ? cursor.isShowHidden() : request.isShowHidden();
    MetadataScope scope = cursor != null ? cursor.getScope() : request.getScope();
    MetadataSearchResponse response = search(new io.cdap.cdap.data2.metadata.dataset.SearchRequest(namespaceAndScopes.getFirst(), query, types, sortInfo, cursorOffsetAndLimits.getOffsetToRequest(), cursorOffsetAndLimits.getLimitToRequest(), request.isCursorRequested() ? 1 : 0, cursorOffsetAndLimits.getCursor(), showHidden, namespaceAndScopes.getSecond()), scope);
    // translate results back and limit them to at most what was requested (see above where we add 1)
    int limitToRespond = cursorOffsetAndLimits.getLimitToRespond();
    int offsetToRespond = cursorOffsetAndLimits.getOffsetToRespond();
    List<MetadataRecord> results = response.getResults().stream().limit(limitToRespond).map(record -> {
        Metadata metadata = Metadata.EMPTY;
        for (Map.Entry<MetadataScope, io.cdap.cdap.api.metadata.Metadata> entry : record.getMetadata().entrySet()) {
            Metadata toAdd = new Metadata(entry.getKey(), entry.getValue().getTags(), entry.getValue().getProperties());
            metadata = mergeDisjointMetadata(metadata, toAdd);
        }
        return new MetadataRecord(record.getMetadataEntity(), metadata);
    }).collect(Collectors.toList());
    Cursor newCursor = null;
    if (response.getCursors() != null && !response.getCursors().isEmpty()) {
        String actualCursor = response.getCursors().get(0);
        if (cursor != null) {
            // the new cursor's offset is the previous cursor's offset plus the number of results
            newCursor = new Cursor(cursor, cursor.getOffset() + results.size(), actualCursor);
        } else {
            newCursor = new Cursor(offsetToRespond + results.size(), limitToRespond, showHidden, scope, namespaces, types, sorting == null ? null : sorting.toString(), actualCursor, query);
        }
    }
    // adjust the total results by the difference of requested offset and the true offset that we respond back
    int totalResults = offsetToRespond - cursorOffsetAndLimits.getOffsetToRequest() + response.getTotal();
    return new SearchResponse(request, newCursor == null ? null : newCursor.toString(), offsetToRespond, limitToRespond, totalResults, results);
}

Example 2 with MetadataSearchResponse

use of io.cdap.cdap.proto.metadata.MetadataSearchResponse in project cdap by caskdata.

the class SearchHelper method search.

private MetadataSearchResponse search(Set<MetadataScope> scopes, SearchRequest request) {
    List<MetadataEntry> results = new LinkedList<>();
    List<String> cursors = new LinkedList<>();
    for (MetadataScope scope : scopes) {
        SearchResults searchResults = execute(context -> context.getDataset(scope).search(request));
        results.addAll(searchResults.getResults());
        cursors.addAll(searchResults.getCursors());
    }
    int offset = request.getOffset();
    int limit = request.getLimit();
    SortInfo sortInfo = request.getSortInfo();
    // sort if required
    Set<MetadataEntity> sortedEntities = getSortedEntities(results, sortInfo);
    int total = sortedEntities.size();
    // pagination is not performed at the dataset level, because:
    // 1. scoring is needed for DEFAULT sort info. So perform it here for now.
    // 2. Even when using custom sorting, we need to remove elements from the beginning to the offset and the cursors
    // at the end
    // TODO: Figure out how all of this can be done server (HBase) side
    int startIndex = Math.min(request.getOffset(), sortedEntities.size());
    // Account for overflow
    int endIndex = (int) Math.min(Integer.MAX_VALUE, (long) offset + limit);
    endIndex = Math.min(endIndex, sortedEntities.size());
    // add 1 to maxIndex because end index is exclusive
    Set<MetadataEntity> subSortedEntities = new LinkedHashSet<>(ImmutableList.copyOf(sortedEntities).subList(startIndex, endIndex));
    // Fetch metadata for entities in the result list
    // Note: since the fetch is happening in a different transaction, the metadata for entities may have been
    // removed. It is okay not to have metadata for some results in case this happens.
    Set<MetadataSearchResultRecord> finalResults = execute(context -> addMetadataToEntities(subSortedEntities, fetchMetadata(context.getDataset(SYSTEM), subSortedEntities), fetchMetadata(context.getDataset(USER), subSortedEntities)));
    return new MetadataSearchResponse(sortInfo.getSortBy() + " " + sortInfo.getSortOrder(), offset, limit, request.getNumCursors(), total, finalResults, cursors, request.shouldShowHidden(), request.getEntityScopes());
}

Example 3 with MetadataSearchResponse

use of io.cdap.cdap.proto.metadata.MetadataSearchResponse in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSystemScopeArtifacts.

@Test
public void testSystemScopeArtifacts() throws Exception {
    // add a system artifact. currently can't do this through the rest api (by design)
    // so bypass it and use the repository directly
    ArtifactId systemId = NamespaceId.SYSTEM.artifact("app", "1.0.0");
    File systemArtifact = createArtifactJarFile(AllProgramsApp.class, "app", "1.0.0", new Manifest());
    StandaloneTester tester = STANDALONE.get();
    tester.addSystemArtifact(systemId.getArtifact(), Id.Artifact.fromEntityId(systemId).getVersion(), systemArtifact, null);
    // wait until the system metadata has been processed
    Tasks.waitFor(false, () -> getProperties(systemId, MetadataScope.SYSTEM).isEmpty(), 10, TimeUnit.SECONDS);
    // verify that user metadata can be added for system-scope artifacts
    Map<String, String> userProperties = ImmutableMap.of("systemArtifactKey", "systemArtifactValue");
    Set<String> userTags = ImmutableSet.of();
    addProperties(systemId, userProperties);
    addTags(systemId, userTags);
    // verify that user and system metadata can be retrieved for system-scope artifacts
    Assert.assertEquals(ImmutableSet.of(new MetadataRecord(systemId, MetadataScope.USER, userProperties, userTags), new MetadataRecord(systemId, MetadataScope.SYSTEM, ImmutableMap.of(MetadataConstants.ENTITY_NAME_KEY, systemId.getEntityName()), ImmutableSet.of())), removeCreationTime(getMetadata(systemId.toMetadataEntity())));
    // verify that system scope artifacts can be returned by a search in the default namespace
    // with no target type
    MetadataSearchResponse response = searchMetadata(ImmutableList.of(NamespaceId.DEFAULT, NamespaceId.SYSTEM), "system*");
    MetadataEntity expectedEntity = systemId.toMetadataEntity();
    Set<MetadataEntity> actualEntities = new HashSet<>(extractMetadataEntities(response.getResults()));
    Assert.assertTrue(actualEntities.contains(expectedEntity));
    // with target type as artifact
    assertSearch(searchMetadata(ImmutableList.of(NamespaceId.DEFAULT, NamespaceId.SYSTEM), "system*", MetadataEntity.ARTIFACT), systemId);
    // verify that user metadata can be deleted for system-scope artifacts
    removeMetadata(systemId);
    Assert.assertEquals(ImmutableSet.of(new MetadataRecord(systemId, MetadataScope.USER, ImmutableMap.of(), ImmutableSet.of()), new MetadataRecord(systemId, MetadataScope.SYSTEM, ImmutableMap.of(MetadataConstants.ENTITY_NAME_KEY, systemId.getEntityName()), ImmutableSet.of())), removeCreationTime(getMetadata(systemId.toMetadataEntity())));
    artifactClient.delete(systemId);
}

Example 4 with MetadataSearchResponse

use of io.cdap.cdap.proto.metadata.MetadataSearchResponse in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testCrossNamespaceSearchMetadata.

@Test
public void testCrossNamespaceSearchMetadata() throws Exception {
    NamespaceId namespace1 = new NamespaceId("ns1");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace1).build());
    NamespaceId namespace2 = new NamespaceId("ns2");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace2).build());
    try {
        appClient.deploy(namespace1, createAppJarFile(AllProgramsApp.class));
        appClient.deploy(namespace2, createAppJarFile(AllProgramsApp.class));
        // Add metadata to app
        Map<String, String> props = ImmutableMap.of("key1", "value1");
        Metadata meta = new Metadata(props, Collections.emptySet());
        ApplicationId app1Id = namespace1.app(AllProgramsApp.NAME);
        addProperties(app1Id, props);
        ApplicationId app2Id = namespace2.app(AllProgramsApp.NAME);
        addProperties(app2Id, props);
        MetadataSearchResponse results = super.searchMetadata(ImmutableList.of(), "value*", Collections.emptySet(), null, 0, 10, 0, null, false);
        Map<MetadataEntity, Metadata> expected = new HashMap<>();
        expected.put(app1Id.toMetadataEntity(), meta);
        expected.put(app2Id.toMetadataEntity(), meta);
        Map<MetadataEntity, Metadata> actual = new HashMap<>();
        for (MetadataSearchResultRecord record : results.getResults()) {
            actual.put(record.getMetadataEntity(), record.getMetadata().get(MetadataScope.USER));
        }
        Assert.assertEquals(expected, actual);
    } finally {
        namespaceClient.delete(namespace1);
        namespaceClient.delete(namespace2);
    }
}

Example 5 with MetadataSearchResponse

use of io.cdap.cdap.proto.metadata.MetadataSearchResponse in project cdap by caskdata.

the class MetadataServiceMainTest method testMetadataService.

@Test
public void testMetadataService() throws Exception {
    Injector injector = getServiceMainInstance(MetadataServiceMain.class).getInjector();
    DatasetId datasetId = NamespaceId.DEFAULT.dataset("testds");
    // Create a dataset, a metadata should get published.
    DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    long beforeCreation = System.currentTimeMillis();
    datasetFramework.addInstance(KeyValueTable.class.getName(), datasetId, DatasetProperties.EMPTY);
    // Query the metadata
    DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    Discoverable metadataEndpoint = new RandomEndpointStrategy(() -> discoveryServiceClient.discover(Constants.Service.METADATA_SERVICE)).pick(5, TimeUnit.SECONDS);
    Assert.assertNotNull(metadataEndpoint);
    // Try to query the metadata
    InetSocketAddress metadataAddr = metadataEndpoint.getSocketAddress();
    ConnectionConfig connConfig = ConnectionConfig.builder().setSSLEnabled(URIScheme.HTTPS.isMatch(metadataEndpoint)).setHostname(metadataAddr.getHostName()).setPort(metadataAddr.getPort()).build();
    MetadataClient metadataClient = new MetadataClient(ClientConfig.builder().setVerifySSLCert(false).setConnectionConfig(connConfig).build());
    MetadataSearchResponse response = metadataClient.searchMetadata(datasetId.getNamespaceId(), "*", (String) null);
    Set<MetadataSearchResultRecord> results = response.getResults();
    Assert.assertFalse(results.isEmpty());
    long creationTime = results.stream().filter(r -> datasetId.equals(r.getEntityId())).map(MetadataSearchResultRecord::getMetadata).map(metadata -> metadata.get(MetadataScope.SYSTEM).getProperties().get(MetadataConstants.CREATION_TIME_KEY)).map(Long::parseLong).findFirst().orElse(-1L);
    // The creation time should be between the beforeCreation time and the current time
    Assert.assertTrue(creationTime >= beforeCreation);
    Assert.assertTrue(creationTime <= System.currentTimeMillis());
}