Examples with MetadataSearchResultRecord io.cdap.cdap.proto.metadata.MetadataSearchResultRecord used on opensource projects

Example 1 with MetadataSearchResultRecord

use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class SearchHelper method search.

private MetadataSearchResponse search(Set<MetadataScope> scopes, SearchRequest request) {
    List<MetadataEntry> results = new LinkedList<>();
    List<String> cursors = new LinkedList<>();
    for (MetadataScope scope : scopes) {
        SearchResults searchResults = execute(context -> context.getDataset(scope).search(request));
        results.addAll(searchResults.getResults());
        cursors.addAll(searchResults.getCursors());
    }
    int offset = request.getOffset();
    int limit = request.getLimit();
    SortInfo sortInfo = request.getSortInfo();
    // sort if required
    Set<MetadataEntity> sortedEntities = getSortedEntities(results, sortInfo);
    int total = sortedEntities.size();
    // pagination is not performed at the dataset level, because:
    // 1. scoring is needed for DEFAULT sort info. So perform it here for now.
    // 2. Even when using custom sorting, we need to remove elements from the beginning to the offset and the cursors
    // at the end
    // TODO: Figure out how all of this can be done server (HBase) side
    int startIndex = Math.min(request.getOffset(), sortedEntities.size());
    // Account for overflow
    int endIndex = (int) Math.min(Integer.MAX_VALUE, (long) offset + limit);
    endIndex = Math.min(endIndex, sortedEntities.size());
    // add 1 to maxIndex because end index is exclusive
    Set<MetadataEntity> subSortedEntities = new LinkedHashSet<>(ImmutableList.copyOf(sortedEntities).subList(startIndex, endIndex));
    // Fetch metadata for entities in the result list
    // Note: since the fetch is happening in a different transaction, the metadata for entities may have been
    // removed. It is okay not to have metadata for some results in case this happens.
    Set<MetadataSearchResultRecord> finalResults = execute(context -> addMetadataToEntities(subSortedEntities, fetchMetadata(context.getDataset(SYSTEM), subSortedEntities), fetchMetadata(context.getDataset(USER), subSortedEntities)));
    return new MetadataSearchResponse(sortInfo.getSortBy() + " " + sortInfo.getSortOrder(), offset, limit, request.getNumCursors(), total, finalResults, cursors, request.shouldShowHidden(), request.getEntityScopes());
}

Example 2 with MetadataSearchResultRecord

use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchMetadata.

@Test
public void testSearchMetadata() throws Exception {
    appClient.deploy(NamespaceId.DEFAULT, createAppJarFile(AllProgramsApp.class));
    // wait for the system metadata to be processed
    ApplicationId appId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    DatasetId datasetId = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME);
    Tasks.waitFor(false, () -> getProperties(appId, MetadataScope.SYSTEM).isEmpty(), 10, TimeUnit.SECONDS);
    Tasks.waitFor(false, () -> getProperties(datasetId, MetadataScope.SYSTEM).isEmpty(), 10, TimeUnit.SECONDS);
    Map<NamespacedEntityId, Metadata> expectedUserMetadata = new HashMap<>();
    // Add metadata to app
    Map<String, String> props = ImmutableMap.of("key1", "value1");
    Set<String> tags = ImmutableSet.of("tag1", "tag2");
    addProperties(appId, props);
    addTags(appId, tags);
    expectedUserMetadata.put(appId, new Metadata(props, tags));
    // Add metadata to dataset
    props = ImmutableMap.of("key10", "value10", "key11", "value11");
    tags = ImmutableSet.of("tag11");
    addProperties(datasetId, props);
    addTags(datasetId, tags);
    expectedUserMetadata.put(datasetId, new Metadata(props, tags));
    Set<MetadataSearchResultRecord> results = searchMetadata(NamespaceId.DEFAULT, "value*").getResults();
    // Verify results
    Assert.assertEquals(expectedUserMetadata.keySet(), extractEntityIds(results));
    for (MetadataSearchResultRecord result : results) {
        // User metadata has to match exactly since we know what we have set
        Assert.assertEquals(expectedUserMetadata.get(result.getEntityId()), result.getMetadata().get(MetadataScope.USER));
        // Make sure system metadata is returned, we cannot check for exact match since we haven't set it
        Metadata systemMetadata = result.getMetadata().get(MetadataScope.SYSTEM);
        Assert.assertNotNull(systemMetadata);
        Assert.assertFalse(systemMetadata.getProperties().isEmpty());
        Assert.assertFalse(systemMetadata.getTags().isEmpty());
    }
    // add metadata to field (custom entity)
    props = ImmutableMap.of("fKey1", "fValue1", "fKey2", "fValue2");
    tags = ImmutableSet.of("fTag1");
    MetadataEntity metadataEntity = MetadataEntity.builder(datasetId.toMetadataEntity()).appendAsType("field", "someField").build();
    addProperties(metadataEntity, props);
    addTags(metadataEntity, tags);
    Map<MetadataEntity, Metadata> expectedUserMetadataV2 = new HashMap<>();
    expectedUserMetadataV2.put(metadataEntity, new Metadata(props, tags));
    Set<MetadataSearchResultRecord> resultsV2 = super.searchMetadata(ImmutableList.of(NamespaceId.DEFAULT), "fValue*", ImmutableSet.of(), null, 0, Integer.MAX_VALUE, 0, null, false).getResults();
    // Verify results
    Assert.assertEquals(expectedUserMetadataV2.keySet(), ImmutableSet.copyOf(extractMetadataEntities(resultsV2)));
    for (MetadataSearchResultRecord result : resultsV2) {
        // User metadata has to match exactly since we know what we have set
        Assert.assertEquals(expectedUserMetadataV2.get(result.getMetadataEntity()), result.getMetadata().get(MetadataScope.USER));
        // Make sure system metadata is returned, we cannot check for exact match since we haven't set it
        Metadata systemMetadata = result.getMetadata().get(MetadataScope.SYSTEM);
        // custom entity should not have any system metadata for it
        Assert.assertNull(systemMetadata);
    }
}

Example 3 with MetadataSearchResultRecord

use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testCrossNamespaceSearchMetadata.

@Test
public void testCrossNamespaceSearchMetadata() throws Exception {
    NamespaceId namespace1 = new NamespaceId("ns1");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace1).build());
    NamespaceId namespace2 = new NamespaceId("ns2");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace2).build());
    try {
        appClient.deploy(namespace1, createAppJarFile(AllProgramsApp.class));
        appClient.deploy(namespace2, createAppJarFile(AllProgramsApp.class));
        // Add metadata to app
        Map<String, String> props = ImmutableMap.of("key1", "value1");
        Metadata meta = new Metadata(props, Collections.emptySet());
        ApplicationId app1Id = namespace1.app(AllProgramsApp.NAME);
        addProperties(app1Id, props);
        ApplicationId app2Id = namespace2.app(AllProgramsApp.NAME);
        addProperties(app2Id, props);
        MetadataSearchResponse results = super.searchMetadata(ImmutableList.of(), "value*", Collections.emptySet(), null, 0, 10, 0, null, false);
        Map<MetadataEntity, Metadata> expected = new HashMap<>();
        expected.put(app1Id.toMetadataEntity(), meta);
        expected.put(app2Id.toMetadataEntity(), meta);
        Map<MetadataEntity, Metadata> actual = new HashMap<>();
        for (MetadataSearchResultRecord record : results.getResults()) {
            actual.put(record.getMetadataEntity(), record.getMetadata().get(MetadataScope.USER));
        }
        Assert.assertEquals(expected, actual);
    } finally {
        namespaceClient.delete(namespace1);
        namespaceClient.delete(namespace2);
    }
}

Example 4 with MetadataSearchResultRecord

use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataCompatibility method toV5Results.

/**
 * Convert a list of {@link MetadataRecord}s to an ordered set of 5.x {@link MetadataSearchResultRecord}s.
 *
 * The 5.x convention was that the results only contain non-empty records.
 */
private static Set<MetadataSearchResultRecord> toV5Results(List<MetadataRecord> results) {
    Set<MetadataSearchResultRecord> records = new LinkedHashSet<>();
    for (MetadataRecord record : results) {
        Map<MetadataScope, io.cdap.cdap.api.metadata.Metadata> map = toV5Metadata(record.getMetadata());
        records.add(new MetadataSearchResultRecord(record.getEntity(), Maps.filterValues(map, meta -> meta != null && !(meta.getProperties().isEmpty() && meta.getTags().isEmpty()))));
    }
    return records;
}

Example 5 with MetadataSearchResultRecord

use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataServiceMainTest method testMetadataService.

@Test
public void testMetadataService() throws Exception {
    Injector injector = getServiceMainInstance(MetadataServiceMain.class).getInjector();
    DatasetId datasetId = NamespaceId.DEFAULT.dataset("testds");
    // Create a dataset, a metadata should get published.
    DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    long beforeCreation = System.currentTimeMillis();
    datasetFramework.addInstance(KeyValueTable.class.getName(), datasetId, DatasetProperties.EMPTY);
    // Query the metadata
    DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    Discoverable metadataEndpoint = new RandomEndpointStrategy(() -> discoveryServiceClient.discover(Constants.Service.METADATA_SERVICE)).pick(5, TimeUnit.SECONDS);
    Assert.assertNotNull(metadataEndpoint);
    // Try to query the metadata
    InetSocketAddress metadataAddr = metadataEndpoint.getSocketAddress();
    ConnectionConfig connConfig = ConnectionConfig.builder().setSSLEnabled(URIScheme.HTTPS.isMatch(metadataEndpoint)).setHostname(metadataAddr.getHostName()).setPort(metadataAddr.getPort()).build();
    MetadataClient metadataClient = new MetadataClient(ClientConfig.builder().setVerifySSLCert(false).setConnectionConfig(connConfig).build());
    MetadataSearchResponse response = metadataClient.searchMetadata(datasetId.getNamespaceId(), "*", (String) null);
    Set<MetadataSearchResultRecord> results = response.getResults();
    Assert.assertFalse(results.isEmpty());
    long creationTime = results.stream().filter(r -> datasetId.equals(r.getEntityId())).map(MetadataSearchResultRecord::getMetadata).map(metadata -> metadata.get(MetadataScope.SYSTEM).getProperties().get(MetadataConstants.CREATION_TIME_KEY)).map(Long::parseLong).findFirst().orElse(-1L);
    // The creation time should be between the beforeCreation time and the current time
    Assert.assertTrue(creationTime >= beforeCreation);
    Assert.assertTrue(creationTime <= System.currentTimeMillis());
}