Search in sources :

Example 1 with MetadataSearchResultRecord

use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class SearchHelper method search.

private MetadataSearchResponse search(Set<MetadataScope> scopes, SearchRequest request) {
    List<MetadataEntry> results = new LinkedList<>();
    List<String> cursors = new LinkedList<>();
    for (MetadataScope scope : scopes) {
        SearchResults searchResults = execute(context -> context.getDataset(scope).search(request));
        results.addAll(searchResults.getResults());
        cursors.addAll(searchResults.getCursors());
    }
    int offset = request.getOffset();
    int limit = request.getLimit();
    SortInfo sortInfo = request.getSortInfo();
    // sort if required
    Set<MetadataEntity> sortedEntities = getSortedEntities(results, sortInfo);
    int total = sortedEntities.size();
    // pagination is not performed at the dataset level, because:
    // 1. scoring is needed for DEFAULT sort info. So perform it here for now.
    // 2. Even when using custom sorting, we need to remove elements from the beginning to the offset and the cursors
    // at the end
    // TODO: Figure out how all of this can be done server (HBase) side
    int startIndex = Math.min(request.getOffset(), sortedEntities.size());
    // Account for overflow
    int endIndex = (int) Math.min(Integer.MAX_VALUE, (long) offset + limit);
    endIndex = Math.min(endIndex, sortedEntities.size());
    // add 1 to maxIndex because end index is exclusive
    Set<MetadataEntity> subSortedEntities = new LinkedHashSet<>(ImmutableList.copyOf(sortedEntities).subList(startIndex, endIndex));
    // Fetch metadata for entities in the result list
    // Note: since the fetch is happening in a different transaction, the metadata for entities may have been
    // removed. It is okay not to have metadata for some results in case this happens.
    Set<MetadataSearchResultRecord> finalResults = execute(context -> addMetadataToEntities(subSortedEntities, fetchMetadata(context.getDataset(SYSTEM), subSortedEntities), fetchMetadata(context.getDataset(USER), subSortedEntities)));
    return new MetadataSearchResponse(sortInfo.getSortBy() + " " + sortInfo.getSortOrder(), offset, limit, request.getNumCursors(), total, finalResults, cursors, request.shouldShowHidden(), request.getEntityScopes());
}
Also used : LinkedHashSet(java.util.LinkedHashSet) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) SearchResults(io.cdap.cdap.data2.metadata.dataset.SearchResults) LinkedList(java.util.LinkedList) SortInfo(io.cdap.cdap.data2.metadata.dataset.SortInfo) MetadataSearchResultRecord(io.cdap.cdap.proto.metadata.MetadataSearchResultRecord) MetadataEntry(io.cdap.cdap.data2.metadata.dataset.MetadataEntry) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope)

Example 2 with MetadataSearchResultRecord

use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testSearchMetadata.

@Test
public void testSearchMetadata() throws Exception {
    appClient.deploy(NamespaceId.DEFAULT, createAppJarFile(AllProgramsApp.class));
    // wait for the system metadata to be processed
    ApplicationId appId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    DatasetId datasetId = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME);
    Tasks.waitFor(false, () -> getProperties(appId, MetadataScope.SYSTEM).isEmpty(), 10, TimeUnit.SECONDS);
    Tasks.waitFor(false, () -> getProperties(datasetId, MetadataScope.SYSTEM).isEmpty(), 10, TimeUnit.SECONDS);
    Map<NamespacedEntityId, Metadata> expectedUserMetadata = new HashMap<>();
    // Add metadata to app
    Map<String, String> props = ImmutableMap.of("key1", "value1");
    Set<String> tags = ImmutableSet.of("tag1", "tag2");
    addProperties(appId, props);
    addTags(appId, tags);
    expectedUserMetadata.put(appId, new Metadata(props, tags));
    // Add metadata to dataset
    props = ImmutableMap.of("key10", "value10", "key11", "value11");
    tags = ImmutableSet.of("tag11");
    addProperties(datasetId, props);
    addTags(datasetId, tags);
    expectedUserMetadata.put(datasetId, new Metadata(props, tags));
    Set<MetadataSearchResultRecord> results = searchMetadata(NamespaceId.DEFAULT, "value*").getResults();
    // Verify results
    Assert.assertEquals(expectedUserMetadata.keySet(), extractEntityIds(results));
    for (MetadataSearchResultRecord result : results) {
        // User metadata has to match exactly since we know what we have set
        Assert.assertEquals(expectedUserMetadata.get(result.getEntityId()), result.getMetadata().get(MetadataScope.USER));
        // Make sure system metadata is returned, we cannot check for exact match since we haven't set it
        Metadata systemMetadata = result.getMetadata().get(MetadataScope.SYSTEM);
        Assert.assertNotNull(systemMetadata);
        Assert.assertFalse(systemMetadata.getProperties().isEmpty());
        Assert.assertFalse(systemMetadata.getTags().isEmpty());
    }
    // add metadata to field (custom entity)
    props = ImmutableMap.of("fKey1", "fValue1", "fKey2", "fValue2");
    tags = ImmutableSet.of("fTag1");
    MetadataEntity metadataEntity = MetadataEntity.builder(datasetId.toMetadataEntity()).appendAsType("field", "someField").build();
    addProperties(metadataEntity, props);
    addTags(metadataEntity, tags);
    Map<MetadataEntity, Metadata> expectedUserMetadataV2 = new HashMap<>();
    expectedUserMetadataV2.put(metadataEntity, new Metadata(props, tags));
    Set<MetadataSearchResultRecord> resultsV2 = super.searchMetadata(ImmutableList.of(NamespaceId.DEFAULT), "fValue*", ImmutableSet.of(), null, 0, Integer.MAX_VALUE, 0, null, false).getResults();
    // Verify results
    Assert.assertEquals(expectedUserMetadataV2.keySet(), ImmutableSet.copyOf(extractMetadataEntities(resultsV2)));
    for (MetadataSearchResultRecord result : resultsV2) {
        // User metadata has to match exactly since we know what we have set
        Assert.assertEquals(expectedUserMetadataV2.get(result.getMetadataEntity()), result.getMetadata().get(MetadataScope.USER));
        // Make sure system metadata is returned, we cannot check for exact match since we haven't set it
        Metadata systemMetadata = result.getMetadata().get(MetadataScope.SYSTEM);
        // custom entity should not have any system metadata for it
        Assert.assertNull(systemMetadata);
    }
}
Also used : MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) HashMap(java.util.HashMap) Metadata(io.cdap.cdap.api.metadata.Metadata) AllProgramsApp(io.cdap.cdap.client.app.AllProgramsApp) DatasetId(io.cdap.cdap.proto.id.DatasetId) NamespacedEntityId(io.cdap.cdap.proto.id.NamespacedEntityId) MetadataSearchResultRecord(io.cdap.cdap.proto.metadata.MetadataSearchResultRecord) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Example 3 with MetadataSearchResultRecord

use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataHttpHandlerTestRun method testCrossNamespaceSearchMetadata.

@Test
public void testCrossNamespaceSearchMetadata() throws Exception {
    NamespaceId namespace1 = new NamespaceId("ns1");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace1).build());
    NamespaceId namespace2 = new NamespaceId("ns2");
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace2).build());
    try {
        appClient.deploy(namespace1, createAppJarFile(AllProgramsApp.class));
        appClient.deploy(namespace2, createAppJarFile(AllProgramsApp.class));
        // Add metadata to app
        Map<String, String> props = ImmutableMap.of("key1", "value1");
        Metadata meta = new Metadata(props, Collections.emptySet());
        ApplicationId app1Id = namespace1.app(AllProgramsApp.NAME);
        addProperties(app1Id, props);
        ApplicationId app2Id = namespace2.app(AllProgramsApp.NAME);
        addProperties(app2Id, props);
        MetadataSearchResponse results = super.searchMetadata(ImmutableList.of(), "value*", Collections.emptySet(), null, 0, 10, 0, null, false);
        Map<MetadataEntity, Metadata> expected = new HashMap<>();
        expected.put(app1Id.toMetadataEntity(), meta);
        expected.put(app2Id.toMetadataEntity(), meta);
        Map<MetadataEntity, Metadata> actual = new HashMap<>();
        for (MetadataSearchResultRecord record : results.getResults()) {
            actual.put(record.getMetadataEntity(), record.getMetadata().get(MetadataScope.USER));
        }
        Assert.assertEquals(expected, actual);
    } finally {
        namespaceClient.delete(namespace1);
        namespaceClient.delete(namespace2);
    }
}
Also used : MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) HashMap(java.util.HashMap) Metadata(io.cdap.cdap.api.metadata.Metadata) MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) AllProgramsApp(io.cdap.cdap.client.app.AllProgramsApp) MetadataSearchResultRecord(io.cdap.cdap.proto.metadata.MetadataSearchResultRecord) NamespaceMeta(io.cdap.cdap.proto.NamespaceMeta) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Example 4 with MetadataSearchResultRecord

use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataCompatibility method toV5Results.

/**
 * Convert a list of {@link MetadataRecord}s to an ordered set of 5.x {@link MetadataSearchResultRecord}s.
 *
 * The 5.x convention was that the results only contain non-empty records.
 */
private static Set<MetadataSearchResultRecord> toV5Results(List<MetadataRecord> results) {
    Set<MetadataSearchResultRecord> records = new LinkedHashSet<>();
    for (MetadataRecord record : results) {
        Map<MetadataScope, io.cdap.cdap.api.metadata.Metadata> map = toV5Metadata(record.getMetadata());
        records.add(new MetadataSearchResultRecord(record.getEntity(), Maps.filterValues(map, meta -> meta != null && !(meta.getProperties().isEmpty() && meta.getTags().isEmpty()))));
    }
    return records;
}
Also used : LinkedHashSet(java.util.LinkedHashSet) MetadataSearchResultRecord(io.cdap.cdap.proto.metadata.MetadataSearchResultRecord) Metadata(io.cdap.cdap.spi.metadata.Metadata) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope)

Example 5 with MetadataSearchResultRecord

use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.

the class MetadataServiceMainTest method testMetadataService.

@Test
public void testMetadataService() throws Exception {
    Injector injector = getServiceMainInstance(MetadataServiceMain.class).getInjector();
    DatasetId datasetId = NamespaceId.DEFAULT.dataset("testds");
    // Create a dataset, a metadata should get published.
    DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    long beforeCreation = System.currentTimeMillis();
    datasetFramework.addInstance(KeyValueTable.class.getName(), datasetId, DatasetProperties.EMPTY);
    // Query the metadata
    DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    Discoverable metadataEndpoint = new RandomEndpointStrategy(() -> discoveryServiceClient.discover(Constants.Service.METADATA_SERVICE)).pick(5, TimeUnit.SECONDS);
    Assert.assertNotNull(metadataEndpoint);
    // Try to query the metadata
    InetSocketAddress metadataAddr = metadataEndpoint.getSocketAddress();
    ConnectionConfig connConfig = ConnectionConfig.builder().setSSLEnabled(URIScheme.HTTPS.isMatch(metadataEndpoint)).setHostname(metadataAddr.getHostName()).setPort(metadataAddr.getPort()).build();
    MetadataClient metadataClient = new MetadataClient(ClientConfig.builder().setVerifySSLCert(false).setConnectionConfig(connConfig).build());
    MetadataSearchResponse response = metadataClient.searchMetadata(datasetId.getNamespaceId(), "*", (String) null);
    Set<MetadataSearchResultRecord> results = response.getResults();
    Assert.assertFalse(results.isEmpty());
    long creationTime = results.stream().filter(r -> datasetId.equals(r.getEntityId())).map(MetadataSearchResultRecord::getMetadata).map(metadata -> metadata.get(MetadataScope.SYSTEM).getProperties().get(MetadataConstants.CREATION_TIME_KEY)).map(Long::parseLong).findFirst().orElse(-1L);
    // The creation time should be between the beforeCreation time and the current time
    Assert.assertTrue(creationTime >= beforeCreation);
    Assert.assertTrue(creationTime <= System.currentTimeMillis());
}
Also used : MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) RandomEndpointStrategy(io.cdap.cdap.common.discovery.RandomEndpointStrategy) URIScheme(io.cdap.cdap.common.discovery.URIScheme) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) DatasetFramework(io.cdap.cdap.data2.dataset2.DatasetFramework) Set(java.util.Set) Test(org.junit.Test) InetSocketAddress(java.net.InetSocketAddress) Injector(com.google.inject.Injector) DatasetProperties(io.cdap.cdap.api.dataset.DatasetProperties) TimeUnit(java.util.concurrent.TimeUnit) MetadataClient(io.cdap.cdap.client.MetadataClient) Discoverable(org.apache.twill.discovery.Discoverable) MetadataSearchResultRecord(io.cdap.cdap.proto.metadata.MetadataSearchResultRecord) DatasetId(io.cdap.cdap.proto.id.DatasetId) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) ClientConfig(io.cdap.cdap.client.config.ClientConfig) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataConstants(io.cdap.cdap.spi.metadata.MetadataConstants) ConnectionConfig(io.cdap.cdap.client.config.ConnectionConfig) Constants(io.cdap.cdap.common.conf.Constants) Assert(org.junit.Assert) KeyValueTable(io.cdap.cdap.api.dataset.lib.KeyValueTable) Discoverable(org.apache.twill.discovery.Discoverable) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) InetSocketAddress(java.net.InetSocketAddress) MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) DatasetId(io.cdap.cdap.proto.id.DatasetId) DatasetFramework(io.cdap.cdap.data2.dataset2.DatasetFramework) MetadataClient(io.cdap.cdap.client.MetadataClient) MetadataSearchResultRecord(io.cdap.cdap.proto.metadata.MetadataSearchResultRecord) Injector(com.google.inject.Injector) KeyValueTable(io.cdap.cdap.api.dataset.lib.KeyValueTable) ConnectionConfig(io.cdap.cdap.client.config.ConnectionConfig) RandomEndpointStrategy(io.cdap.cdap.common.discovery.RandomEndpointStrategy) Test(org.junit.Test)

Aggregations

MetadataSearchResultRecord (io.cdap.cdap.proto.metadata.MetadataSearchResultRecord)8 MetadataSearchResponse (io.cdap.cdap.proto.metadata.MetadataSearchResponse)5 MetadataEntity (io.cdap.cdap.api.metadata.MetadataEntity)4 MetadataScope (io.cdap.cdap.api.metadata.MetadataScope)4 Metadata (io.cdap.cdap.api.metadata.Metadata)3 ApplicationId (io.cdap.cdap.proto.id.ApplicationId)3 LinkedHashSet (java.util.LinkedHashSet)3 MetadataClient (io.cdap.cdap.client.MetadataClient)2 AllProgramsApp (io.cdap.cdap.client.app.AllProgramsApp)2 DatasetId (io.cdap.cdap.proto.id.DatasetId)2 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)2 HashMap (java.util.HashMap)2 Set (java.util.Set)2 Test (org.junit.Test)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Splitter (com.google.common.base.Splitter)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Lists (com.google.common.collect.Lists)1 Inject (com.google.inject.Inject)1