use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.
the class SearchHelper method search.
private MetadataSearchResponse search(Set<MetadataScope> scopes, SearchRequest request) {
List<MetadataEntry> results = new LinkedList<>();
List<String> cursors = new LinkedList<>();
for (MetadataScope scope : scopes) {
SearchResults searchResults = execute(context -> context.getDataset(scope).search(request));
results.addAll(searchResults.getResults());
cursors.addAll(searchResults.getCursors());
}
int offset = request.getOffset();
int limit = request.getLimit();
SortInfo sortInfo = request.getSortInfo();
// sort if required
Set<MetadataEntity> sortedEntities = getSortedEntities(results, sortInfo);
int total = sortedEntities.size();
// pagination is not performed at the dataset level, because:
// 1. scoring is needed for DEFAULT sort info. So perform it here for now.
// 2. Even when using custom sorting, we need to remove elements from the beginning to the offset and the cursors
// at the end
// TODO: Figure out how all of this can be done server (HBase) side
int startIndex = Math.min(request.getOffset(), sortedEntities.size());
// Account for overflow
int endIndex = (int) Math.min(Integer.MAX_VALUE, (long) offset + limit);
endIndex = Math.min(endIndex, sortedEntities.size());
// add 1 to maxIndex because end index is exclusive
Set<MetadataEntity> subSortedEntities = new LinkedHashSet<>(ImmutableList.copyOf(sortedEntities).subList(startIndex, endIndex));
// Fetch metadata for entities in the result list
// Note: since the fetch is happening in a different transaction, the metadata for entities may have been
// removed. It is okay not to have metadata for some results in case this happens.
Set<MetadataSearchResultRecord> finalResults = execute(context -> addMetadataToEntities(subSortedEntities, fetchMetadata(context.getDataset(SYSTEM), subSortedEntities), fetchMetadata(context.getDataset(USER), subSortedEntities)));
return new MetadataSearchResponse(sortInfo.getSortBy() + " " + sortInfo.getSortOrder(), offset, limit, request.getNumCursors(), total, finalResults, cursors, request.shouldShowHidden(), request.getEntityScopes());
}
use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.
the class MetadataHttpHandlerTestRun method testSearchMetadata.
@Test
public void testSearchMetadata() throws Exception {
appClient.deploy(NamespaceId.DEFAULT, createAppJarFile(AllProgramsApp.class));
// wait for the system metadata to be processed
ApplicationId appId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
DatasetId datasetId = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME);
Tasks.waitFor(false, () -> getProperties(appId, MetadataScope.SYSTEM).isEmpty(), 10, TimeUnit.SECONDS);
Tasks.waitFor(false, () -> getProperties(datasetId, MetadataScope.SYSTEM).isEmpty(), 10, TimeUnit.SECONDS);
Map<NamespacedEntityId, Metadata> expectedUserMetadata = new HashMap<>();
// Add metadata to app
Map<String, String> props = ImmutableMap.of("key1", "value1");
Set<String> tags = ImmutableSet.of("tag1", "tag2");
addProperties(appId, props);
addTags(appId, tags);
expectedUserMetadata.put(appId, new Metadata(props, tags));
// Add metadata to dataset
props = ImmutableMap.of("key10", "value10", "key11", "value11");
tags = ImmutableSet.of("tag11");
addProperties(datasetId, props);
addTags(datasetId, tags);
expectedUserMetadata.put(datasetId, new Metadata(props, tags));
Set<MetadataSearchResultRecord> results = searchMetadata(NamespaceId.DEFAULT, "value*").getResults();
// Verify results
Assert.assertEquals(expectedUserMetadata.keySet(), extractEntityIds(results));
for (MetadataSearchResultRecord result : results) {
// User metadata has to match exactly since we know what we have set
Assert.assertEquals(expectedUserMetadata.get(result.getEntityId()), result.getMetadata().get(MetadataScope.USER));
// Make sure system metadata is returned, we cannot check for exact match since we haven't set it
Metadata systemMetadata = result.getMetadata().get(MetadataScope.SYSTEM);
Assert.assertNotNull(systemMetadata);
Assert.assertFalse(systemMetadata.getProperties().isEmpty());
Assert.assertFalse(systemMetadata.getTags().isEmpty());
}
// add metadata to field (custom entity)
props = ImmutableMap.of("fKey1", "fValue1", "fKey2", "fValue2");
tags = ImmutableSet.of("fTag1");
MetadataEntity metadataEntity = MetadataEntity.builder(datasetId.toMetadataEntity()).appendAsType("field", "someField").build();
addProperties(metadataEntity, props);
addTags(metadataEntity, tags);
Map<MetadataEntity, Metadata> expectedUserMetadataV2 = new HashMap<>();
expectedUserMetadataV2.put(metadataEntity, new Metadata(props, tags));
Set<MetadataSearchResultRecord> resultsV2 = super.searchMetadata(ImmutableList.of(NamespaceId.DEFAULT), "fValue*", ImmutableSet.of(), null, 0, Integer.MAX_VALUE, 0, null, false).getResults();
// Verify results
Assert.assertEquals(expectedUserMetadataV2.keySet(), ImmutableSet.copyOf(extractMetadataEntities(resultsV2)));
for (MetadataSearchResultRecord result : resultsV2) {
// User metadata has to match exactly since we know what we have set
Assert.assertEquals(expectedUserMetadataV2.get(result.getMetadataEntity()), result.getMetadata().get(MetadataScope.USER));
// Make sure system metadata is returned, we cannot check for exact match since we haven't set it
Metadata systemMetadata = result.getMetadata().get(MetadataScope.SYSTEM);
// custom entity should not have any system metadata for it
Assert.assertNull(systemMetadata);
}
}
use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.
the class MetadataHttpHandlerTestRun method testCrossNamespaceSearchMetadata.
@Test
public void testCrossNamespaceSearchMetadata() throws Exception {
NamespaceId namespace1 = new NamespaceId("ns1");
namespaceClient.create(new NamespaceMeta.Builder().setName(namespace1).build());
NamespaceId namespace2 = new NamespaceId("ns2");
namespaceClient.create(new NamespaceMeta.Builder().setName(namespace2).build());
try {
appClient.deploy(namespace1, createAppJarFile(AllProgramsApp.class));
appClient.deploy(namespace2, createAppJarFile(AllProgramsApp.class));
// Add metadata to app
Map<String, String> props = ImmutableMap.of("key1", "value1");
Metadata meta = new Metadata(props, Collections.emptySet());
ApplicationId app1Id = namespace1.app(AllProgramsApp.NAME);
addProperties(app1Id, props);
ApplicationId app2Id = namespace2.app(AllProgramsApp.NAME);
addProperties(app2Id, props);
MetadataSearchResponse results = super.searchMetadata(ImmutableList.of(), "value*", Collections.emptySet(), null, 0, 10, 0, null, false);
Map<MetadataEntity, Metadata> expected = new HashMap<>();
expected.put(app1Id.toMetadataEntity(), meta);
expected.put(app2Id.toMetadataEntity(), meta);
Map<MetadataEntity, Metadata> actual = new HashMap<>();
for (MetadataSearchResultRecord record : results.getResults()) {
actual.put(record.getMetadataEntity(), record.getMetadata().get(MetadataScope.USER));
}
Assert.assertEquals(expected, actual);
} finally {
namespaceClient.delete(namespace1);
namespaceClient.delete(namespace2);
}
}
use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.
the class MetadataCompatibility method toV5Results.
/**
* Convert a list of {@link MetadataRecord}s to an ordered set of 5.x {@link MetadataSearchResultRecord}s.
*
* The 5.x convention was that the results only contain non-empty records.
*/
private static Set<MetadataSearchResultRecord> toV5Results(List<MetadataRecord> results) {
Set<MetadataSearchResultRecord> records = new LinkedHashSet<>();
for (MetadataRecord record : results) {
Map<MetadataScope, io.cdap.cdap.api.metadata.Metadata> map = toV5Metadata(record.getMetadata());
records.add(new MetadataSearchResultRecord(record.getEntity(), Maps.filterValues(map, meta -> meta != null && !(meta.getProperties().isEmpty() && meta.getTags().isEmpty()))));
}
return records;
}
use of io.cdap.cdap.proto.metadata.MetadataSearchResultRecord in project cdap by caskdata.
the class MetadataServiceMainTest method testMetadataService.
@Test
public void testMetadataService() throws Exception {
Injector injector = getServiceMainInstance(MetadataServiceMain.class).getInjector();
DatasetId datasetId = NamespaceId.DEFAULT.dataset("testds");
// Create a dataset, a metadata should get published.
DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
long beforeCreation = System.currentTimeMillis();
datasetFramework.addInstance(KeyValueTable.class.getName(), datasetId, DatasetProperties.EMPTY);
// Query the metadata
DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
Discoverable metadataEndpoint = new RandomEndpointStrategy(() -> discoveryServiceClient.discover(Constants.Service.METADATA_SERVICE)).pick(5, TimeUnit.SECONDS);
Assert.assertNotNull(metadataEndpoint);
// Try to query the metadata
InetSocketAddress metadataAddr = metadataEndpoint.getSocketAddress();
ConnectionConfig connConfig = ConnectionConfig.builder().setSSLEnabled(URIScheme.HTTPS.isMatch(metadataEndpoint)).setHostname(metadataAddr.getHostName()).setPort(metadataAddr.getPort()).build();
MetadataClient metadataClient = new MetadataClient(ClientConfig.builder().setVerifySSLCert(false).setConnectionConfig(connConfig).build());
MetadataSearchResponse response = metadataClient.searchMetadata(datasetId.getNamespaceId(), "*", (String) null);
Set<MetadataSearchResultRecord> results = response.getResults();
Assert.assertFalse(results.isEmpty());
long creationTime = results.stream().filter(r -> datasetId.equals(r.getEntityId())).map(MetadataSearchResultRecord::getMetadata).map(metadata -> metadata.get(MetadataScope.SYSTEM).getProperties().get(MetadataConstants.CREATION_TIME_KEY)).map(Long::parseLong).findFirst().orElse(-1L);
// The creation time should be between the beforeCreation time and the current time
Assert.assertTrue(creationTime >= beforeCreation);
Assert.assertTrue(creationTime <= System.currentTimeMillis());
}
Aggregations