Search in sources :

Example 1 with MetadataRecord

use of io.cdap.cdap.spi.metadata.MetadataRecord in project cdap by caskdata.

the class DatasetMetadataStorage method search.

@Override
public SearchResponse search(SearchRequest request) {
    Cursor cursor = request.getCursor() != null && !request.getCursor().isEmpty() ? Cursor.fromString(request.getCursor()) : null;
    Set<String> namespaces = cursor == null ? request.getNamespaces() : cursor.getNamespaces();
    ImmutablePair<NamespaceId, Set<EntityScope>> namespaceAndScopes = determineNamespaceAndScopes(namespaces);
    CursorAndOffsetInfo cursorOffsetAndLimits = determineCursorOffsetAndLimits(request, cursor);
    String query = cursor != null ? cursor.getQuery() : request.getQuery() == null || request.getQuery().isEmpty() ? "*" : request.getQuery();
    Set<String> types = cursor != null ? cursor.getTypes() : request.getTypes();
    types = types == null ? Collections.emptySet() : types;
    Sorting sorting = cursor == null ? request.getSorting() : cursor.getSorting() == null ? null : Sorting.of(cursor.getSorting());
    SortInfo sortInfo = sorting == null ? SortInfo.DEFAULT : new SortInfo(sorting.getKey(), SortInfo.SortOrder.valueOf(sorting.getOrder().name()));
    boolean showHidden = cursor != null ? cursor.isShowHidden() : request.isShowHidden();
    MetadataScope scope = cursor != null ? cursor.getScope() : request.getScope();
    MetadataSearchResponse response = search(new io.cdap.cdap.data2.metadata.dataset.SearchRequest(namespaceAndScopes.getFirst(), query, types, sortInfo, cursorOffsetAndLimits.getOffsetToRequest(), cursorOffsetAndLimits.getLimitToRequest(), request.isCursorRequested() ? 1 : 0, cursorOffsetAndLimits.getCursor(), showHidden, namespaceAndScopes.getSecond()), scope);
    // translate results back and limit them to at most what was requested (see above where we add 1)
    int limitToRespond = cursorOffsetAndLimits.getLimitToRespond();
    int offsetToRespond = cursorOffsetAndLimits.getOffsetToRespond();
    List<MetadataRecord> results = response.getResults().stream().limit(limitToRespond).map(record -> {
        Metadata metadata = Metadata.EMPTY;
        for (Map.Entry<MetadataScope, io.cdap.cdap.api.metadata.Metadata> entry : record.getMetadata().entrySet()) {
            Metadata toAdd = new Metadata(entry.getKey(), entry.getValue().getTags(), entry.getValue().getProperties());
            metadata = mergeDisjointMetadata(metadata, toAdd);
        }
        return new MetadataRecord(record.getMetadataEntity(), metadata);
    }).collect(Collectors.toList());
    Cursor newCursor = null;
    if (response.getCursors() != null && !response.getCursors().isEmpty()) {
        String actualCursor = response.getCursors().get(0);
        if (cursor != null) {
            // the new cursor's offset is the previous cursor's offset plus the number of results
            newCursor = new Cursor(cursor, cursor.getOffset() + results.size(), actualCursor);
        } else {
            newCursor = new Cursor(offsetToRespond + results.size(), limitToRespond, showHidden, scope, namespaces, types, sorting == null ? null : sorting.toString(), actualCursor, query);
        }
    }
    // adjust the total results by the difference of requested offset and the true offset that we respond back
    int totalResults = offsetToRespond - cursorOffsetAndLimits.getOffsetToRequest() + response.getTotal();
    return new SearchResponse(request, newCursor == null ? null : newCursor.toString(), offsetToRespond, limitToRespond, totalResults, results);
}
Also used : MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) ImmutablePair(io.cdap.cdap.common.utils.ImmutablePair) MetadataDirective(io.cdap.cdap.spi.metadata.MetadataDirective) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Inject(com.google.inject.Inject) HashMap(java.util.HashMap) USER(io.cdap.cdap.api.metadata.MetadataScope.USER) MetadataChange(io.cdap.cdap.spi.metadata.MetadataChange) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) HashSet(java.util.HashSet) TAG(io.cdap.cdap.spi.metadata.MetadataKind.TAG) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Metadata(io.cdap.cdap.spi.metadata.Metadata) Map(java.util.Map) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) Read(io.cdap.cdap.spi.metadata.Read) EnumSet(java.util.EnumSet) Sorting(io.cdap.cdap.spi.metadata.Sorting) SortInfo(io.cdap.cdap.data2.metadata.dataset.SortInfo) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) PROPERTY(io.cdap.cdap.spi.metadata.MetadataKind.PROPERTY) ImmutableMap(com.google.common.collect.ImmutableMap) Cursor(io.cdap.cdap.common.metadata.Cursor) SYSTEM(io.cdap.cdap.api.metadata.MetadataScope.SYSTEM) Set(java.util.Set) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) EntityScope(io.cdap.cdap.proto.EntityScope) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) Named(com.google.inject.name.Named) Constants(io.cdap.cdap.common.conf.Constants) VisibleForTesting(com.google.common.annotations.VisibleForTesting) DatasetDefinition(io.cdap.cdap.api.dataset.DatasetDefinition) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) MetadataDataset(io.cdap.cdap.data2.metadata.dataset.MetadataDataset) Collections(java.util.Collections) HashSet(java.util.HashSet) EnumSet(java.util.EnumSet) Set(java.util.Set) Metadata(io.cdap.cdap.spi.metadata.Metadata) Cursor(io.cdap.cdap.common.metadata.Cursor) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) Sorting(io.cdap.cdap.spi.metadata.Sorting) SortInfo(io.cdap.cdap.data2.metadata.dataset.SortInfo) MetadataSearchResponse(io.cdap.cdap.proto.metadata.MetadataSearchResponse) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) NamespaceId(io.cdap.cdap.proto.id.NamespaceId)

Example 2 with MetadataRecord

use of io.cdap.cdap.spi.metadata.MetadataRecord in project cdap by caskdata.

the class MetadataCompatibility method toV5Results.

/**
 * Convert a list of {@link MetadataRecord}s to an ordered set of 5.x {@link MetadataSearchResultRecord}s.
 *
 * The 5.x convention was that the results only contain non-empty records.
 */
private static Set<MetadataSearchResultRecord> toV5Results(List<MetadataRecord> results) {
    Set<MetadataSearchResultRecord> records = new LinkedHashSet<>();
    for (MetadataRecord record : results) {
        Map<MetadataScope, io.cdap.cdap.api.metadata.Metadata> map = toV5Metadata(record.getMetadata());
        records.add(new MetadataSearchResultRecord(record.getEntity(), Maps.filterValues(map, meta -> meta != null && !(meta.getProperties().isEmpty() && meta.getTags().isEmpty()))));
    }
    return records;
}
Also used : LinkedHashSet(java.util.LinkedHashSet) MetadataSearchResultRecord(io.cdap.cdap.proto.metadata.MetadataSearchResultRecord) Metadata(io.cdap.cdap.spi.metadata.Metadata) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope)

Example 3 with MetadataRecord

use of io.cdap.cdap.spi.metadata.MetadataRecord in project cdap by caskdata.

the class ElasticsearchMetadataStorageTest method testScrollTimeout.

@Test
public void testScrollTimeout() throws IOException, InterruptedException {
    MetadataStorage mds = getMetadataStorage();
    MutationOptions options = MutationOptions.builder().setAsynchronous(false).build();
    List<MetadataRecord> records = IntStream.range(0, 20).boxed().map(i -> new MetadataRecord(MetadataEntity.ofDataset("ns" + i, "ds" + i), new Metadata(MetadataScope.USER, tags("tag", "t" + i), props("p", "v" + i)))).collect(Collectors.toList());
    mds.batch(records.stream().map(r -> new Update(r.getEntity(), r.getMetadata())).collect(Collectors.toList()), options);
    SearchRequest request = SearchRequest.of("t*").setCursorRequested(true).setLimit(5).build();
    SearchResponse response = mds.search(request);
    Assert.assertEquals(5, response.getResults().size());
    Assert.assertNotNull(response.getCursor());
    SearchRequest request2 = SearchRequest.of("t*").setCursor(response.getCursor()).build();
    SearchResponse response2 = mds.search(request2);
    Assert.assertEquals(5, response2.getResults().size());
    // it works despite a cursor and an offset (which is equal to the cursor's)
    SearchRequest request2a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(5).build();
    SearchResponse response2a = mds.search(request2a);
    Assert.assertEquals(5, response2a.getOffset());
    Assert.assertEquals(5, response2a.getLimit());
    Assert.assertEquals(response2.getResults(), response2a.getResults());
    // it works despite a cursor and an offset (which is different from the cursor's)
    SearchRequest request2b = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response2b = mds.search(request2b);
    Assert.assertEquals(5, response2b.getOffset());
    Assert.assertEquals(5, response2b.getLimit());
    Assert.assertEquals(response2.getResults(), response2b.getResults());
    // sleep 1 sec longer than the configured scroll timeout to invalidate cursor
    TimeUnit.SECONDS.sleep(3);
    SearchResponse response3 = mds.search(request2);
    Assert.assertEquals(response2.getResults(), response3.getResults());
    Assert.assertEquals(response2.getCursor(), response3.getCursor());
    // it works despite an expired cursor and an offset (which is different from the cursor's)
    SearchRequest request3a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response3a = mds.search(request3a);
    Assert.assertEquals(5, response3a.getOffset());
    Assert.assertEquals(5, response3a.getLimit());
    Assert.assertEquals(response2.getResults(), response3a.getResults());
    // create a nonsense cursor and search with that
    Cursor cursor = Cursor.fromString(response.getCursor());
    cursor = new Cursor(cursor, cursor.getOffset(), "nosuchcursor");
    SearchRequest request4 = SearchRequest.of("t*").setCursor(cursor.toString()).build();
    SearchResponse response4 = mds.search(request4);
    // compare only the results, not the entire response (the cursor is different)
    Assert.assertEquals(response2.getResults(), response4.getResults());
    // clean up
    mds.batch(records.stream().map(MetadataRecord::getEntity).map(Drop::new).collect(Collectors.toList()), options);
}
Also used : IntStream(java.util.stream.IntStream) Drop(io.cdap.cdap.spi.metadata.MetadataMutation.Drop) BeforeClass(org.junit.BeforeClass) LoggerFactory(org.slf4j.LoggerFactory) Random(java.util.Random) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) ImmutableList(com.google.common.collect.ImmutableList) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Metadata(io.cdap.cdap.spi.metadata.Metadata) Closeables(com.google.common.io.Closeables) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) AfterClass(org.junit.AfterClass) ImmutableSet(com.google.common.collect.ImmutableSet) Logger(org.slf4j.Logger) Cursor(io.cdap.cdap.common.metadata.Cursor) Update(io.cdap.cdap.spi.metadata.MetadataMutation.Update) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) Test(org.junit.Test) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) Assert(org.junit.Assert) Collections(java.util.Collections) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) Metadata(io.cdap.cdap.spi.metadata.Metadata) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) Update(io.cdap.cdap.spi.metadata.MetadataMutation.Update) Cursor(io.cdap.cdap.common.metadata.Cursor) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) Drop(io.cdap.cdap.spi.metadata.MetadataMutation.Drop) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) Test(org.junit.Test)

Example 4 with MetadataRecord

use of io.cdap.cdap.spi.metadata.MetadataRecord in project cdap by caskdata.

the class ElasticsearchMetadataStorage method createSearchResponse.

private io.cdap.cdap.spi.metadata.SearchResponse createSearchResponse(SearchRequest request, SearchResponse response, String cursor, int offset, int limit) {
    SearchHits hits = response.getHits();
    List<MetadataRecord> results = fromHits(hits);
    return new io.cdap.cdap.spi.metadata.SearchResponse(request, cursor, offset, limit, (int) hits.getTotalHits(), results);
}
Also used : SearchHits(org.elasticsearch.search.SearchHits) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) SearchResponse(org.elasticsearch.action.search.SearchResponse)

Example 5 with MetadataRecord

use of io.cdap.cdap.spi.metadata.MetadataRecord in project cdap by caskdata.

the class DatasetMetadataStorageTest method testCrossNamespacePagination.

// this test is specific to teh DatasetMetadataStorage, because of the specific way it tests pagination:
// it requests offsets that are not a multiple of the page size, which is not supported in all implementations.
@Test
public void testCrossNamespacePagination() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    NamespaceId ns1Id = new NamespaceId("ns1");
    NamespaceId ns2Id = new NamespaceId("ns2");
    MetadataEntity ns1app1 = ns1Id.app("a1").toMetadataEntity();
    MetadataEntity ns1app2 = ns1Id.app("a2").toMetadataEntity();
    MetadataEntity ns1app3 = ns1Id.app("a3").toMetadataEntity();
    MetadataEntity ns2app1 = ns2Id.app("a1").toMetadataEntity();
    MetadataEntity ns2app2 = ns2Id.app("a2").toMetadataEntity();
    mds.batch(ImmutableList.of(new Update(ns1app1, new Metadata(USER, tags("v1"))), new Update(ns1app2, new Metadata(USER, tags("v1"))), new Update(ns1app3, new Metadata(USER, tags("v1"))), new Update(ns2app1, new Metadata(USER, tags("v1"))), new Update(ns2app2, new Metadata(USER, tags("v1")))), MutationOptions.DEFAULT);
    MetadataRecord record11 = new MetadataRecord(ns1app1, new Metadata(USER, tags("v1")));
    MetadataRecord record12 = new MetadataRecord(ns1app2, new Metadata(USER, tags("v1")));
    MetadataRecord record13 = new MetadataRecord(ns1app3, new Metadata(USER, tags("v1")));
    MetadataRecord record21 = new MetadataRecord(ns2app1, new Metadata(USER, tags("v1")));
    MetadataRecord record22 = new MetadataRecord(ns2app2, new Metadata(USER, tags("v1")));
    SearchResponse response = assertResults(mds, SearchRequest.of("*").setLimit(Integer.MAX_VALUE).setCursorRequested(true).build(), record11, record12, record13, record21, record22);
    // iterate over results to find the order in which they are returned
    Iterator<MetadataRecord> resultIter = response.getResults().iterator();
    MetadataRecord[] results = { resultIter.next(), resultIter.next(), resultIter.next(), resultIter.next(), resultIter.next() };
    // get 4 results (guaranteed to have at least one from each namespace), offset 1
    assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(1).setLimit(4).build(), results[1], results[2], results[3], results[4]);
    // get the first four
    assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(0).setLimit(4).build(), results[0], results[1], results[2], results[3]);
    // get middle 3
    assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(1).setLimit(3).build(), results[1], results[2], results[3], results[3]);
    // clean up
    mds.batch(ImmutableList.of(new Drop(ns1app1), new Drop(ns1app2), new Drop(ns1app3), new Drop(ns2app1), new Drop(ns2app2)), MutationOptions.DEFAULT);
}
Also used : MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) Metadata(io.cdap.cdap.spi.metadata.Metadata) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Update(io.cdap.cdap.spi.metadata.MetadataMutation.Update) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) Drop(io.cdap.cdap.spi.metadata.MetadataMutation.Drop) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) Test(org.junit.Test)

Aggregations

MetadataRecord (io.cdap.cdap.spi.metadata.MetadataRecord)6 Metadata (io.cdap.cdap.spi.metadata.Metadata)5 MetadataEntity (io.cdap.cdap.api.metadata.MetadataEntity)4 MetadataStorage (io.cdap.cdap.spi.metadata.MetadataStorage)4 SearchResponse (io.cdap.cdap.spi.metadata.SearchResponse)4 MetadataScope (io.cdap.cdap.api.metadata.MetadataScope)3 Cursor (io.cdap.cdap.common.metadata.Cursor)3 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)3 MutationOptions (io.cdap.cdap.spi.metadata.MutationOptions)3 ScopedNameOfKind (io.cdap.cdap.spi.metadata.ScopedNameOfKind)3 VisibleForTesting (com.google.common.annotations.VisibleForTesting)2 ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 Closeables (com.google.common.io.Closeables)2 SYSTEM (io.cdap.cdap.api.metadata.MetadataScope.SYSTEM)2 USER (io.cdap.cdap.api.metadata.MetadataScope.USER)2 ImmutablePair (io.cdap.cdap.common.utils.ImmutablePair)2 EntityScope (io.cdap.cdap.proto.EntityScope)2 MetadataKind (io.cdap.cdap.spi.metadata.MetadataKind)2