Search in sources :

Example 1 with MutationOptions

use of io.cdap.cdap.spi.metadata.MutationOptions in project cdap by caskdata.

the class ElasticsearchMetadataStorageTest method testScrollTimeout.

@Test
public void testScrollTimeout() throws IOException, InterruptedException {
    MetadataStorage mds = getMetadataStorage();
    MutationOptions options = MutationOptions.builder().setAsynchronous(false).build();
    List<MetadataRecord> records = IntStream.range(0, 20).boxed().map(i -> new MetadataRecord(MetadataEntity.ofDataset("ns" + i, "ds" + i), new Metadata(MetadataScope.USER, tags("tag", "t" + i), props("p", "v" + i)))).collect(Collectors.toList());
    mds.batch(records.stream().map(r -> new Update(r.getEntity(), r.getMetadata())).collect(Collectors.toList()), options);
    SearchRequest request = SearchRequest.of("t*").setCursorRequested(true).setLimit(5).build();
    SearchResponse response = mds.search(request);
    Assert.assertEquals(5, response.getResults().size());
    Assert.assertNotNull(response.getCursor());
    SearchRequest request2 = SearchRequest.of("t*").setCursor(response.getCursor()).build();
    SearchResponse response2 = mds.search(request2);
    Assert.assertEquals(5, response2.getResults().size());
    // it works despite a cursor and an offset (which is equal to the cursor's)
    SearchRequest request2a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(5).build();
    SearchResponse response2a = mds.search(request2a);
    Assert.assertEquals(5, response2a.getOffset());
    Assert.assertEquals(5, response2a.getLimit());
    Assert.assertEquals(response2.getResults(), response2a.getResults());
    // it works despite a cursor and an offset (which is different from the cursor's)
    SearchRequest request2b = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response2b = mds.search(request2b);
    Assert.assertEquals(5, response2b.getOffset());
    Assert.assertEquals(5, response2b.getLimit());
    Assert.assertEquals(response2.getResults(), response2b.getResults());
    // sleep 1 sec longer than the configured scroll timeout to invalidate cursor
    TimeUnit.SECONDS.sleep(3);
    SearchResponse response3 = mds.search(request2);
    Assert.assertEquals(response2.getResults(), response3.getResults());
    Assert.assertEquals(response2.getCursor(), response3.getCursor());
    // it works despite an expired cursor and an offset (which is different from the cursor's)
    SearchRequest request3a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response3a = mds.search(request3a);
    Assert.assertEquals(5, response3a.getOffset());
    Assert.assertEquals(5, response3a.getLimit());
    Assert.assertEquals(response2.getResults(), response3a.getResults());
    // create a nonsense cursor and search with that
    Cursor cursor = Cursor.fromString(response.getCursor());
    cursor = new Cursor(cursor, cursor.getOffset(), "nosuchcursor");
    SearchRequest request4 = SearchRequest.of("t*").setCursor(cursor.toString()).build();
    SearchResponse response4 = mds.search(request4);
    // compare only the results, not the entire response (the cursor is different)
    Assert.assertEquals(response2.getResults(), response4.getResults());
    // clean up
    mds.batch(records.stream().map(MetadataRecord::getEntity).map(Drop::new).collect(Collectors.toList()), options);
}
Also used : IntStream(java.util.stream.IntStream) Drop(io.cdap.cdap.spi.metadata.MetadataMutation.Drop) BeforeClass(org.junit.BeforeClass) LoggerFactory(org.slf4j.LoggerFactory) Random(java.util.Random) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) ImmutableList(com.google.common.collect.ImmutableList) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Metadata(io.cdap.cdap.spi.metadata.Metadata) Closeables(com.google.common.io.Closeables) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) AfterClass(org.junit.AfterClass) ImmutableSet(com.google.common.collect.ImmutableSet) Logger(org.slf4j.Logger) Cursor(io.cdap.cdap.common.metadata.Cursor) Update(io.cdap.cdap.spi.metadata.MetadataMutation.Update) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) Test(org.junit.Test) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) Assert(org.junit.Assert) Collections(java.util.Collections) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) Metadata(io.cdap.cdap.spi.metadata.Metadata) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) Update(io.cdap.cdap.spi.metadata.MetadataMutation.Update) Cursor(io.cdap.cdap.common.metadata.Cursor) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) Drop(io.cdap.cdap.spi.metadata.MetadataMutation.Drop) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) Test(org.junit.Test)

Aggregations

ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Closeables (com.google.common.io.Closeables)1 MetadataEntity (io.cdap.cdap.api.metadata.MetadataEntity)1 MetadataScope (io.cdap.cdap.api.metadata.MetadataScope)1 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)1 SConfiguration (io.cdap.cdap.common.conf.SConfiguration)1 Cursor (io.cdap.cdap.common.metadata.Cursor)1 Metadata (io.cdap.cdap.spi.metadata.Metadata)1 MetadataKind (io.cdap.cdap.spi.metadata.MetadataKind)1 Drop (io.cdap.cdap.spi.metadata.MetadataMutation.Drop)1 Update (io.cdap.cdap.spi.metadata.MetadataMutation.Update)1 MetadataRecord (io.cdap.cdap.spi.metadata.MetadataRecord)1 MetadataStorage (io.cdap.cdap.spi.metadata.MetadataStorage)1 MetadataStorageTest (io.cdap.cdap.spi.metadata.MetadataStorageTest)1 MutationOptions (io.cdap.cdap.spi.metadata.MutationOptions)1 ScopedName (io.cdap.cdap.spi.metadata.ScopedName)1 ScopedNameOfKind (io.cdap.cdap.spi.metadata.ScopedNameOfKind)1 SearchRequest (io.cdap.cdap.spi.metadata.SearchRequest)1 SearchResponse (io.cdap.cdap.spi.metadata.SearchResponse)1