Examples with Update io.cdap.cdap.spi.metadata.MetadataMutation.Update used on opensource projects

Example 1 with Update

use of io.cdap.cdap.spi.metadata.MetadataMutation.Update in project cdap by caskdata.

the class ElasticsearchMetadataStorageTest method testScrollTimeout.

@Test
public void testScrollTimeout() throws IOException, InterruptedException {
    MetadataStorage mds = getMetadataStorage();
    MutationOptions options = MutationOptions.builder().setAsynchronous(false).build();
    List<MetadataRecord> records = IntStream.range(0, 20).boxed().map(i -> new MetadataRecord(MetadataEntity.ofDataset("ns" + i, "ds" + i), new Metadata(MetadataScope.USER, tags("tag", "t" + i), props("p", "v" + i)))).collect(Collectors.toList());
    mds.batch(records.stream().map(r -> new Update(r.getEntity(), r.getMetadata())).collect(Collectors.toList()), options);
    SearchRequest request = SearchRequest.of("t*").setCursorRequested(true).setLimit(5).build();
    SearchResponse response = mds.search(request);
    Assert.assertEquals(5, response.getResults().size());
    Assert.assertNotNull(response.getCursor());
    SearchRequest request2 = SearchRequest.of("t*").setCursor(response.getCursor()).build();
    SearchResponse response2 = mds.search(request2);
    Assert.assertEquals(5, response2.getResults().size());
    // it works despite a cursor and an offset (which is equal to the cursor's)
    SearchRequest request2a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(5).build();
    SearchResponse response2a = mds.search(request2a);
    Assert.assertEquals(5, response2a.getOffset());
    Assert.assertEquals(5, response2a.getLimit());
    Assert.assertEquals(response2.getResults(), response2a.getResults());
    // it works despite a cursor and an offset (which is different from the cursor's)
    SearchRequest request2b = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response2b = mds.search(request2b);
    Assert.assertEquals(5, response2b.getOffset());
    Assert.assertEquals(5, response2b.getLimit());
    Assert.assertEquals(response2.getResults(), response2b.getResults());
    // sleep 1 sec longer than the configured scroll timeout to invalidate cursor
    TimeUnit.SECONDS.sleep(3);
    SearchResponse response3 = mds.search(request2);
    Assert.assertEquals(response2.getResults(), response3.getResults());
    Assert.assertEquals(response2.getCursor(), response3.getCursor());
    // it works despite an expired cursor and an offset (which is different from the cursor's)
    SearchRequest request3a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response3a = mds.search(request3a);
    Assert.assertEquals(5, response3a.getOffset());
    Assert.assertEquals(5, response3a.getLimit());
    Assert.assertEquals(response2.getResults(), response3a.getResults());
    // create a nonsense cursor and search with that
    Cursor cursor = Cursor.fromString(response.getCursor());
    cursor = new Cursor(cursor, cursor.getOffset(), "nosuchcursor");
    SearchRequest request4 = SearchRequest.of("t*").setCursor(cursor.toString()).build();
    SearchResponse response4 = mds.search(request4);
    // compare only the results, not the entire response (the cursor is different)
    Assert.assertEquals(response2.getResults(), response4.getResults());
    // clean up
    mds.batch(records.stream().map(MetadataRecord::getEntity).map(Drop::new).collect(Collectors.toList()), options);
}

Example 2 with Update

use of io.cdap.cdap.spi.metadata.MetadataMutation.Update in project cdap by caskdata.

the class MetadataStorageTest method testSearchOnSchema.

@Test
public void testSearchOnSchema() throws IOException {
    Schema bytesArraySchema = Schema.arrayOf(Schema.of(Schema.Type.BYTES));
    Schema stringArraySchema = Schema.arrayOf(Schema.of(Schema.Type.STRING));
    Schema booleanBytesMapSchema = Schema.mapOf(Schema.of(Schema.Type.BOOLEAN), Schema.of(Schema.Type.BYTES));
    Schema nestedMapSchema = Schema.mapOf(bytesArraySchema, booleanBytesMapSchema);
    Schema record22Schema = Schema.recordOf("record22", Schema.Field.of("a", nestedMapSchema));
    Schema record22ArraySchema = Schema.arrayOf(record22Schema);
    Schema bytesDoubleMapSchema = Schema.mapOf(Schema.of(Schema.Type.BYTES), Schema.of(Schema.Type.DOUBLE));
    Schema record21Schema = Schema.recordOf("record21", Schema.Field.of("x", Schema.of(Schema.Type.STRING)), Schema.Field.of("y", stringArraySchema), Schema.Field.of("z", bytesDoubleMapSchema));
    Schema record21to22MapSchema = Schema.mapOf(record21Schema, record22ArraySchema);
    Schema nullableIntSchema = Schema.nullableOf(Schema.of(Schema.Type.INT));
    Schema tripeUnionSchema = Schema.unionOf(Schema.of(Schema.Type.INT), Schema.of(Schema.Type.LONG), Schema.of(Schema.Type.NULL));
    Schema complexSchema = Schema.recordOf("record1", Schema.Field.of("map1", record21to22MapSchema), Schema.Field.of("i", nullableIntSchema), Schema.Field.of("j", tripeUnionSchema));
    Schema anotherComplexSchema = Schema.arrayOf(Schema.of(Schema.Type.STRING));
    Schema superComplexSchema = Schema.unionOf(complexSchema, anotherComplexSchema, Schema.of(Schema.Type.NULL));
    String[] expectedTermsInIndex = { "record1", "record1:RECORD", "map1", "map1:MAP", "record21", "record21:RECORD", "x", "x:STRING", "y", "y:ARRAY", "z", "z:MAP", "record22", "record22:RECORD", "a", "a:MAP", "i", "i:INT", "j", "j:UNION" };
    MetadataEntity entity = MetadataEntity.ofDataset("myDs");
    Metadata meta = new Metadata(SYSTEM, props(MetadataConstants.ENTITY_NAME_KEY, "myDs", MetadataConstants.SCHEMA_KEY, superComplexSchema.toString()));
    MetadataRecord record = new MetadataRecord(entity, meta);
    MetadataStorage mds = getMetadataStorage();
    mds.apply(new Update(entity, meta), MutationOptions.DEFAULT);
    assertResults(mds, SearchRequest.of("myds").build(), record);
    assertResults(mds, SearchRequest.of("schema:*").build(), record);
    assertResults(mds, SearchRequest.of("properties:schema").build(), record);
    for (String expectedTerm : expectedTermsInIndex) {
        assertResults(mds, SearchRequest.of(expectedTerm).build(), record);
        assertResults(mds, SearchRequest.of("schema:" + expectedTerm).build(), record);
    }
    // clean up
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}

Example 3 with Update

use of io.cdap.cdap.spi.metadata.MetadataMutation.Update in project cdap by caskdata.

the class MetadataStorageTest method testCrossNamespaceDefaultSearch.

@Test
public void testCrossNamespaceDefaultSearch() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity ns1app = ofApp("ns1", "a");
    MetadataEntity ns2app = ofApp("ns2", "a");
    MetadataRecord app1Record = new MetadataRecord(ns1app, new Metadata(USER, props("k1", "v1", "k2", "v2")));
    MetadataRecord app2Record = new MetadataRecord(ns2app, new Metadata(USER, props("k1", "v1")));
    mds.batch(batch(new Update(ns1app, app1Record.getMetadata()), new Update(ns2app, app2Record.getMetadata())), MutationOptions.DEFAULT);
    assertResults(mds, SearchRequest.of("v1").build(), app1Record, app2Record);
    assertResults(mds, SearchRequest.of("v2").build(), app1Record);
    assertResults(mds, SearchRequest.of("*").build(), app1Record, app2Record);
    // clean up
    mds.batch(batch(new Drop(ns1app), new Drop(ns2app)), MutationOptions.DEFAULT);
}

Example 4 with Update

use of io.cdap.cdap.spi.metadata.MetadataMutation.Update in project cdap by caskdata.

the class MetadataStorageTest method testAsyncMutations.

@Test
public void testAsyncMutations() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity entity = ofDataset(DEFAULT_NAMESPACE, "entity");
    // get metadata for non-existing entity
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // drop metadata for non-existing entity succeeds
    MetadataChange change = mds.apply(new Drop(entity), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // remove metadata for non-existing entity succeeds
    mds.apply(new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "ut1"), new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), new ScopedNameOfKind(PROPERTY, USER, "up2"))), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // update metadata for non-existing entity creates it
    Metadata metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "a"), new ScopedName(USER, "b")), ImmutableMap.of(new ScopedName(SYSTEM, "p"), "v", new ScopedName(USER, "k"), "v1"));
    change = mds.apply(new Update(entity, metadata), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // test that update is idempotent
    change = mds.apply(new Update(entity, metadata), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // create metadata replaces existing metadata
    Metadata previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(USER, "ut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(USER, "up1"), "uv1", new ScopedName(USER, "up2"), "uv2"));
    MetadataMutation create = new Create(entity, metadata, Collections.emptyMap());
    change = mds.apply(create, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    // verify the metadata with variations of scope and kind
    verifyMetadataAsync(mds, entity, metadata);
    // verify the metadata with a select subset of tags and properties
    verifyMetadataSelectionAsync(mds, entity, metadata, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), new ScopedNameOfKind(PROPERTY, SYSTEM, "nosuch"), new ScopedNameOfKind(PROPERTY, USER, "up2"), new ScopedNameOfKind(PROPERTY, USER, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, USER, "ut1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch")));
    // verify that a non-matching set tags and properties returns empty metadata
    verifyMetadataSelectionAsync(mds, entity, metadata, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch")));
    // replace the system metadata with directives, user metadata should remain unchanged
    Metadata recreatedMetadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "nst0")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp0"), "sv0"));
    MetadataMutation recreate = new Create(entity, recreatedMetadata, ImmutableMap.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), MetadataDirective.KEEP, new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), MetadataDirective.KEEP));
    previousMetadata = metadata;
    // this is the new metadata according to directives
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst0"), new ScopedName(USER, "ut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp0"), "sv0", new ScopedName(USER, "up1"), "uv1", new ScopedName(USER, "up2"), "uv2"));
    change = mds.apply(recreate, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // replace the metadata with directives
    recreatedMetadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp2"), "sv2", new ScopedName(USER, "up3"), "uv3"));
    recreate = new Create(entity, recreatedMetadata, ImmutableMap.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), MetadataDirective.KEEP, new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), MetadataDirective.KEEP, new ScopedNameOfKind(PROPERTY, USER, "up2"), MetadataDirective.PRESERVE));
    previousMetadata = metadata;
    // this is the new metadata according to directives
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp2"), "sv2", new ScopedName(USER, "up2"), "uv2", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(recreate, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // update some tags and properties
    MetadataMutation update = new Update(entity, new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(USER, "aut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp2"), "nsv2", new ScopedName(USER, "up2"), "nuv2", new ScopedName(USER, "up3"), "uv3")));
    // verify new metadata after update
    previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "aut1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp2"), "nsv2", new ScopedName(USER, "up2"), "nuv2", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(update, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // test that update is idempotent
    change = mds.apply(update, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // remove some tags and properties
    MetadataMutation remove = new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nut1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch"), new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), new ScopedNameOfKind(PROPERTY, SYSTEM, "nsp2"), new ScopedNameOfKind(PROPERTY, USER, "up2")));
    previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "aut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(remove, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // test that remove is idemtpotent
    change = mds.apply(remove, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // drop all metadata for the entity
    change = mds.apply(new Drop(entity), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // drop is idempotent
    change = mds.apply(new Drop(entity), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
}

Example 5 with Update

use of io.cdap.cdap.spi.metadata.MetadataMutation.Update in project cdap by caskdata.

the class MetadataStorageTest method testSearchOnTTL.

@Test
public void testSearchOnTTL() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity ds = ofDataset("ns1", "ds");
    Metadata metaWithTTL = new Metadata(SYSTEM, props(TTL_KEY, "3600"));
    MetadataRecord dsRecord = new MetadataRecord(ds, metaWithTTL);
    mds.apply(new Update(ds, metaWithTTL), MutationOptions.DEFAULT);
    assertEmpty(mds, SearchRequest.of("ttl:3600").setScope(USER).build());
    assertResults(mds, SearchRequest.of("ttl:3600").build(), dsRecord);
    assertResults(mds, SearchRequest.of("ttl:3600").setScope(SYSTEM).build(), dsRecord);
    List<String> moreQueries = new ArrayList<>(ImmutableList.of("3600", "properties:ttl", "ttl:*", "TTL:3600"));
    moreQueries.addAll(getAdditionalTTLQueries());
    for (String query : moreQueries) {
        try {
            assertResults(mds, SearchRequest.of(query).build(), dsRecord);
        } catch (Throwable e) {
            throw new Exception("Search failed for query: " + query);
        }
    }
    mds.apply(new Drop(ds), MutationOptions.DEFAULT);
}