Examples with MetadataEntity io.cdap.cdap.api.metadata.MetadataEntity used on opensource projects

Example 61 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testSearchOnTypes.

@Test
public void testSearchOnTypes() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity myDs = ofDataset(DEFAULT_NAMESPACE, "myDs");
    MetadataEntity myField1 = MetadataEntity.builder(myDs).appendAsType("field", "myField1").build();
    MetadataEntity myField2 = MetadataEntity.builder(myDs).appendAsType("field", "myField2").build();
    MetadataRecord record1 = new MetadataRecord(myField1, new Metadata(USER, props("testKey1", "testValue1")));
    MetadataRecord record2 = new MetadataRecord(myField2, new Metadata(USER, props("testKey2", "testValue2")));
    mds.batch(batch(new Update(myField1, record1.getMetadata()), new Update(myField2, record2.getMetadata())), MutationOptions.DEFAULT);
    // Search for it based on value
    assertResults(mds, SearchRequest.of("field:myField1").build(), record1);
    // should return both fields
    assertResults(mds, SearchRequest.of("field:myFie*").build(), record1, record2);
    assertResults(mds, SearchRequest.of("field*").build(), record1, record2);
    // searching an invalid type should return nothing
    assertEmpty(mds, SearchRequest.of("x*").addType("invalid").build());
    // clean up
    mds.batch(batch(new Drop(myField1), new Drop(myField2)), MutationOptions.DEFAULT);
}

Example 62 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testMutations.

@Test
public void testMutations() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity entity = ofDataset(DEFAULT_NAMESPACE, "entity");
    // get metadata for non-existing entity
    verifyMetadata(mds, entity, Metadata.EMPTY);
    // drop metadata for non-existing entity succeeds
    MetadataChange change = mds.apply(new Drop(entity), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadata(mds, entity, Metadata.EMPTY);
    // remove metadata for non-existing entity succeeds
    mds.apply(new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "ut1"), new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), new ScopedNameOfKind(PROPERTY, USER, "up2"))), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadata(mds, entity, Metadata.EMPTY);
    // update metadata for non-existing entity creates it
    Metadata metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "a"), new ScopedName(USER, "b")), ImmutableMap.of(new ScopedName(SYSTEM, "p"), "v", new ScopedName(USER, "k"), "v1"));
    change = mds.apply(new Update(entity, metadata), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, metadata), change);
    verifyMetadata(mds, entity, metadata);
    // test that update is idempotent
    change = mds.apply(new Update(entity, metadata), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    // create metadata replaces existing metadata
    Metadata previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(USER, "ut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(USER, "up1"), "uv1", new ScopedName(USER, "up2"), "uv2"));
    MetadataMutation create = new Create(entity, metadata, Collections.emptyMap());
    change = mds.apply(create, MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    // verify the metadata with variations of scope and kind
    verifyMetadata(mds, entity, metadata);
    // verify the metadata with a select subset of tags and properties
    verifyMetadataSelection(mds, entity, metadata, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), new ScopedNameOfKind(PROPERTY, SYSTEM, "nosuch"), new ScopedNameOfKind(PROPERTY, USER, "up2"), new ScopedNameOfKind(PROPERTY, USER, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, USER, "ut1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch")));
    // verify that a non-matching set tags and properties returns empty metadata
    verifyMetadataSelection(mds, entity, metadata, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch")));
    // replace the system metadata with directives, user metadata should remain unchanged
    Metadata recreatedMetadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "nst0")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp0"), "sv0"));
    MetadataMutation recreate = new Create(entity, recreatedMetadata, ImmutableMap.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), MetadataDirective.KEEP, new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), MetadataDirective.KEEP));
    previousMetadata = metadata;
    // this is the new metadata according to directives
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst0"), new ScopedName(USER, "ut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp0"), "sv0", new ScopedName(USER, "up1"), "uv1", new ScopedName(USER, "up2"), "uv2"));
    change = mds.apply(recreate, MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    // replace the metadata with directives
    recreatedMetadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp2"), "sv2", new ScopedName(USER, "up3"), "uv3"));
    recreate = new Create(entity, recreatedMetadata, ImmutableMap.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), MetadataDirective.KEEP, new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), MetadataDirective.KEEP, new ScopedNameOfKind(PROPERTY, USER, "up2"), MetadataDirective.PRESERVE));
    previousMetadata = metadata;
    // this is the new metadata according to directives
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp2"), "sv2", new ScopedName(USER, "up2"), "uv2", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(recreate, MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    // update some tags and properties
    MetadataMutation update = new Update(entity, new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(USER, "aut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp2"), "nsv2", new ScopedName(USER, "up2"), "nuv2", new ScopedName(USER, "up3"), "uv3")));
    // verify new metadata after update
    previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "aut1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp2"), "nsv2", new ScopedName(USER, "up2"), "nuv2", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(update, MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    // test that update is idempotent
    change = mds.apply(update, MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    // remove some tags and properties
    MetadataMutation remove = new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nut1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch"), new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), new ScopedNameOfKind(PROPERTY, SYSTEM, "nsp2"), new ScopedNameOfKind(PROPERTY, USER, "up2")));
    previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "aut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(remove, MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    // test that remove is idemtpotent
    change = mds.apply(remove, MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    // drop all metadata for the entity
    change = mds.apply(new Drop(entity), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, metadata, Metadata.EMPTY), change);
    verifyMetadata(mds, entity, Metadata.EMPTY);
    // drop is idempotent
    change = mds.apply(new Drop(entity), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadata(mds, entity, Metadata.EMPTY);
}

Example 63 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testUpdateDropConflict.

/**
 * It's not trivial to test conflicts between updates and drops, because the outcome is not
 * deterministic. Here we start with an entity that has one tag "a". We issue a Drop and an
 * Update to add tag "b" concurrently. After both operations complete (possibly with retry
 * after conflict), only two cases are possible:
 *
 * 1. The delete completes first: Tag "a" is gone and tag "b" is the only metadata.
 *
 * 2. The update completes first: Tag "b" is added and deleted along with "a" right after,
 *    and the metadata is empty.
 *
 * Because the race between the two mutations does not always happen, we run this 10 times.
 */
@Test
@Category(SlowTests.class)
public void testUpdateDropConflict() throws IOException {
    MetadataEntity entity = MetadataEntity.ofDataset("myds");
    MetadataStorage mds = getMetadataStorage();
    int numTests = 10;
    IntStream.range(0, numTests).forEach(x -> {
        try {
            mds.apply(new Create(entity, new Metadata(USER, tags("a")), Collections.emptyMap()), MutationOptions.DEFAULT);
            ExecutorService executor = Executors.newFixedThreadPool(2);
            CompletionService<MetadataChange> completionService = new ExecutorCompletionService<>(executor);
            MetadataMutation update = new Update(entity, new Metadata(USER, tags("b")));
            MetadataMutation drop = new Drop(entity);
            completionService.submit(() -> mds.apply(update, MutationOptions.DEFAULT));
            completionService.submit(() -> mds.apply(drop, MutationOptions.DEFAULT));
            completionService.take();
            completionService.take();
            // each entity is either dropped then updated (and then it has tag "b" only)
            // or it first update and then dropped (and then it has empty metadata)
            Assert.assertTrue(ImmutableSet.of(Metadata.EMPTY, new Metadata(USER, tags("b"))).contains(mds.read(new Read(entity))));
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    });
    // clean up
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}

Example 64 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testSearchOnTagsUpdate.

@Test
public void testSearchOnTagsUpdate() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity entity = ofWorkflow(ofApp(DEFAULT_NAMESPACE, "appX"), "wtf");
    Metadata meta = new Metadata(SYSTEM, tags("tag1", "tag2"));
    mds.apply(new Update(entity, meta), MutationOptions.DEFAULT);
    Assert.assertEquals(meta.getTags(SYSTEM), mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM));
    assertResults(mds, SearchRequest.of("tag1").build(), new MetadataRecord(entity, meta));
    // add an more tags
    mds.apply(new Update(entity, new Metadata(SYSTEM, tags("tag3", "tag4"))), MutationOptions.DEFAULT);
    Set<String> newTags = tags("tag1", "tag2", "tag3", "tag4");
    Metadata newMeta = new Metadata(SYSTEM, newTags);
    Assert.assertEquals(newTags, mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM));
    for (String expectedTag : newTags) {
        assertResults(mds, SearchRequest.of(expectedTag).build(), new MetadataRecord(entity, newMeta));
    }
    // add an empty set of tags. This should have no effect on retrieval or search of tags
    mds.apply(new Update(entity, new Metadata(SYSTEM, tags())), MutationOptions.DEFAULT);
    Assert.assertEquals(newTags, mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM));
    for (String expectedTag : newTags) {
        assertResults(mds, SearchRequest.of(expectedTag).build(), new MetadataRecord(entity, newMeta));
    }
    // clean up
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}

Example 65 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testCrossNamespaceSearch.

@Test
public void testCrossNamespaceSearch() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    String ns1 = "ns1";
    String ns2 = "ns2";
    MetadataEntity ns1app1 = ofApp(ns1, "a1");
    MetadataEntity ns1app2 = ofApp(ns1, "a2");
    MetadataEntity ns1app3 = ofApp(ns1, "a3");
    MetadataEntity ns2app1 = ofApp(ns2, "a1");
    MetadataEntity ns2app2 = ofApp(ns2, "a2");
    MetadataRecord record11 = new MetadataRecord(ns1app1, new Metadata(USER, tags("v1"), props("k1", "v1"))), record12 = new MetadataRecord(ns1app2, new Metadata(USER, props("k1", "v1", "k2", "v2"))), record13 = new MetadataRecord(ns1app3, new Metadata(USER, props("k1", "v1", "k3", "v3"))), record21 = new MetadataRecord(ns2app1, new Metadata(USER, props("k1", "v1", "k2", "v2"))), record22 = new MetadataRecord(ns2app2, new Metadata(USER, tags("v2", "v3"), props("k1", "v1")));
    MetadataRecord[] records = { record11, record12, record13, record21, record22 };
    // apply all metadata in batch
    mds.batch(Arrays.stream(records).map(record -> new Update(record.getEntity(), record.getMetadata())).collect(Collectors.toList()), MutationOptions.DEFAULT);
    // everything should match 'v1'
    assertResults(mds, SearchRequest.of("v1").setLimit(10).build(), record11, record12, record13, record21, record22);
    // ns1app2, ns2app1, and ns2app2 should match 'v2'
    assertResults(mds, SearchRequest.of("v2").setLimit(10).build(), record12, record21, record22);
    // ns1app3 and ns2app2 should match 'v3'
    assertResults(mds, SearchRequest.of("v3").setLimit(10).build(), record13, record22);
    // clean up
    mds.batch(batch(new Drop(ns1app1), new Drop(ns1app2), new Drop(ns1app3), new Drop(ns2app1), new Drop(ns2app2)), MutationOptions.DEFAULT);
}