Examples with MetadataEntity io.cdap.cdap.api.metadata.MetadataEntity used on opensource projects

Example 56 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testSearchPagination.

@Test
public void testSearchPagination() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    String ns = "ns";
    MetadataEntity app1 = ofApp(ns, "app");
    MetadataEntity app2 = ofApp(ns, "app2");
    MetadataEntity service = ofService(app1, "service");
    MetadataEntity worker = ofWorker(app2, "worker");
    MetadataEntity dataset = ofDataset(ns, "dataset");
    MetadataEntity hidden = ofDataset(ns, "_auditLog");
    MetadataRecord serviceRecord = new MetadataRecord(service, new Metadata(USER, tags("tag", "tag1")));
    MetadataRecord workerRecord = new MetadataRecord(worker, new Metadata(USER, tags("tag2", "tag3 tag4")));
    MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(USER, tags("tag5 tag6", "tag7 tag8")));
    MetadataRecord hiddenRecord = new MetadataRecord(hidden, new Metadata(USER, tags("tag9", "tag10", "tag11", "tag12", "tag13")));
    mds.batch(batch(new Update(service, serviceRecord.getMetadata()), new Update(worker, workerRecord.getMetadata()), new Update(dataset, datasetRecord.getMetadata()), new Update(hidden, hiddenRecord.getMetadata())), MutationOptions.DEFAULT);
    // assert that search returns all records and determine the order of relevance )it varies by implementation)
    SearchResponse response = mds.search(SearchRequest.of("tag*").addNamespace(ns).setShowHidden(true).setLimit(Integer.MAX_VALUE).build());
    List<MetadataRecord> results = response.getResults();
    List<MetadataRecord> noHidden = results.stream().filter(record -> !record.equals(hiddenRecord)).collect(Collectors.toList());
    Assert.assertEquals(ImmutableSet.of(serviceRecord, workerRecord, datasetRecord, hiddenRecord), ImmutableSet.copyOf(results));
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), noHidden.get(0), noHidden.get(1), noHidden.get(2));
    // hidden entity should now be returned since showHidden is true
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setShowHidden(true).build(), results.get(0), results.get(1), results.get(2), results.get(3));
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setLimit(2).setCursorRequested(true).build(), noHidden.get(0), noHidden.get(1));
    // skipping hidden entity should not affect the offset
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).setLimit(2).build(), noHidden.get(1), noHidden.get(2));
    // if showHidden is true, the hidden entity should affect the offset
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).setLimit(3).setShowHidden(true).build(), results.get(1), results.get(2), results.get(3));
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(2).setLimit(2).build(), noHidden.get(2));
    assertEmpty(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(4).setLimit(2).build());
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).build(), noHidden.get(1), noHidden.get(2));
    // clean up
    mds.batch(batch(new Drop(service), new Drop(worker), new Drop(dataset), new Drop(hidden)), MutationOptions.DEFAULT);
}

Example 57 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testSearchIncludesSystemEntities.

@Test
public void testSearchIncludesSystemEntities() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    String ns1 = "ns1";
    String ns2 = "ns2";
    MetadataEntity program = ofWorker(ofApp(ns1, "app1"), "wk1");
    // Use the same artifact in two different namespaces - system and ns2
    MetadataEntity artifact = ofArtifact(ns2, "artifact", "1.0");
    MetadataEntity sysArtifact = ofArtifact(SYSTEM_NAMESPACE, "artifact", "1.0");
    final String multiWordKey = "multiword";
    final String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    Metadata meta = new Metadata(SYSTEM, props(multiWordKey, multiWordValue));
    MetadataRecord programRecord = new MetadataRecord(program, meta);
    MetadataRecord artifactRecord = new MetadataRecord(artifact, meta);
    MetadataRecord sysArtifactRecord = new MetadataRecord(sysArtifact, meta);
    mds.batch(batch(new Update(program, meta), new Update(artifact, meta), new Update(sysArtifact, meta)), MutationOptions.DEFAULT);
    // perform the exact same multiword search in the 'ns1' namespace. It should return the system artifact along with
    // matched entities in the 'ns1' namespace
    assertResults(mds, SearchRequest.of("aV5").addNamespace(ns1).addSystemNamespace().build(), programRecord, sysArtifactRecord);
    // search only programs - should only return flow
    assertResults(mds, SearchRequest.of("aV5").addNamespace(ns1).addType(TYPE_PROGRAM).build(), programRecord);
    assertResults(mds, SearchRequest.of("multiword:aV5").addNamespace(ns1).addType(TYPE_PROGRAM).build(), programRecord);
    // search only artifacts - should only return system artifact
    assertResults(mds, SearchRequest.of("multiword:" + multiWordValue).addNamespace(ns1).addSystemNamespace().addType(TYPE_ARTIFACT).build(), sysArtifactRecord);
    // search all entities in namespace 'ns2' - should return the system artifact and the same artifact in ns2
    assertResults(mds, SearchRequest.of("multiword:aV4").addNamespace(ns2).addSystemNamespace().build(), artifactRecord, sysArtifactRecord);
    // search only programs in a namespace 'ns2'. Should return empty
    assertEmpty(mds, SearchRequest.of("aV*").addNamespace(ns2).addSystemNamespace().addType(TYPE_PROGRAM).build());
    // search all entities in non-existent namespace 'ns3'. Should return only the system artifact
    assertResults(mds, SearchRequest.of("av*").addNamespace("ns3").addSystemNamespace().build(), sysArtifactRecord);
    // search the system namespace for all entities. Should return only the system artifact
    assertResults(mds, SearchRequest.of("av*").addSystemNamespace().build(), sysArtifactRecord);
    // clean up
    mds.batch(batch(new Drop(program), new Drop(artifact), new Drop(sysArtifact)), MutationOptions.DEFAULT);
}

Example 58 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testCrossNamespaceCustomSearch.

@Test
public void testCrossNamespaceCustomSearch() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    String appName = "app";
    MetadataEntity ns1App = ofApp("ns1", appName);
    MetadataEntity ns2App = ofApp("ns2", appName);
    Metadata meta = new Metadata(SYSTEM, props(ENTITY_NAME_KEY, appName));
    MetadataRecord app1Record = new MetadataRecord(ns1App, meta);
    MetadataRecord app2Record = new MetadataRecord(ns2App, meta);
    mds.batch(batch(new Update(ns1App, meta), new Update(ns2App, meta)), MutationOptions.DEFAULT);
    assertInOrder(mds, SearchRequest.of("*").setSorting(new Sorting(ENTITY_NAME_KEY, Sorting.Order.ASC)).build(), app1Record, app2Record);
    // clean up
    mds.batch(batch(new Drop(ns1App), new Drop(ns2App)), MutationOptions.DEFAULT);
}

Example 59 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testAsyncMutations.

@Test
public void testAsyncMutations() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity entity = ofDataset(DEFAULT_NAMESPACE, "entity");
    // get metadata for non-existing entity
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // drop metadata for non-existing entity succeeds
    MetadataChange change = mds.apply(new Drop(entity), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // remove metadata for non-existing entity succeeds
    mds.apply(new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "ut1"), new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), new ScopedNameOfKind(PROPERTY, USER, "up2"))), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // update metadata for non-existing entity creates it
    Metadata metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "a"), new ScopedName(USER, "b")), ImmutableMap.of(new ScopedName(SYSTEM, "p"), "v", new ScopedName(USER, "k"), "v1"));
    change = mds.apply(new Update(entity, metadata), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // test that update is idempotent
    change = mds.apply(new Update(entity, metadata), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // create metadata replaces existing metadata
    Metadata previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(USER, "ut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(USER, "up1"), "uv1", new ScopedName(USER, "up2"), "uv2"));
    MetadataMutation create = new Create(entity, metadata, Collections.emptyMap());
    change = mds.apply(create, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    // verify the metadata with variations of scope and kind
    verifyMetadataAsync(mds, entity, metadata);
    // verify the metadata with a select subset of tags and properties
    verifyMetadataSelectionAsync(mds, entity, metadata, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), new ScopedNameOfKind(PROPERTY, SYSTEM, "nosuch"), new ScopedNameOfKind(PROPERTY, USER, "up2"), new ScopedNameOfKind(PROPERTY, USER, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, USER, "ut1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch")));
    // verify that a non-matching set tags and properties returns empty metadata
    verifyMetadataSelectionAsync(mds, entity, metadata, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch")));
    // replace the system metadata with directives, user metadata should remain unchanged
    Metadata recreatedMetadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "nst0")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp0"), "sv0"));
    MetadataMutation recreate = new Create(entity, recreatedMetadata, ImmutableMap.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), MetadataDirective.KEEP, new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), MetadataDirective.KEEP));
    previousMetadata = metadata;
    // this is the new metadata according to directives
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst0"), new ScopedName(USER, "ut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp0"), "sv0", new ScopedName(USER, "up1"), "uv1", new ScopedName(USER, "up2"), "uv2"));
    change = mds.apply(recreate, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // replace the metadata with directives
    recreatedMetadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp2"), "sv2", new ScopedName(USER, "up3"), "uv3"));
    recreate = new Create(entity, recreatedMetadata, ImmutableMap.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), MetadataDirective.KEEP, new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), MetadataDirective.KEEP, new ScopedNameOfKind(PROPERTY, USER, "up2"), MetadataDirective.PRESERVE));
    previousMetadata = metadata;
    // this is the new metadata according to directives
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp2"), "sv2", new ScopedName(USER, "up2"), "uv2", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(recreate, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // update some tags and properties
    MetadataMutation update = new Update(entity, new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(USER, "aut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp2"), "nsv2", new ScopedName(USER, "up2"), "nuv2", new ScopedName(USER, "up3"), "uv3")));
    // verify new metadata after update
    previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "aut1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp2"), "nsv2", new ScopedName(USER, "up2"), "nuv2", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(update, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // test that update is idempotent
    change = mds.apply(update, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // remove some tags and properties
    MetadataMutation remove = new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nut1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch"), new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), new ScopedNameOfKind(PROPERTY, SYSTEM, "nsp2"), new ScopedNameOfKind(PROPERTY, USER, "up2")));
    previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "aut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(remove, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // test that remove is idemtpotent
    change = mds.apply(remove, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // drop all metadata for the entity
    change = mds.apply(new Drop(entity), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // drop is idempotent
    change = mds.apply(new Drop(entity), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
}

Example 60 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testConcurrency.

@Test
public void testConcurrency() throws IOException {
    int numThreads = 10;
    ExecutorService executor = Executors.newFixedThreadPool(numThreads);
    CompletionService<MetadataChange> completionService = new ExecutorCompletionService<>(executor);
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity entity = MetadataEntity.ofDataset("myds");
    // add "r<i>" tags to be removed by the individual threads
    mds.apply(new Update(entity, new Metadata(USER, IntStream.range(0, numThreads).mapToObj(i -> "r" + i).collect(Collectors.toSet()))), MutationOptions.DEFAULT);
    // create normally replaces. Add directives to preserve all tags. Also create the expected tags
    Set<String> expectedTags = new HashSet<>();
    Map<ScopedNameOfKind, MetadataDirective> directives = new HashMap<>();
    IntStream.range(0, numThreads).forEach(i -> {
        directives.put(new ScopedNameOfKind(TAG, USER, "t" + i), MetadataDirective.KEEP);
        directives.put(new ScopedNameOfKind(TAG, USER, "r" + i), MetadataDirective.KEEP);
        directives.put(new ScopedNameOfKind(TAG, USER, "c" + i), MetadataDirective.KEEP);
        // create threads will add c<i> and update threads will add t<i>; all r<i> will be removed
        expectedTags.add("t" + i);
        expectedTags.add("c" + i);
    });
    try {
        // create conflicting threads that perform create, update, and remove on the same entity
        IntStream.range(0, numThreads).forEach(i -> {
            completionService.submit(() -> mds.apply(new Create(entity, new Metadata(USER, tags("c" + i)), directives), MutationOptions.DEFAULT));
            completionService.submit(() -> mds.apply(new Update(entity, new Metadata(USER, tags("t" + i))), MutationOptions.DEFAULT));
            completionService.submit(() -> mds.apply(new Remove(entity, Collections.singleton(new ScopedNameOfKind(TAG, USER, "r" + i))), MutationOptions.DEFAULT));
        });
        IntStream.range(0, numThreads * 3).forEach(i -> {
            try {
                completionService.take();
            } catch (InterruptedException e) {
                throw Throwables.propagate(e);
            }
        });
        // validate that all "r" tags were removed and all "c" and "t" tags were added
        Assert.assertEquals(expectedTags, mds.read(new Read(entity)).getTags(USER));
    } finally {
        // clean up
        mds.apply(new Drop(entity), MutationOptions.DEFAULT);
    }
}