Examples with USER io.cdap.cdap.api.metadata.MetadataScope.USER used on opensource projects

Example 1 with USER

use of io.cdap.cdap.api.metadata.MetadataScope.USER in project cdap by caskdata.

the class MetadataStorageTest method testBatchConcurrency.

@Test
public void testBatchConcurrency() throws IOException {
    // T threads
    int numThreads = 10;
    // N entities
    int numEntities = 20;
    MetadataStorage mds = getMetadataStorage();
    ExecutorService executor = Executors.newFixedThreadPool(numThreads);
    CompletionService<List<MetadataChange>> completionService = new ExecutorCompletionService<>(executor);
    // Set up N entities with T tags (one to be removed per thread)
    Set<String> allRTags = IntStream.range(0, numThreads).mapToObj(t -> "r" + t).collect(Collectors.toSet());
    Map<Integer, MetadataEntity> entities = IntStream.range(0, numEntities).boxed().collect(Collectors.toMap(i -> i, i -> MetadataEntity.ofDataset("myds" + i)));
    mds.batch(entities.values().stream().map(entity -> new Update(entity, new Metadata(USER, allRTags))).collect(Collectors.toList()), MutationOptions.DEFAULT);
    // Generate a random but conflicting set of batches of mutations, one for each thread.
    // Construct the expected results for each entity along with the mutations
    // Also, because some threads will perform a Create, create a set of directives to preserve all other tags
    Map<Integer, Set<String>> expected = IntStream.range(0, numEntities).boxed().collect(Collectors.toMap(i -> i, i -> new HashSet<>(allRTags)));
    Map<Integer, List<MetadataMutation>> mutations = IntStream.range(0, numThreads).boxed().collect(Collectors.toMap(i -> i, i -> new ArrayList<>()));
    Map<ScopedNameOfKind, MetadataDirective> directives = new HashMap<>();
    Random rand = new Random(System.currentTimeMillis());
    IntStream.range(0, numThreads).forEach(t -> {
        directives.put(new ScopedNameOfKind(TAG, USER, "t" + t), MetadataDirective.KEEP);
        directives.put(new ScopedNameOfKind(TAG, USER, "r" + t), MetadataDirective.KEEP);
        directives.put(new ScopedNameOfKind(TAG, USER, "c" + t), MetadataDirective.KEEP);
        IntStream.range(0, numEntities).forEach(e -> {
            int random = rand.nextInt(100);
            if (random < 30) {
                expected.get(e).add("t" + t);
                mutations.get(t).add(new Update(entities.get(e), new Metadata(USER, tags("t" + t))));
            } else if (random < 60) {
                expected.get(e).add("c" + t);
                mutations.get(t).add(new Create(entities.get(e), new Metadata(USER, tags("c" + t)), directives));
            } else if (random < 90) {
                expected.get(e).remove("r" + t);
                mutations.get(t).add(new Remove(entities.get(e), Collections.singleton(new ScopedNameOfKind(TAG, USER, "r" + t))));
            }
        });
    });
    // submit all tasks and wait for their completion
    IntStream.range(0, numThreads).forEach(t -> completionService.submit(() -> mds.batch(mutations.get(t), MutationOptions.DEFAULT)));
    IntStream.range(0, numThreads).forEach(t -> {
        try {
            completionService.take();
        } catch (InterruptedException e) {
            throw Throwables.propagate(e);
        }
    });
    // validate that all "r" tags were removed and all "c" and "t" tags were added
    IntStream.range(0, numEntities).forEach(e -> {
        try {
            Assert.assertEquals("For entity " + entities.get(e), expected.get(e), mds.read(new Read(entities.get(e))).getTags(USER));
        } catch (Exception ex) {
            throw Throwables.propagate(ex);
        }
    });
    // clean up
    mds.batch(entities.values().stream().map(Drop::new).collect(Collectors.toList()), MutationOptions.DEFAULT);
}

Example 2 with USER

use of io.cdap.cdap.api.metadata.MetadataScope.USER in project cdap by caskdata.

the class MetadataStorageTest method testUpdateDropConflictInBatch.

/**
 * See {@link #testUpdateDropConflict()} for a description. The difference in this test is that
 * we issue batches of mutations over a collection of entities. The same assumptions apply,
 * however, for each entity.
 */
@Test
@Category(SlowTests.class)
public void testUpdateDropConflictInBatch() throws IOException {
    int numTests = 10;
    int numThreads = 2;
    int numEntities = 20;
    MetadataStorage mds = getMetadataStorage();
    ExecutorService executor = Executors.newFixedThreadPool(numThreads);
    CompletionService<List<MetadataChange>> completionService = new ExecutorCompletionService<>(executor);
    Map<Integer, MetadataEntity> entities = IntStream.range(0, numEntities).boxed().collect(Collectors.toMap(i -> i, i -> MetadataEntity.ofDataset("myds" + i)));
    List<MetadataMutation> creates = entities.values().stream().map(e -> new Create(e, new Metadata(USER, tags("a")), Collections.emptyMap())).collect(Collectors.toList());
    Random rand = new Random(System.currentTimeMillis());
    IntStream.range(0, numTests).forEach(x -> {
        try {
            mds.batch(creates, MutationOptions.DEFAULT);
            Map<Integer, List<MetadataMutation>> mutations = IntStream.range(0, numThreads).boxed().collect(Collectors.toMap(i -> i, i -> new ArrayList<>()));
            IntStream.range(0, numEntities).forEach(e -> {
                // ensure that at least one thread attempts to drop this entity
                int dropThread = rand.nextInt(numThreads);
                IntStream.range(0, numThreads).forEach(t -> {
                    if (t == dropThread || rand.nextInt(100) < 50) {
                        mutations.get(t).add(new Drop(entities.get(e)));
                    } else {
                        mutations.get(t).add(new Update(entities.get(e), new Metadata(USER, tags("b"))));
                    }
                });
            });
            IntStream.range(0, numThreads).forEach(t -> completionService.submit(() -> mds.batch(mutations.get(t), MutationOptions.DEFAULT)));
            IntStream.range(0, numThreads).forEach(t -> {
                try {
                    completionService.take();
                } catch (InterruptedException e) {
                    throw Throwables.propagate(e);
                }
            });
            IntStream.range(0, numEntities).forEach(e -> {
                try {
                    // each entity is either dropped then updated (and then it has tag "b" only)
                    // or it first update and then dropped (and then it has empty metadata)
                    Assert.assertTrue(ImmutableSet.of(Metadata.EMPTY, new Metadata(USER, tags("b"))).contains(mds.read(new Read(entities.get(e)))));
                } catch (Exception ex) {
                    throw Throwables.propagate(ex);
                }
            });
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    });
    mds.batch(entities.values().stream().map(Drop::new).collect(Collectors.toList()), MutationOptions.DEFAULT);
}

Example 3 with USER

use of io.cdap.cdap.api.metadata.MetadataScope.USER in project cdap by caskdata.

the class MetadataStorageTest method testBatch.

@Test
public void testBatch() throws IOException {
    MetadataEntity entity = MetadataEntity.ofDataset("a", "b");
    Map<ScopedNameOfKind, MetadataDirective> directives = ImmutableMap.of(new ScopedNameOfKind(PROPERTY, SYSTEM, CREATION_TIME_KEY), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, DESCRIPTION_KEY), MetadataDirective.KEEP);
    MetadataStorage mds = getMetadataStorage();
    Create create = new Create(entity, new Metadata(SYSTEM, tags("batch"), props(CREATION_TIME_KEY, "12345678", DESCRIPTION_KEY, "hello", "other", "value")), directives);
    MetadataChange change = mds.apply(create, MutationOptions.DEFAULT);
    Assert.assertEquals(Metadata.EMPTY, change.getBefore());
    Assert.assertEquals(create.getMetadata(), change.getAfter());
    List<MetadataMutation> mutations = ImmutableList.of(new Update(entity, new Metadata(USER, tags("tag1", "tag2"))), new Drop(entity), new Create(entity, new Metadata(SYSTEM, tags("batch"), props(CREATION_TIME_KEY, "23456789", "other", "different")), directives), new Update(entity, new Metadata(USER, tags("tag3"), props("key", "value"))), new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "other"), new ScopedNameOfKind(TAG, USER, "tag2"))), new Create(entity, new Metadata(SYSTEM, tags("realtime"), props(CREATION_TIME_KEY, "33456789", DESCRIPTION_KEY, "new description", "other", "yet other")), directives));
    // apply all mutations in sequence
    List<MetadataChange> changes = mutations.stream().map(mutation -> {
        try {
            return mds.apply(mutation, MutationOptions.DEFAULT);
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }).collect(Collectors.toList());
    // drop and recreate the entity
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
    change = mds.apply(create, MutationOptions.DEFAULT);
    Assert.assertEquals(Metadata.EMPTY, change.getBefore());
    Assert.assertEquals(create.getMetadata(), change.getAfter());
    // apply all mutations in batch
    List<MetadataChange> batchChanges = mds.batch(mutations, MutationOptions.DEFAULT);
    // make sure the same mutations were applied
    Assert.assertEquals(changes, batchChanges);
    // clean up
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}

Example 4 with USER

use of io.cdap.cdap.api.metadata.MetadataScope.USER in project cdap by caskdata.

the class MetadataStorageTest method testSearchPagination.

@Test
public void testSearchPagination() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    String ns = "ns";
    MetadataEntity app1 = ofApp(ns, "app");
    MetadataEntity app2 = ofApp(ns, "app2");
    MetadataEntity service = ofService(app1, "service");
    MetadataEntity worker = ofWorker(app2, "worker");
    MetadataEntity dataset = ofDataset(ns, "dataset");
    MetadataEntity hidden = ofDataset(ns, "_auditLog");
    MetadataRecord serviceRecord = new MetadataRecord(service, new Metadata(USER, tags("tag", "tag1")));
    MetadataRecord workerRecord = new MetadataRecord(worker, new Metadata(USER, tags("tag2", "tag3 tag4")));
    MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(USER, tags("tag5 tag6", "tag7 tag8")));
    MetadataRecord hiddenRecord = new MetadataRecord(hidden, new Metadata(USER, tags("tag9", "tag10", "tag11", "tag12", "tag13")));
    mds.batch(batch(new Update(service, serviceRecord.getMetadata()), new Update(worker, workerRecord.getMetadata()), new Update(dataset, datasetRecord.getMetadata()), new Update(hidden, hiddenRecord.getMetadata())), MutationOptions.DEFAULT);
    // assert that search returns all records and determine the order of relevance )it varies by implementation)
    SearchResponse response = mds.search(SearchRequest.of("tag*").addNamespace(ns).setShowHidden(true).setLimit(Integer.MAX_VALUE).build());
    List<MetadataRecord> results = response.getResults();
    List<MetadataRecord> noHidden = results.stream().filter(record -> !record.equals(hiddenRecord)).collect(Collectors.toList());
    Assert.assertEquals(ImmutableSet.of(serviceRecord, workerRecord, datasetRecord, hiddenRecord), ImmutableSet.copyOf(results));
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), noHidden.get(0), noHidden.get(1), noHidden.get(2));
    // hidden entity should now be returned since showHidden is true
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setShowHidden(true).build(), results.get(0), results.get(1), results.get(2), results.get(3));
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setLimit(2).setCursorRequested(true).build(), noHidden.get(0), noHidden.get(1));
    // skipping hidden entity should not affect the offset
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).setLimit(2).build(), noHidden.get(1), noHidden.get(2));
    // if showHidden is true, the hidden entity should affect the offset
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).setLimit(3).setShowHidden(true).build(), results.get(1), results.get(2), results.get(3));
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(2).setLimit(2).build(), noHidden.get(2));
    assertEmpty(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(4).setLimit(2).build());
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).build(), noHidden.get(1), noHidden.get(2));
    // clean up
    mds.batch(batch(new Drop(service), new Drop(worker), new Drop(dataset), new Drop(hidden)), MutationOptions.DEFAULT);
}

Example 5 with USER

use of io.cdap.cdap.api.metadata.MetadataScope.USER in project cdap by caskdata.

the class DatasetMetadataStorageTest method testSearchWeight.

// this tests is not in MetadataStorageTest,
// because it tests result scoring and sorting specific to the dataset-based implementation
@Test
public void testSearchWeight() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    String ns = "ns1";
    NamespaceId nsId = new NamespaceId(ns);
    MetadataEntity service1 = nsId.app("app1").service("service1").toMetadataEntity();
    MetadataEntity dataset1 = nsId.dataset("ds1").toMetadataEntity();
    MetadataEntity dataset2 = nsId.dataset("ds2").toMetadataEntity();
    // Add metadata
    String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    Map<String, String> userProps = ImmutableMap.of("key1", "value1", "key2", "value2", "multiword", multiWordValue);
    Map<String, String> systemProps = ImmutableMap.of("sysKey1", "sysValue1");
    Set<String> userTags = ImmutableSet.of("tag1", "tag2");
    Set<String> temporaryUserTags = ImmutableSet.of("tag3", "tag4");
    Map<String, String> dataset1UserProps = ImmutableMap.of("sKey1", "sValuee1 sValuee2");
    Map<String, String> dataset2UserProps = ImmutableMap.of("sKey1", "sValue1 sValue2", "Key1", "Value1");
    Set<String> sysTags = ImmutableSet.of("sysTag1");
    MetadataRecord service1Record = new MetadataRecord(service1, union(new Metadata(USER, userTags, userProps), new Metadata(SYSTEM, sysTags, systemProps)));
    mds.apply(new Update(service1Record.getEntity(), service1Record.getMetadata()), MutationOptions.DEFAULT);
    // dd and then remove some metadata for dataset2
    mds.apply(new Update(dataset2, new Metadata(USER, temporaryUserTags, userProps)), MutationOptions.DEFAULT);
    mds.apply(new Remove(dataset2, temporaryUserTags.stream().map(tag -> new ScopedNameOfKind(TAG, USER, tag)).collect(Collectors.toSet())), MutationOptions.DEFAULT);
    mds.apply(new Remove(dataset2, userProps.keySet().stream().map(tag -> new ScopedNameOfKind(PROPERTY, USER, tag)).collect(Collectors.toSet())), MutationOptions.DEFAULT);
    MetadataRecord dataset1Record = new MetadataRecord(dataset1, new Metadata(USER, tags(), dataset1UserProps));
    MetadataRecord dataset2Record = new MetadataRecord(dataset2, new Metadata(USER, tags(), dataset2UserProps));
    mds.batch(ImmutableList.of(new Update(dataset1Record.getEntity(), dataset1Record.getMetadata()), new Update(dataset2Record.getEntity(), dataset2Record.getMetadata())), MutationOptions.DEFAULT);
    // Test score and metadata match
    assertInOrder(mds, SearchRequest.of("value1 multiword:av2").addNamespace(ns).build(), service1Record, dataset2Record);
    assertInOrder(mds, SearchRequest.of("value1 sValue*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), dataset2Record, dataset1Record, service1Record);
    assertResults(mds, SearchRequest.of("*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), dataset2Record, dataset1Record, service1Record);
    // clean up
    mds.batch(ImmutableList.of(new Drop(service1), new Drop(dataset1), new Drop(dataset2)), MutationOptions.DEFAULT);
}