Examples with Drop io.cdap.cdap.spi.metadata.MetadataMutation.Drop used on opensource projects

Example 21 with Drop

use of io.cdap.cdap.spi.metadata.MetadataMutation.Drop in project cdap by caskdata.

the class MetadataStorageTest method testUpdateDropConflict.

/**
 * It's not trivial to test conflicts between updates and drops, because the outcome is not
 * deterministic. Here we start with an entity that has one tag "a". We issue a Drop and an
 * Update to add tag "b" concurrently. After both operations complete (possibly with retry
 * after conflict), only two cases are possible:
 *
 * 1. The delete completes first: Tag "a" is gone and tag "b" is the only metadata.
 *
 * 2. The update completes first: Tag "b" is added and deleted along with "a" right after,
 *    and the metadata is empty.
 *
 * Because the race between the two mutations does not always happen, we run this 10 times.
 */
@Test
@Category(SlowTests.class)
public void testUpdateDropConflict() throws IOException {
    MetadataEntity entity = MetadataEntity.ofDataset("myds");
    MetadataStorage mds = getMetadataStorage();
    int numTests = 10;
    IntStream.range(0, numTests).forEach(x -> {
        try {
            mds.apply(new Create(entity, new Metadata(USER, tags("a")), Collections.emptyMap()), MutationOptions.DEFAULT);
            ExecutorService executor = Executors.newFixedThreadPool(2);
            CompletionService<MetadataChange> completionService = new ExecutorCompletionService<>(executor);
            MetadataMutation update = new Update(entity, new Metadata(USER, tags("b")));
            MetadataMutation drop = new Drop(entity);
            completionService.submit(() -> mds.apply(update, MutationOptions.DEFAULT));
            completionService.submit(() -> mds.apply(drop, MutationOptions.DEFAULT));
            completionService.take();
            completionService.take();
            // each entity is either dropped then updated (and then it has tag "b" only)
            // or it first update and then dropped (and then it has empty metadata)
            Assert.assertTrue(ImmutableSet.of(Metadata.EMPTY, new Metadata(USER, tags("b"))).contains(mds.read(new Read(entity))));
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    });
    // clean up
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}

Example 22 with Drop

use of io.cdap.cdap.spi.metadata.MetadataMutation.Drop in project cdap by caskdata.

the class MetadataStorageTest method testBatch.

@Test
public void testBatch() throws IOException {
    MetadataEntity entity = MetadataEntity.ofDataset("a", "b");
    Map<ScopedNameOfKind, MetadataDirective> directives = ImmutableMap.of(new ScopedNameOfKind(PROPERTY, SYSTEM, CREATION_TIME_KEY), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, DESCRIPTION_KEY), MetadataDirective.KEEP);
    MetadataStorage mds = getMetadataStorage();
    Create create = new Create(entity, new Metadata(SYSTEM, tags("batch"), props(CREATION_TIME_KEY, "12345678", DESCRIPTION_KEY, "hello", "other", "value")), directives);
    MetadataChange change = mds.apply(create, MutationOptions.DEFAULT);
    Assert.assertEquals(Metadata.EMPTY, change.getBefore());
    Assert.assertEquals(create.getMetadata(), change.getAfter());
    List<MetadataMutation> mutations = ImmutableList.of(new Update(entity, new Metadata(USER, tags("tag1", "tag2"))), new Drop(entity), new Create(entity, new Metadata(SYSTEM, tags("batch"), props(CREATION_TIME_KEY, "23456789", "other", "different")), directives), new Update(entity, new Metadata(USER, tags("tag3"), props("key", "value"))), new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "other"), new ScopedNameOfKind(TAG, USER, "tag2"))), new Create(entity, new Metadata(SYSTEM, tags("realtime"), props(CREATION_TIME_KEY, "33456789", DESCRIPTION_KEY, "new description", "other", "yet other")), directives));
    // apply all mutations in sequence
    List<MetadataChange> changes = mutations.stream().map(mutation -> {
        try {
            return mds.apply(mutation, MutationOptions.DEFAULT);
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }).collect(Collectors.toList());
    // drop and recreate the entity
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
    change = mds.apply(create, MutationOptions.DEFAULT);
    Assert.assertEquals(Metadata.EMPTY, change.getBefore());
    Assert.assertEquals(create.getMetadata(), change.getAfter());
    // apply all mutations in batch
    List<MetadataChange> batchChanges = mds.batch(mutations, MutationOptions.DEFAULT);
    // make sure the same mutations were applied
    Assert.assertEquals(changes, batchChanges);
    // clean up
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}

Example 23 with Drop

use of io.cdap.cdap.spi.metadata.MetadataMutation.Drop in project cdap by caskdata.

the class MetadataStorageTest method testSearchPagination.

@Test
public void testSearchPagination() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    String ns = "ns";
    MetadataEntity app1 = ofApp(ns, "app");
    MetadataEntity app2 = ofApp(ns, "app2");
    MetadataEntity service = ofService(app1, "service");
    MetadataEntity worker = ofWorker(app2, "worker");
    MetadataEntity dataset = ofDataset(ns, "dataset");
    MetadataEntity hidden = ofDataset(ns, "_auditLog");
    MetadataRecord serviceRecord = new MetadataRecord(service, new Metadata(USER, tags("tag", "tag1")));
    MetadataRecord workerRecord = new MetadataRecord(worker, new Metadata(USER, tags("tag2", "tag3 tag4")));
    MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(USER, tags("tag5 tag6", "tag7 tag8")));
    MetadataRecord hiddenRecord = new MetadataRecord(hidden, new Metadata(USER, tags("tag9", "tag10", "tag11", "tag12", "tag13")));
    mds.batch(batch(new Update(service, serviceRecord.getMetadata()), new Update(worker, workerRecord.getMetadata()), new Update(dataset, datasetRecord.getMetadata()), new Update(hidden, hiddenRecord.getMetadata())), MutationOptions.DEFAULT);
    // assert that search returns all records and determine the order of relevance )it varies by implementation)
    SearchResponse response = mds.search(SearchRequest.of("tag*").addNamespace(ns).setShowHidden(true).setLimit(Integer.MAX_VALUE).build());
    List<MetadataRecord> results = response.getResults();
    List<MetadataRecord> noHidden = results.stream().filter(record -> !record.equals(hiddenRecord)).collect(Collectors.toList());
    Assert.assertEquals(ImmutableSet.of(serviceRecord, workerRecord, datasetRecord, hiddenRecord), ImmutableSet.copyOf(results));
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), noHidden.get(0), noHidden.get(1), noHidden.get(2));
    // hidden entity should now be returned since showHidden is true
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setShowHidden(true).build(), results.get(0), results.get(1), results.get(2), results.get(3));
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setLimit(2).setCursorRequested(true).build(), noHidden.get(0), noHidden.get(1));
    // skipping hidden entity should not affect the offset
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).setLimit(2).build(), noHidden.get(1), noHidden.get(2));
    // if showHidden is true, the hidden entity should affect the offset
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).setLimit(3).setShowHidden(true).build(), results.get(1), results.get(2), results.get(3));
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(2).setLimit(2).build(), noHidden.get(2));
    assertEmpty(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(4).setLimit(2).build());
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).build(), noHidden.get(1), noHidden.get(2));
    // clean up
    mds.batch(batch(new Drop(service), new Drop(worker), new Drop(dataset), new Drop(hidden)), MutationOptions.DEFAULT);
}

Example 24 with Drop

use of io.cdap.cdap.spi.metadata.MetadataMutation.Drop in project cdap by caskdata.

the class MetadataStorageTest method testCrossNamespaceCustomSearch.

@Test
public void testCrossNamespaceCustomSearch() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    String appName = "app";
    MetadataEntity ns1App = ofApp("ns1", appName);
    MetadataEntity ns2App = ofApp("ns2", appName);
    Metadata meta = new Metadata(SYSTEM, props(ENTITY_NAME_KEY, appName));
    MetadataRecord app1Record = new MetadataRecord(ns1App, meta);
    MetadataRecord app2Record = new MetadataRecord(ns2App, meta);
    mds.batch(batch(new Update(ns1App, meta), new Update(ns2App, meta)), MutationOptions.DEFAULT);
    assertInOrder(mds, SearchRequest.of("*").setSorting(new Sorting(ENTITY_NAME_KEY, Sorting.Order.ASC)).build(), app1Record, app2Record);
    // clean up
    mds.batch(batch(new Drop(ns1App), new Drop(ns2App)), MutationOptions.DEFAULT);
}

Example 25 with Drop

use of io.cdap.cdap.spi.metadata.MetadataMutation.Drop in project cdap by caskdata.

the class MetadataStorageTest method testSearchIncludesSystemEntities.

@Test
public void testSearchIncludesSystemEntities() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    String ns1 = "ns1";
    String ns2 = "ns2";
    MetadataEntity program = ofWorker(ofApp(ns1, "app1"), "wk1");
    // Use the same artifact in two different namespaces - system and ns2
    MetadataEntity artifact = ofArtifact(ns2, "artifact", "1.0");
    MetadataEntity sysArtifact = ofArtifact(SYSTEM_NAMESPACE, "artifact", "1.0");
    final String multiWordKey = "multiword";
    final String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    Metadata meta = new Metadata(SYSTEM, props(multiWordKey, multiWordValue));
    MetadataRecord programRecord = new MetadataRecord(program, meta);
    MetadataRecord artifactRecord = new MetadataRecord(artifact, meta);
    MetadataRecord sysArtifactRecord = new MetadataRecord(sysArtifact, meta);
    mds.batch(batch(new Update(program, meta), new Update(artifact, meta), new Update(sysArtifact, meta)), MutationOptions.DEFAULT);
    // perform the exact same multiword search in the 'ns1' namespace. It should return the system artifact along with
    // matched entities in the 'ns1' namespace
    assertResults(mds, SearchRequest.of("aV5").addNamespace(ns1).addSystemNamespace().build(), programRecord, sysArtifactRecord);
    // search only programs - should only return flow
    assertResults(mds, SearchRequest.of("aV5").addNamespace(ns1).addType(TYPE_PROGRAM).build(), programRecord);
    assertResults(mds, SearchRequest.of("multiword:aV5").addNamespace(ns1).addType(TYPE_PROGRAM).build(), programRecord);
    // search only artifacts - should only return system artifact
    assertResults(mds, SearchRequest.of("multiword:" + multiWordValue).addNamespace(ns1).addSystemNamespace().addType(TYPE_ARTIFACT).build(), sysArtifactRecord);
    // search all entities in namespace 'ns2' - should return the system artifact and the same artifact in ns2
    assertResults(mds, SearchRequest.of("multiword:aV4").addNamespace(ns2).addSystemNamespace().build(), artifactRecord, sysArtifactRecord);
    // search only programs in a namespace 'ns2'. Should return empty
    assertEmpty(mds, SearchRequest.of("aV*").addNamespace(ns2).addSystemNamespace().addType(TYPE_PROGRAM).build());
    // search all entities in non-existent namespace 'ns3'. Should return only the system artifact
    assertResults(mds, SearchRequest.of("av*").addNamespace("ns3").addSystemNamespace().build(), sysArtifactRecord);
    // search the system namespace for all entities. Should return only the system artifact
    assertResults(mds, SearchRequest.of("av*").addSystemNamespace().build(), sysArtifactRecord);
    // clean up
    mds.batch(batch(new Drop(program), new Drop(artifact), new Drop(sysArtifact)), MutationOptions.DEFAULT);
}