Examples with Update io.cdap.cdap.spi.metadata.MetadataMutation.Update used on opensource projects

Example 26 with Update

use of io.cdap.cdap.spi.metadata.MetadataMutation.Update in project cdap by cdapio.

the class MetadataStorageTest method testSearchOnValue.

@Test
public void testSearchOnValue() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity program = ofWorker(ofApp("ns1", "app1"), "wk1");
    MetadataEntity dataset = ofDataset("ns1", "ds2");
    // Add some metadata
    final String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    MetadataRecord programRecord = new MetadataRecord(program, new Metadata(USER, props("key1", "value1", "key2", "value2", "multiword", multiWordValue)));
    mds.apply(new Update(program, programRecord.getMetadata()), MutationOptions.DEFAULT);
    // Search for it based on value
    assertResults(mds, SearchRequest.of("value1").build(), programRecord);
    assertResults(mds, SearchRequest.of("  aV1   ").addType(TYPE_PROGRAM).build(), programRecord);
    assertEmpty(mds, SearchRequest.of("  aV1   ").addType(TYPE_ARTIFACT).build());
    // Search for it based split patterns to make sure nothing is matched
    assertEmpty(mds, SearchRequest.of("-").build());
    assertEmpty(mds, SearchRequest.of(",").build());
    assertEmpty(mds, SearchRequest.of("_").build());
    assertEmpty(mds, SearchRequest.of(", ,").build());
    assertEmpty(mds, SearchRequest.of(", - ,").build());
    // Search for it based on a word in value
    assertResults(mds, SearchRequest.of("av5").addType(TYPE_PROGRAM).build(), programRecord);
    // Case insensitive
    assertResults(mds, SearchRequest.of("ValUe1").addType(TYPE_PROGRAM).build(), programRecord);
    // add a property for the program
    mds.apply(new Update(program, new Metadata(SYSTEM, props("key3", "value1"))), MutationOptions.DEFAULT);
    programRecord = new MetadataRecord(program, new Metadata(ImmutableSet.of(), ImmutableMap.of(new ScopedName(USER, "key1"), "value1", new ScopedName(USER, "key2"), "value2", new ScopedName(USER, "multiword"), multiWordValue, new ScopedName(SYSTEM, "key3"), "value1")));
    // search by value
    assertResults(mds, SearchRequest.of("value1").addType(TYPE_PROGRAM).build(), programRecord);
    // add a property for the dataset
    MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(USER, props("key21", "value21")));
    mds.apply(new Update(dataset, datasetRecord.getMetadata()), MutationOptions.DEFAULT);
    // Search based on value prefix
    assertResults(mds, SearchRequest.of("value2*").build(), programRecord, datasetRecord);
    // Search based on value prefix in the wrong namespace
    assertEmpty(mds, SearchRequest.of("value2*").addNamespace("ns12").build());
    // clean up
    mds.batch(batch(new Drop(program), new Drop(dataset)), MutationOptions.DEFAULT);
}

Example 27 with Update

use of io.cdap.cdap.spi.metadata.MetadataMutation.Update in project cdap by cdapio.

the class MetadataStorageTest method testSearchOnTags.

@Test
public void testSearchOnTags() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    String ns1 = "ns1";
    String ns2 = "ns2";
    MetadataEntity app1 = ofApp(ns1, "app1");
    MetadataEntity app2 = ofApp(ns2, "app1");
    MetadataEntity program1 = ofWorker(app1, "wk1");
    MetadataEntity dataset1 = ofDataset(ns1, "ds1");
    MetadataEntity dataset2 = ofDataset(ns1, "ds2");
    MetadataEntity file1 = MetadataEntity.builder(dataset1).appendAsType("file", "f1").build();
    List<MetadataEntity> entities = ImmutableList.of(app1, app2, program1, dataset1, dataset2, file1);
    for (MetadataEntity entity : entities) {
        Assert.assertEquals(Metadata.EMPTY, mds.read(new Read(entity)));
    }
    // add tags for these entities
    MetadataRecord app1Record = new MetadataRecord(app1, new Metadata(USER, tags("tag1", "tag2", "tag3")));
    MetadataRecord app2Record = new MetadataRecord(app2, new Metadata(USER, tags("tag1", "tag2", "tag3_more")));
    MetadataRecord program1Record = new MetadataRecord(program1, new Metadata(USER, tags("tag1")));
    MetadataRecord dataset1Record = new MetadataRecord(dataset1, new Metadata(USER, tags("tag3", "tag2", "tag12-tag33")));
    MetadataRecord dataset2Record = new MetadataRecord(dataset2, new Metadata(USER, tags("tag2", "tag4")));
    MetadataRecord file1Record = new MetadataRecord(file1, new Metadata(USER, tags("tag2", "tag5")));
    mds.batch(ImmutableList.of(app1Record, app2Record, program1Record, dataset1Record, dataset2Record, file1Record).stream().map(record -> new Update(record.getEntity(), record.getMetadata())).collect(Collectors.toList()), MutationOptions.DEFAULT);
    // Try to search on all tags
    assertResults(mds, SearchRequest.of("tags:*").addNamespace(ns1).build(), app1Record, program1Record, dataset1Record, dataset2Record, file1Record);
    // Try to search for tag1*
    assertResults(mds, SearchRequest.of("tags:tag1*").addNamespace(ns1).build(), app1Record, program1Record, dataset1Record);
    // Try to search for tag1 with spaces in search query and mixed case of tags keyword
    assertResults(mds, SearchRequest.of("  tAGS  :  tag1  ").addNamespace(ns1).build(), app1Record, program1Record);
    // Try to search for tag5
    assertResults(mds, SearchRequest.of("tags:tag5").addNamespace(ns1).build(), file1Record);
    // Try to search for tag2
    assertResults(mds, SearchRequest.of("tags:tag2").addNamespace(ns1).build(), app1Record, dataset1Record, dataset2Record, file1Record);
    // Try to search for tag4
    assertResults(mds, SearchRequest.of("tags:tag4").addNamespace(ns1).build(), dataset2Record);
    // Try to search for tag33
    assertResults(mds, SearchRequest.of("tags:tag33").addNamespace(ns1).build(), dataset1Record);
    // Try to search for a tag which has - in it
    assertResults(mds, SearchRequest.of("tag12-tag33").addNamespace(ns1).build(), dataset1Record);
    // Try to search for tag33 with spaces in query
    assertResults(mds, SearchRequest.of("  tag33  ").addNamespace(ns1).build(), dataset1Record);
    // Try wildcard search for tag3*
    assertResults(mds, SearchRequest.of("tags:tag3*").addNamespace(ns1).build(), app1Record, dataset1Record);
    // try search in another namespace
    assertResults(mds, SearchRequest.of("tags:tag1").addNamespace(ns2).build(), app2Record);
    assertResults(mds, SearchRequest.of("tag3").addNamespace(ns2).build(), app2Record);
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns2).build(), app2Record);
    // try to search across namespaces
    assertResults(mds, SearchRequest.of("tags:tag1").build(), app1Record, app2Record, program1Record);
    // cleanup
    mds.batch(entities.stream().map(Drop::new).collect(Collectors.toList()), MutationOptions.DEFAULT);
    // Search should be empty after deleting tags
    assertEmpty(mds, SearchRequest.of("*").setLimit(10).build());
}

Example 28 with Update

use of io.cdap.cdap.spi.metadata.MetadataMutation.Update in project cdap by cdapio.

the class MetadataStorageTest method testCursorsOffsetsAndTotals.

@Test
public void testCursorsOffsetsAndTotals() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    List<MetadataRecord> records = IntStream.range(0, 20).mapToObj(i -> new MetadataRecord(ofDataset(DEFAULT_NAMESPACE, "ds" + i), new Metadata(SYSTEM, props(ENTITY_NAME_KEY, "ds" + i)))).collect(Collectors.toList());
    mds.batch(records.stream().map(record -> new Update(record.getEntity(), record.getMetadata())).collect(Collectors.toList()), MutationOptions.DEFAULT);
    // no cursors
    validateCursorAndOffset(mds, 0, 10, null, false, 10, 0, 10, true, false);
    validateCursorAndOffset(mds, 5, 10, null, false, 10, 5, 10, true, false);
    validateCursorAndOffset(mds, 10, 10, null, false, 10, 10, 10, false, false);
    validateCursorAndOffset(mds, 15, 10, null, false, 5, 15, 10, false, false);
    validateCursorAndOffset(mds, 20, 10, null, false, 0, 20, 10, false, false);
    validateCursorAndOffset(mds, 25, 10, null, false, 0, 25, 10, false, false);
    // request cursors, but don't use them
    validateCursorAndOffset(mds, 0, 10, null, true, 10, 0, 10, true, true);
    validateCursorAndOffset(mds, 0, 20, null, true, 20, 0, 20, false, false);
    validateCursorAndOffset(mds, 0, 30, null, true, 20, 0, 30, false, false);
    // test that passing in an empty string as the cursor has the same effect as null
    validateCursorAndOffset(mds, 0, 10, "", true, 10, 0, 10, true, true);
    validateCursorAndOffset(mds, 0, 20, "", true, 20, 0, 20, false, false);
    validateCursorAndOffset(mds, 0, 30, "", true, 20, 0, 30, false, false);
    // request cursor, and use it
    String cursor = validateCursorAndOffset(mds, 0, 8, null, true, 8, 0, 8, true, true);
    cursor = validateCursorAndOffset(mds, 0, 8, cursor, true, 8, 8, 8, true, true);
    validateCursorAndOffset(mds, 0, 8, cursor, true, 4, 16, 8, false, false);
    // request a cursor that matches evenly with the number of results
    cursor = validateCursorAndOffset(mds, 0, 10, null, true, 10, 0, 10, true, true);
    validateCursorAndOffset(mds, 0, 10, cursor, true, 10, 10, 10, false, false);
    // ensure that offset and limit are superseded by cursor
    cursor = validateCursorAndOffset(mds, 0, 4, null, true, 4, 0, 4, true, true);
    cursor = validateCursorAndOffset(mds, 0, 0, cursor, true, 4, 4, 4, true, true);
    cursor = validateCursorAndOffset(mds, 10, 100, cursor, true, 4, 8, 4, true, true);
    cursor = validateCursorAndOffset(mds, 12, 2, cursor, true, 4, 12, 4, true, true);
    validateCursorAndOffset(mds, 1, 1, cursor, true, 4, 16, 4, false, false);
    // ensure that we can start searching without cursor, with offset, and request a cursor
    // whether a cursor is returned, is implementation dependent
    cursor = validateCursorAndOffset(mds, 4, 4, null, true, 4, 4, 4, true, null);
    validateCursorAndOffset(mds, 8, 4, cursor, true, 4, 8, 4, true, null);
    // clean up
    mds.batch(records.stream().map(MetadataRecord::getEntity).map(Drop::new).collect(Collectors.toList()), MutationOptions.DEFAULT);
}

Example 29 with Update

use of io.cdap.cdap.spi.metadata.MetadataMutation.Update in project cdap by caskdata.

the class MetadataStorageTest method testSearchOnKeyValue.

@Test
public void testSearchOnKeyValue() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity program = ofWorker(ofApp("ns1", "app1"), "wk1");
    MetadataEntity dataset = ofDataset("ns1", "ds2");
    // add properties for program and dataset
    final String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    MetadataRecord programRecord = new MetadataRecord(program, new Metadata(USER, props("key1", "value1", "key2", "value2", "multiword", multiWordValue)));
    MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(ImmutableSet.of(), ImmutableMap.of(new ScopedName(SYSTEM, "sKey1"), "sValue1", new ScopedName(USER, "Key1"), "Value1")));
    mds.batch(batch(new Update(program, programRecord.getMetadata()), new Update(dataset, datasetRecord.getMetadata())), MutationOptions.DEFAULT);
    // search based on the names of properties
    assertResults(mds, SearchRequest.of("properties:key1").build(), programRecord, datasetRecord);
    assertResults(mds, SearchRequest.of("properties:skey1").setScope(SYSTEM).build(), datasetRecord);
    assertResults(mds, SearchRequest.of("properties:multi*").setScope(USER).build(), programRecord);
    // Search for it based on key and value
    assertResults(mds, SearchRequest.of("key1:value1").addType(TYPE_PROGRAM).build(), programRecord);
    assertResults(mds, SearchRequest.of("key1:value1").build(), programRecord, datasetRecord);
    // Search for it based on a word in value
    assertResults(mds, SearchRequest.of("multiword:aV5").build(), programRecord);
    // Search for it based on a word in value with spaces in search query
    assertResults(mds, SearchRequest.of("  multiword:aV1   ").build(), programRecord);
    // remove the multiword property
    mds.apply(new Remove(program, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, USER, "multiword"))), MutationOptions.DEFAULT);
    programRecord = new MetadataRecord(program, new Metadata(USER, props("key1", "value1", "key2", "value2")));
    // search results should be empty after removing this key as the indexes are deleted
    assertEmpty(mds, SearchRequest.of("multiword:aV5").build());
    // Test wrong ns
    assertEmpty(mds, SearchRequest.of("key1:value1").addNamespace("ns12").build());
    // Test multi word query
    assertResults(mds, SearchRequest.of("  value1  av2  ").build(), programRecord, datasetRecord);
    assertResults(mds, SearchRequest.of("  value1  sValue1  ").build(), programRecord, datasetRecord);
    assertResults(mds, SearchRequest.of("  valu*  sVal*  ").build(), programRecord, datasetRecord);
    // search with explicit set of target types
    assertResults(mds, SearchRequest.of("  valu*  sVal*  ").addType(TYPE_PROGRAM).addType(TYPE_DATASET).build(), programRecord, datasetRecord);
    // search all target types
    assertResults(mds, SearchRequest.of("  valu*  sVal*  ").build(), programRecord, datasetRecord);
    // clean up
    mds.batch(batch(new Drop(program), new Drop(dataset)), MutationOptions.DEFAULT);
}

Example 30 with Update

use of io.cdap.cdap.spi.metadata.MetadataMutation.Update in project cdap by caskdata.

the class MetadataStorageTest method testSearchOnTagsUpdate.

@Test
public void testSearchOnTagsUpdate() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity entity = ofWorkflow(ofApp(DEFAULT_NAMESPACE, "appX"), "wtf");
    Metadata meta = new Metadata(SYSTEM, tags("tag1", "tag2"));
    mds.apply(new Update(entity, meta), MutationOptions.DEFAULT);
    Assert.assertEquals(meta.getTags(SYSTEM), mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM));
    assertResults(mds, SearchRequest.of("tag1").build(), new MetadataRecord(entity, meta));
    // add an more tags
    mds.apply(new Update(entity, new Metadata(SYSTEM, tags("tag3", "tag4"))), MutationOptions.DEFAULT);
    Set<String> newTags = tags("tag1", "tag2", "tag3", "tag4");
    Metadata newMeta = new Metadata(SYSTEM, newTags);
    Assert.assertEquals(newTags, mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM));
    for (String expectedTag : newTags) {
        assertResults(mds, SearchRequest.of(expectedTag).build(), new MetadataRecord(entity, newMeta));
    }
    // add an empty set of tags. This should have no effect on retrieval or search of tags
    mds.apply(new Update(entity, new Metadata(SYSTEM, tags())), MutationOptions.DEFAULT);
    Assert.assertEquals(newTags, mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM));
    for (String expectedTag : newTags) {
        assertResults(mds, SearchRequest.of(expectedTag).build(), new MetadataRecord(entity, newMeta));
    }
    // clean up
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}