Examples with MetadataEntity io.cdap.cdap.api.metadata.MetadataEntity used on opensource projects

Example 31 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataTest method testToEntityId.

@Test
public void testToEntityId() {
    // should be able to get get an EntityId if Metadata belong to a cdap entity id
    DatasetId myDs = NamespaceId.DEFAULT.dataset("myDs");
    MetadataDataset.Record metadata1 = new MetadataDataset.Record(myDs);
    Assert.assertEquals(myDs, metadata1.getEntityId());
    MetadataEntity metadataEntity = MetadataEntity.builder(MetadataEntity.ofDataset(NamespaceId.DEFAULT.getEntityName(), "myDs")).appendAsType("field", "myField").build();
    MetadataDataset.Record metadata2 = new MetadataDataset.Record(metadataEntity);
    try {
        metadata2.getEntityId();
        Assert.fail();
    } catch (IllegalArgumentException e) {
    // expected
    }
}

Example 32 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataKeyTest method testGetTargetTypeChild.

@Test
public void testGetTargetTypeChild() {
    ApplicationId expectedAppId = new ApplicationId("ns1", "app1");
    MDSKey mdsValueKey = MetadataKey.createValueRowKey(expectedAppId.toMetadataEntity(), "key1");
    ProgramId expectedProgramId = expectedAppId.spark("spark1");
    MDSKey mdsValueKey2 = MetadataKey.createValueRowKey(expectedProgramId.toMetadataEntity(), "key2");
    // assert that the key for parent child are independent and correct
    MetadataEntity actualAppId = MetadataKey.extractMetadataEntityFromKey(mdsValueKey.getKey());
    Assert.assertEquals(expectedAppId.toMetadataEntity(), actualAppId);
    MetadataEntity actualProgramId = MetadataKey.extractMetadataEntityFromKey(mdsValueKey2.getKey());
    Assert.assertEquals(expectedProgramId.toMetadataEntity(), actualProgramId);
}

Example 33 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataStorageTest method testSearchOnSchema.

@Test
public void testSearchOnSchema() throws IOException {
    Schema bytesArraySchema = Schema.arrayOf(Schema.of(Schema.Type.BYTES));
    Schema stringArraySchema = Schema.arrayOf(Schema.of(Schema.Type.STRING));
    Schema booleanBytesMapSchema = Schema.mapOf(Schema.of(Schema.Type.BOOLEAN), Schema.of(Schema.Type.BYTES));
    Schema nestedMapSchema = Schema.mapOf(bytesArraySchema, booleanBytesMapSchema);
    Schema record22Schema = Schema.recordOf("record22", Schema.Field.of("a", nestedMapSchema));
    Schema record22ArraySchema = Schema.arrayOf(record22Schema);
    Schema bytesDoubleMapSchema = Schema.mapOf(Schema.of(Schema.Type.BYTES), Schema.of(Schema.Type.DOUBLE));
    Schema record21Schema = Schema.recordOf("record21", Schema.Field.of("x", Schema.of(Schema.Type.STRING)), Schema.Field.of("y", stringArraySchema), Schema.Field.of("z", bytesDoubleMapSchema));
    Schema record21to22MapSchema = Schema.mapOf(record21Schema, record22ArraySchema);
    Schema nullableIntSchema = Schema.nullableOf(Schema.of(Schema.Type.INT));
    Schema tripeUnionSchema = Schema.unionOf(Schema.of(Schema.Type.INT), Schema.of(Schema.Type.LONG), Schema.of(Schema.Type.NULL));
    Schema complexSchema = Schema.recordOf("record1", Schema.Field.of("map1", record21to22MapSchema), Schema.Field.of("i", nullableIntSchema), Schema.Field.of("j", tripeUnionSchema));
    Schema anotherComplexSchema = Schema.arrayOf(Schema.of(Schema.Type.STRING));
    Schema superComplexSchema = Schema.unionOf(complexSchema, anotherComplexSchema, Schema.of(Schema.Type.NULL));
    String[] expectedTermsInIndex = { "record1", "record1:RECORD", "map1", "map1:MAP", "record21", "record21:RECORD", "x", "x:STRING", "y", "y:ARRAY", "z", "z:MAP", "record22", "record22:RECORD", "a", "a:MAP", "i", "i:INT", "j", "j:UNION" };
    MetadataEntity entity = MetadataEntity.ofDataset("myDs");
    Metadata meta = new Metadata(SYSTEM, props(MetadataConstants.ENTITY_NAME_KEY, "myDs", MetadataConstants.SCHEMA_KEY, superComplexSchema.toString()));
    MetadataRecord record = new MetadataRecord(entity, meta);
    MetadataStorage mds = getMetadataStorage();
    mds.apply(new Update(entity, meta), MutationOptions.DEFAULT);
    assertResults(mds, SearchRequest.of("myds").build(), record);
    assertResults(mds, SearchRequest.of("schema:*").build(), record);
    assertResults(mds, SearchRequest.of("properties:schema").build(), record);
    for (String expectedTerm : expectedTermsInIndex) {
        assertResults(mds, SearchRequest.of(expectedTerm).build(), record);
        assertResults(mds, SearchRequest.of("schema:" + expectedTerm).build(), record);
    }
    // clean up
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}

Example 34 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataStorageTest method testCrossNamespaceDefaultSearch.

@Test
public void testCrossNamespaceDefaultSearch() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity ns1app = ofApp("ns1", "a");
    MetadataEntity ns2app = ofApp("ns2", "a");
    MetadataRecord app1Record = new MetadataRecord(ns1app, new Metadata(USER, props("k1", "v1", "k2", "v2")));
    MetadataRecord app2Record = new MetadataRecord(ns2app, new Metadata(USER, props("k1", "v1")));
    mds.batch(batch(new Update(ns1app, app1Record.getMetadata()), new Update(ns2app, app2Record.getMetadata())), MutationOptions.DEFAULT);
    assertResults(mds, SearchRequest.of("v1").build(), app1Record, app2Record);
    assertResults(mds, SearchRequest.of("v2").build(), app1Record);
    assertResults(mds, SearchRequest.of("*").build(), app1Record, app2Record);
    // clean up
    mds.batch(batch(new Drop(ns1app), new Drop(ns2app)), MutationOptions.DEFAULT);
}

Example 35 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataStorageTest method testAsyncMutations.

@Test
public void testAsyncMutations() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity entity = ofDataset(DEFAULT_NAMESPACE, "entity");
    // get metadata for non-existing entity
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // drop metadata for non-existing entity succeeds
    MetadataChange change = mds.apply(new Drop(entity), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // remove metadata for non-existing entity succeeds
    mds.apply(new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "ut1"), new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), new ScopedNameOfKind(PROPERTY, USER, "up2"))), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // update metadata for non-existing entity creates it
    Metadata metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "a"), new ScopedName(USER, "b")), ImmutableMap.of(new ScopedName(SYSTEM, "p"), "v", new ScopedName(USER, "k"), "v1"));
    change = mds.apply(new Update(entity, metadata), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // test that update is idempotent
    change = mds.apply(new Update(entity, metadata), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // create metadata replaces existing metadata
    Metadata previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(USER, "ut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(USER, "up1"), "uv1", new ScopedName(USER, "up2"), "uv2"));
    MetadataMutation create = new Create(entity, metadata, Collections.emptyMap());
    change = mds.apply(create, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    // verify the metadata with variations of scope and kind
    verifyMetadataAsync(mds, entity, metadata);
    // verify the metadata with a select subset of tags and properties
    verifyMetadataSelectionAsync(mds, entity, metadata, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), new ScopedNameOfKind(PROPERTY, SYSTEM, "nosuch"), new ScopedNameOfKind(PROPERTY, USER, "up2"), new ScopedNameOfKind(PROPERTY, USER, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, USER, "ut1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch")));
    // verify that a non-matching set tags and properties returns empty metadata
    verifyMetadataSelectionAsync(mds, entity, metadata, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch")));
    // replace the system metadata with directives, user metadata should remain unchanged
    Metadata recreatedMetadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "nst0")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp0"), "sv0"));
    MetadataMutation recreate = new Create(entity, recreatedMetadata, ImmutableMap.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), MetadataDirective.KEEP, new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), MetadataDirective.KEEP));
    previousMetadata = metadata;
    // this is the new metadata according to directives
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst0"), new ScopedName(USER, "ut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp0"), "sv0", new ScopedName(USER, "up1"), "uv1", new ScopedName(USER, "up2"), "uv2"));
    change = mds.apply(recreate, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // replace the metadata with directives
    recreatedMetadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp2"), "sv2", new ScopedName(USER, "up3"), "uv3"));
    recreate = new Create(entity, recreatedMetadata, ImmutableMap.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), MetadataDirective.KEEP, new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), MetadataDirective.KEEP, new ScopedNameOfKind(PROPERTY, USER, "up2"), MetadataDirective.PRESERVE));
    previousMetadata = metadata;
    // this is the new metadata according to directives
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "sv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp2"), "sv2", new ScopedName(USER, "up2"), "uv2", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(recreate, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // update some tags and properties
    MetadataMutation update = new Update(entity, new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(USER, "aut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "nsp2"), "nsv2", new ScopedName(USER, "up2"), "nuv2", new ScopedName(USER, "up3"), "uv3")));
    // verify new metadata after update
    previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(SYSTEM, "st1"), new ScopedName(SYSTEM, "st2"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "aut1"), new ScopedName(USER, "nut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(SYSTEM, "sp2"), "sv2", new ScopedName(SYSTEM, "nsp2"), "nsv2", new ScopedName(USER, "up2"), "nuv2", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(update, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // test that update is idempotent
    change = mds.apply(update, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // remove some tags and properties
    MetadataMutation remove = new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st1"), new ScopedNameOfKind(MetadataKind.TAG, SYSTEM, "st2"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nut1"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch"), new ScopedNameOfKind(PROPERTY, SYSTEM, "sp2"), new ScopedNameOfKind(PROPERTY, SYSTEM, "nsp2"), new ScopedNameOfKind(PROPERTY, USER, "up2")));
    previousMetadata = metadata;
    metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "ast1"), new ScopedName(SYSTEM, "nst1"), new ScopedName(USER, "aut1")), ImmutableMap.of(new ScopedName(SYSTEM, "sp1"), "nsv1", new ScopedName(USER, "up3"), "uv3"));
    change = mds.apply(remove, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // test that remove is idemtpotent
    change = mds.apply(remove, ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, metadata), change);
    verifyMetadataAsync(mds, entity, metadata);
    // drop all metadata for the entity
    change = mds.apply(new Drop(entity), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, metadata, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
    // drop is idempotent
    change = mds.apply(new Drop(entity), ASYNC);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change);
    verifyMetadataAsync(mds, entity, Metadata.EMPTY);
}