Examples with MetadataEntity io.cdap.cdap.api.metadata.MetadataEntity used on opensource projects

Example 41 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataStorageTest method testUpdateDropConflictInBatch.

/**
 * See {@link #testUpdateDropConflict()} for a description. The difference in this test is that
 * we issue batches of mutations over a collection of entities. The same assumptions apply,
 * however, for each entity.
 */
@Test
@Category(SlowTests.class)
public void testUpdateDropConflictInBatch() throws IOException {
    int numTests = 10;
    int numThreads = 2;
    int numEntities = 20;
    MetadataStorage mds = getMetadataStorage();
    ExecutorService executor = Executors.newFixedThreadPool(numThreads);
    CompletionService<List<MetadataChange>> completionService = new ExecutorCompletionService<>(executor);
    Map<Integer, MetadataEntity> entities = IntStream.range(0, numEntities).boxed().collect(Collectors.toMap(i -> i, i -> MetadataEntity.ofDataset("myds" + i)));
    List<MetadataMutation> creates = entities.values().stream().map(e -> new Create(e, new Metadata(USER, tags("a")), Collections.emptyMap())).collect(Collectors.toList());
    Random rand = new Random(System.currentTimeMillis());
    IntStream.range(0, numTests).forEach(x -> {
        try {
            mds.batch(creates, MutationOptions.DEFAULT);
            Map<Integer, List<MetadataMutation>> mutations = IntStream.range(0, numThreads).boxed().collect(Collectors.toMap(i -> i, i -> new ArrayList<>()));
            IntStream.range(0, numEntities).forEach(e -> {
                // ensure that at least one thread attempts to drop this entity
                int dropThread = rand.nextInt(numThreads);
                IntStream.range(0, numThreads).forEach(t -> {
                    if (t == dropThread || rand.nextInt(100) < 50) {
                        mutations.get(t).add(new Drop(entities.get(e)));
                    } else {
                        mutations.get(t).add(new Update(entities.get(e), new Metadata(USER, tags("b"))));
                    }
                });
            });
            IntStream.range(0, numThreads).forEach(t -> completionService.submit(() -> mds.batch(mutations.get(t), MutationOptions.DEFAULT)));
            IntStream.range(0, numThreads).forEach(t -> {
                try {
                    completionService.take();
                } catch (InterruptedException e) {
                    throw Throwables.propagate(e);
                }
            });
            IntStream.range(0, numEntities).forEach(e -> {
                try {
                    // each entity is either dropped then updated (and then it has tag "b" only)
                    // or it first update and then dropped (and then it has empty metadata)
                    Assert.assertTrue(ImmutableSet.of(Metadata.EMPTY, new Metadata(USER, tags("b"))).contains(mds.read(new Read(entities.get(e)))));
                } catch (Exception ex) {
                    throw Throwables.propagate(ex);
                }
            });
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    });
    mds.batch(entities.values().stream().map(Drop::new).collect(Collectors.toList()), MutationOptions.DEFAULT);
}

Example 42 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataStorageTest method testSearchOnValue.

@Test
public void testSearchOnValue() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity program = ofWorker(ofApp("ns1", "app1"), "wk1");
    MetadataEntity dataset = ofDataset("ns1", "ds2");
    // Add some metadata
    final String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    MetadataRecord programRecord = new MetadataRecord(program, new Metadata(USER, props("key1", "value1", "key2", "value2", "multiword", multiWordValue)));
    mds.apply(new Update(program, programRecord.getMetadata()), MutationOptions.DEFAULT);
    // Search for it based on value
    assertResults(mds, SearchRequest.of("value1").build(), programRecord);
    assertResults(mds, SearchRequest.of("  aV1   ").addType(TYPE_PROGRAM).build(), programRecord);
    assertEmpty(mds, SearchRequest.of("  aV1   ").addType(TYPE_ARTIFACT).build());
    // Search for it based split patterns to make sure nothing is matched
    assertEmpty(mds, SearchRequest.of("-").build());
    assertEmpty(mds, SearchRequest.of(",").build());
    assertEmpty(mds, SearchRequest.of("_").build());
    assertEmpty(mds, SearchRequest.of(", ,").build());
    assertEmpty(mds, SearchRequest.of(", - ,").build());
    // Search for it based on a word in value
    assertResults(mds, SearchRequest.of("av5").addType(TYPE_PROGRAM).build(), programRecord);
    // Case insensitive
    assertResults(mds, SearchRequest.of("ValUe1").addType(TYPE_PROGRAM).build(), programRecord);
    // add a property for the program
    mds.apply(new Update(program, new Metadata(SYSTEM, props("key3", "value1"))), MutationOptions.DEFAULT);
    programRecord = new MetadataRecord(program, new Metadata(ImmutableSet.of(), ImmutableMap.of(new ScopedName(USER, "key1"), "value1", new ScopedName(USER, "key2"), "value2", new ScopedName(USER, "multiword"), multiWordValue, new ScopedName(SYSTEM, "key3"), "value1")));
    // search by value
    assertResults(mds, SearchRequest.of("value1").addType(TYPE_PROGRAM).build(), programRecord);
    // add a property for the dataset
    MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(USER, props("key21", "value21")));
    mds.apply(new Update(dataset, datasetRecord.getMetadata()), MutationOptions.DEFAULT);
    // Search based on value prefix
    assertResults(mds, SearchRequest.of("value2*").build(), programRecord, datasetRecord);
    // Search based on value prefix in the wrong namespace
    assertEmpty(mds, SearchRequest.of("value2*").addNamespace("ns12").build());
    // clean up
    mds.batch(batch(new Drop(program), new Drop(dataset)), MutationOptions.DEFAULT);
}

Example 43 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class EntityIdTest method testGetSelfOrParentEntityId.

@Test
public void testGetSelfOrParentEntityId() {
    // entity is a known entityId
    MetadataEntity metadataEntity = MetadataEntity.ofDataset("testNs", "testDs");
    Assert.assertEquals(new DatasetId("testNs", "testDs"), EntityId.getSelfOrParentEntityId(metadataEntity));
    // entity's parent is not a known entity
    metadataEntity = MetadataEntity.builder().append("ab", "cd").appendAsType("unkonwType", "value").append("ef", "gh").build();
    try {
        EntityId.getSelfOrParentEntityId(metadataEntity);
        Assert.fail("Should have failed to get a parent entity id");
    } catch (IllegalArgumentException e) {
    // expected
    }
    metadataEntity = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "testNs").append(MetadataEntity.DATASET, "testDs").appendAsType("field", "testField").build();
    Assert.assertEquals(new DatasetId("testNs", "testDs"), EntityId.getSelfOrParentEntityId(metadataEntity));
    metadataEntity = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "testNs").append(MetadataEntity.APPLICATION, "testApp").appendAsType("custom", "custValue").build();
    Assert.assertEquals(new ApplicationId("testNs", "testApp"), EntityId.getSelfOrParentEntityId(metadataEntity));
    metadataEntity = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "testNs").append(MetadataEntity.APPLICATION, "testApp").append(MetadataEntity.TYPE, ProgramType.WORKER.getPrettyName()).append(MetadataEntity.PROGRAM, "testProg").appendAsType("subType", "realtime").build();
    Assert.assertEquals(new ProgramId("testNs", "testApp", ProgramType.WORKER, "testProg"), EntityId.getSelfOrParentEntityId(metadataEntity));
    ArtifactId artifactId = new ArtifactId("testNs", "testArtifact-1.0.0.jar");
    metadataEntity = MetadataEntity.builder(artifactId.toMetadataEntity()).appendAsType("time", "t2").build();
    Assert.assertEquals(artifactId, EntityId.getSelfOrParentEntityId(metadataEntity));
}

Example 44 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class EntityIdTest method testFromMetadataEntity.

@Test
public void testFromMetadataEntity() {
    ApplicationId applicationId = new ApplicationId("testNs", "app1");
    Assert.assertEquals(applicationId, EntityId.fromMetadataEntity(applicationId.toMetadataEntity()));
    MetadataEntity metadataEntity = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "testNs").appendAsType(MetadataEntity.APPLICATION, "app1").build();
    Assert.assertEquals(applicationId, EntityId.fromMetadataEntity(metadataEntity));
    metadataEntity = MetadataEntity.builder(MetadataEntity.ofDataset("testNs", "testDs")).appendAsType("field", "testField").build();
    try {
        EntityId.fromMetadataEntity(metadataEntity);
        Assert.fail("Should have failed to get create an EntityId from MetadataEntity");
    } catch (IllegalArgumentException e) {
    // expected
    }
    ProgramId programId = new ProgramId(applicationId, ProgramType.WORKER, "testWorker");
    Assert.assertEquals(programId, EntityId.fromMetadataEntity(programId.toMetadataEntity()));
    // artifact
    ArtifactId artifactId = new ArtifactId("testNs", "testArtifact-1.0.0.jar");
    Assert.assertEquals(artifactId, EntityId.fromMetadataEntity(artifactId.toMetadataEntity()));
    // schedule
    ScheduleId scheduleId = applicationId.schedule("sch");
    metadataEntity = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "testNs").append(MetadataEntity.APPLICATION, "app1").appendAsType(MetadataEntity.SCHEDULE, "sch").build();
    Assert.assertEquals(scheduleId, EntityId.fromMetadataEntity(metadataEntity));
}

Example 45 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataStorageTest method testUpdateRemove.

@Test
public void testUpdateRemove() throws IOException {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity entity = ofDataset(DEFAULT_NAMESPACE, "entity");
    // get metadata for non-existing entity
    verifyMetadata(mds, entity, Metadata.EMPTY);
    // add some metadata for the entity
    Metadata metadata = new Metadata(ImmutableSet.of(new ScopedName(SYSTEM, "sysTag"), new ScopedName(USER, "userTag")), ImmutableMap.of(new ScopedName(SYSTEM, "sysProp"), "sysVal", new ScopedName(USER, "userProp"), "userVal"));
    MetadataChange change = mds.apply(new Update(entity, metadata), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, metadata), change);
    verifyMetadata(mds, entity, metadata);
    // remove everything
    change = mds.apply(new Remove(entity), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, metadata, Metadata.EMPTY), change);
    verifyMetadata(mds, entity, Metadata.EMPTY);
    // add back all metadata, then remove everything in user scope
    change = mds.apply(new Update(entity, metadata), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, metadata), change);
    verifyMetadata(mds, entity, metadata);
    change = mds.apply(new Remove(entity, USER), MutationOptions.DEFAULT);
    Metadata newMetadata = filterBy(metadata, SYSTEM, null);
    Assert.assertEquals(new MetadataChange(entity, metadata, newMetadata), change);
    verifyMetadata(mds, entity, newMetadata);
    // add back all metadata, then remove everything in system scope
    change = mds.apply(new Update(entity, metadata), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, newMetadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    change = mds.apply(new Remove(entity, SYSTEM), MutationOptions.DEFAULT);
    newMetadata = filterBy(metadata, USER, null);
    Assert.assertEquals(new MetadataChange(entity, metadata, newMetadata), change);
    verifyMetadata(mds, entity, newMetadata);
    // add back all metadata, then remove all tags
    change = mds.apply(new Update(entity, metadata), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, newMetadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    change = mds.apply(new Remove(entity, TAG), MutationOptions.DEFAULT);
    newMetadata = filterBy(metadata, null, PROPERTY);
    Assert.assertEquals(new MetadataChange(entity, metadata, newMetadata), change);
    verifyMetadata(mds, entity, newMetadata);
    // add back all metadata, then remove all properties
    change = mds.apply(new Update(entity, metadata), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, newMetadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    change = mds.apply(new Remove(entity, PROPERTY), MutationOptions.DEFAULT);
    newMetadata = filterBy(metadata, null, TAG);
    Assert.assertEquals(new MetadataChange(entity, metadata, newMetadata), change);
    verifyMetadata(mds, entity, newMetadata);
    // add back all metadata, then remove all properties in system scope
    change = mds.apply(new Update(entity, metadata), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, newMetadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    change = mds.apply(new Remove(entity, SYSTEM, PROPERTY), MutationOptions.DEFAULT);
    newMetadata = union(filterBy(metadata, SYSTEM, TAG), filterBy(metadata, USER, null));
    Assert.assertEquals(new MetadataChange(entity, metadata, newMetadata), change);
    verifyMetadata(mds, entity, newMetadata);
    // add back all metadata, then remove all tags in user scope
    change = mds.apply(new Update(entity, metadata), MutationOptions.DEFAULT);
    Assert.assertEquals(new MetadataChange(entity, newMetadata, metadata), change);
    verifyMetadata(mds, entity, metadata);
    change = mds.apply(new Remove(entity, USER, TAG), MutationOptions.DEFAULT);
    newMetadata = union(filterBy(metadata, USER, PROPERTY), filterBy(metadata, SYSTEM, null));
    Assert.assertEquals(new MetadataChange(entity, metadata, newMetadata), change);
    verifyMetadata(mds, entity, newMetadata);
    // clean up
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}