Examples with MetadataEntity io.cdap.cdap.api.metadata.MetadataEntity used on opensource projects

Example 66 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testUpdateDropConflictInBatch.

/**
 * See {@link #testUpdateDropConflict()} for a description. The difference in this test is that
 * we issue batches of mutations over a collection of entities. The same assumptions apply,
 * however, for each entity.
 */
@Test
@Category(SlowTests.class)
public void testUpdateDropConflictInBatch() throws IOException {
    int numTests = 10;
    int numThreads = 2;
    int numEntities = 20;
    MetadataStorage mds = getMetadataStorage();
    ExecutorService executor = Executors.newFixedThreadPool(numThreads);
    CompletionService<List<MetadataChange>> completionService = new ExecutorCompletionService<>(executor);
    Map<Integer, MetadataEntity> entities = IntStream.range(0, numEntities).boxed().collect(Collectors.toMap(i -> i, i -> MetadataEntity.ofDataset("myds" + i)));
    List<MetadataMutation> creates = entities.values().stream().map(e -> new Create(e, new Metadata(USER, tags("a")), Collections.emptyMap())).collect(Collectors.toList());
    Random rand = new Random(System.currentTimeMillis());
    IntStream.range(0, numTests).forEach(x -> {
        try {
            mds.batch(creates, MutationOptions.DEFAULT);
            Map<Integer, List<MetadataMutation>> mutations = IntStream.range(0, numThreads).boxed().collect(Collectors.toMap(i -> i, i -> new ArrayList<>()));
            IntStream.range(0, numEntities).forEach(e -> {
                // ensure that at least one thread attempts to drop this entity
                int dropThread = rand.nextInt(numThreads);
                IntStream.range(0, numThreads).forEach(t -> {
                    if (t == dropThread || rand.nextInt(100) < 50) {
                        mutations.get(t).add(new Drop(entities.get(e)));
                    } else {
                        mutations.get(t).add(new Update(entities.get(e), new Metadata(USER, tags("b"))));
                    }
                });
            });
            IntStream.range(0, numThreads).forEach(t -> completionService.submit(() -> mds.batch(mutations.get(t), MutationOptions.DEFAULT)));
            IntStream.range(0, numThreads).forEach(t -> {
                try {
                    completionService.take();
                } catch (InterruptedException e) {
                    throw Throwables.propagate(e);
                }
            });
            IntStream.range(0, numEntities).forEach(e -> {
                try {
                    // each entity is either dropped then updated (and then it has tag "b" only)
                    // or it first update and then dropped (and then it has empty metadata)
                    Assert.assertTrue(ImmutableSet.of(Metadata.EMPTY, new Metadata(USER, tags("b"))).contains(mds.read(new Read(entities.get(e)))));
                } catch (Exception ex) {
                    throw Throwables.propagate(ex);
                }
            });
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    });
    mds.batch(entities.values().stream().map(Drop::new).collect(Collectors.toList()), MutationOptions.DEFAULT);
}

Example 67 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testSearchOnValue.

@Test
public void testSearchOnValue() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    MetadataEntity program = ofWorker(ofApp("ns1", "app1"), "wk1");
    MetadataEntity dataset = ofDataset("ns1", "ds2");
    // Add some metadata
    final String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    MetadataRecord programRecord = new MetadataRecord(program, new Metadata(USER, props("key1", "value1", "key2", "value2", "multiword", multiWordValue)));
    mds.apply(new Update(program, programRecord.getMetadata()), MutationOptions.DEFAULT);
    // Search for it based on value
    assertResults(mds, SearchRequest.of("value1").build(), programRecord);
    assertResults(mds, SearchRequest.of("  aV1   ").addType(TYPE_PROGRAM).build(), programRecord);
    assertEmpty(mds, SearchRequest.of("  aV1   ").addType(TYPE_ARTIFACT).build());
    // Search for it based split patterns to make sure nothing is matched
    assertEmpty(mds, SearchRequest.of("-").build());
    assertEmpty(mds, SearchRequest.of(",").build());
    assertEmpty(mds, SearchRequest.of("_").build());
    assertEmpty(mds, SearchRequest.of(", ,").build());
    assertEmpty(mds, SearchRequest.of(", - ,").build());
    // Search for it based on a word in value
    assertResults(mds, SearchRequest.of("av5").addType(TYPE_PROGRAM).build(), programRecord);
    // Case insensitive
    assertResults(mds, SearchRequest.of("ValUe1").addType(TYPE_PROGRAM).build(), programRecord);
    // add a property for the program
    mds.apply(new Update(program, new Metadata(SYSTEM, props("key3", "value1"))), MutationOptions.DEFAULT);
    programRecord = new MetadataRecord(program, new Metadata(ImmutableSet.of(), ImmutableMap.of(new ScopedName(USER, "key1"), "value1", new ScopedName(USER, "key2"), "value2", new ScopedName(USER, "multiword"), multiWordValue, new ScopedName(SYSTEM, "key3"), "value1")));
    // search by value
    assertResults(mds, SearchRequest.of("value1").addType(TYPE_PROGRAM).build(), programRecord);
    // add a property for the dataset
    MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(USER, props("key21", "value21")));
    mds.apply(new Update(dataset, datasetRecord.getMetadata()), MutationOptions.DEFAULT);
    // Search based on value prefix
    assertResults(mds, SearchRequest.of("value2*").build(), programRecord, datasetRecord);
    // Search based on value prefix in the wrong namespace
    assertEmpty(mds, SearchRequest.of("value2*").addNamespace("ns12").build());
    // clean up
    mds.batch(batch(new Drop(program), new Drop(dataset)), MutationOptions.DEFAULT);
}

Example 68 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataStorageTest method testSearchOnTags.

@Test
public void testSearchOnTags() throws Exception {
    MetadataStorage mds = getMetadataStorage();
    String ns1 = "ns1";
    String ns2 = "ns2";
    MetadataEntity app1 = ofApp(ns1, "app1");
    MetadataEntity app2 = ofApp(ns2, "app1");
    MetadataEntity program1 = ofWorker(app1, "wk1");
    MetadataEntity dataset1 = ofDataset(ns1, "ds1");
    MetadataEntity dataset2 = ofDataset(ns1, "ds2");
    MetadataEntity file1 = MetadataEntity.builder(dataset1).appendAsType("file", "f1").build();
    List<MetadataEntity> entities = ImmutableList.of(app1, app2, program1, dataset1, dataset2, file1);
    for (MetadataEntity entity : entities) {
        Assert.assertEquals(Metadata.EMPTY, mds.read(new Read(entity)));
    }
    // add tags for these entities
    MetadataRecord app1Record = new MetadataRecord(app1, new Metadata(USER, tags("tag1", "tag2", "tag3")));
    MetadataRecord app2Record = new MetadataRecord(app2, new Metadata(USER, tags("tag1", "tag2", "tag3_more")));
    MetadataRecord program1Record = new MetadataRecord(program1, new Metadata(USER, tags("tag1")));
    MetadataRecord dataset1Record = new MetadataRecord(dataset1, new Metadata(USER, tags("tag3", "tag2", "tag12-tag33")));
    MetadataRecord dataset2Record = new MetadataRecord(dataset2, new Metadata(USER, tags("tag2", "tag4")));
    MetadataRecord file1Record = new MetadataRecord(file1, new Metadata(USER, tags("tag2", "tag5")));
    mds.batch(ImmutableList.of(app1Record, app2Record, program1Record, dataset1Record, dataset2Record, file1Record).stream().map(record -> new Update(record.getEntity(), record.getMetadata())).collect(Collectors.toList()), MutationOptions.DEFAULT);
    // Try to search on all tags
    assertResults(mds, SearchRequest.of("tags:*").addNamespace(ns1).build(), app1Record, program1Record, dataset1Record, dataset2Record, file1Record);
    // Try to search for tag1*
    assertResults(mds, SearchRequest.of("tags:tag1*").addNamespace(ns1).build(), app1Record, program1Record, dataset1Record);
    // Try to search for tag1 with spaces in search query and mixed case of tags keyword
    assertResults(mds, SearchRequest.of("  tAGS  :  tag1  ").addNamespace(ns1).build(), app1Record, program1Record);
    // Try to search for tag5
    assertResults(mds, SearchRequest.of("tags:tag5").addNamespace(ns1).build(), file1Record);
    // Try to search for tag2
    assertResults(mds, SearchRequest.of("tags:tag2").addNamespace(ns1).build(), app1Record, dataset1Record, dataset2Record, file1Record);
    // Try to search for tag4
    assertResults(mds, SearchRequest.of("tags:tag4").addNamespace(ns1).build(), dataset2Record);
    // Try to search for tag33
    assertResults(mds, SearchRequest.of("tags:tag33").addNamespace(ns1).build(), dataset1Record);
    // Try to search for a tag which has - in it
    assertResults(mds, SearchRequest.of("tag12-tag33").addNamespace(ns1).build(), dataset1Record);
    // Try to search for tag33 with spaces in query
    assertResults(mds, SearchRequest.of("  tag33  ").addNamespace(ns1).build(), dataset1Record);
    // Try wildcard search for tag3*
    assertResults(mds, SearchRequest.of("tags:tag3*").addNamespace(ns1).build(), app1Record, dataset1Record);
    // try search in another namespace
    assertResults(mds, SearchRequest.of("tags:tag1").addNamespace(ns2).build(), app2Record);
    assertResults(mds, SearchRequest.of("tag3").addNamespace(ns2).build(), app2Record);
    assertResults(mds, SearchRequest.of("tag*").addNamespace(ns2).build(), app2Record);
    // try to search across namespaces
    assertResults(mds, SearchRequest.of("tags:tag1").build(), app1Record, app2Record, program1Record);
    // cleanup
    mds.batch(entities.stream().map(Drop::new).collect(Collectors.toList()), MutationOptions.DEFAULT);
    // Search should be empty after deleting tags
    assertEmpty(mds, SearchRequest.of("*").setLimit(10).build());
}

Example 69 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class AuditMessageTypeAdapter method deserialize.

@Override
public AuditMessage deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = json.getAsJsonObject();
    long timeMillis = jsonObj.get("time").getAsLong();
    MetadataEntity metadataEntity;
    EntityId entityId = context.deserialize(jsonObj.getAsJsonObject("entityId"), EntityId.class);
    if (entityId != null) {
        metadataEntity = entityId.toMetadataEntity();
    } else {
        metadataEntity = context.deserialize(jsonObj.getAsJsonObject("metadataEntity"), MetadataEntity.class);
    }
    String user = jsonObj.get("user").getAsString();
    AuditType auditType = context.deserialize(jsonObj.getAsJsonPrimitive("type"), AuditType.class);
    AuditPayload payload;
    JsonObject jsonPayload = jsonObj.getAsJsonObject("payload");
    switch(auditType) {
        case METADATA_CHANGE:
            payload = context.deserialize(jsonPayload, MetadataPayload.class);
            break;
        case ACCESS:
            payload = context.deserialize(jsonPayload, AccessPayload.class);
            break;
        default:
            payload = AuditPayload.EMPTY_PAYLOAD;
    }
    return new AuditMessage(timeMillis, metadataEntity, user, auditType, payload);
}

Example 70 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataHttpHandler method addTags.

@POST
@Path("/**/metadata/tags")
@AuditPolicy(AuditDetail.REQUEST_BODY)
public void addTags(FullHttpRequest request, HttpResponder responder, @QueryParam("type") String type, @QueryParam("async") @DefaultValue("false") Boolean async) throws Exception {
    MetadataEntity metadataEntity = getMetadataEntityFromPath(request.uri(), type, "/metadata/tags");
    enforce(metadataEntity, StandardPermission.UPDATE);
    metadataAdmin.addTags(metadataEntity, readTags(request), async ? ASYNC : SYNC);
    responder.sendString(HttpResponseStatus.OK, String.format("Metadata tags for %s added successfully.", metadataEntity));
}