Examples with MetadataEntity io.cdap.cdap.api.metadata.MetadataEntity used on opensource projects

Example 96 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataHttpHandlerTestRun method testSearchMetadata.

@Test
public void testSearchMetadata() throws Exception {
    appClient.deploy(NamespaceId.DEFAULT, createAppJarFile(AllProgramsApp.class));
    // wait for the system metadata to be processed
    ApplicationId appId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    DatasetId datasetId = NamespaceId.DEFAULT.dataset(AllProgramsApp.DATASET_NAME);
    Tasks.waitFor(false, () -> getProperties(appId, MetadataScope.SYSTEM).isEmpty(), 10, TimeUnit.SECONDS);
    Tasks.waitFor(false, () -> getProperties(datasetId, MetadataScope.SYSTEM).isEmpty(), 10, TimeUnit.SECONDS);
    Map<NamespacedEntityId, Metadata> expectedUserMetadata = new HashMap<>();
    // Add metadata to app
    Map<String, String> props = ImmutableMap.of("key1", "value1");
    Set<String> tags = ImmutableSet.of("tag1", "tag2");
    addProperties(appId, props);
    addTags(appId, tags);
    expectedUserMetadata.put(appId, new Metadata(props, tags));
    // Add metadata to dataset
    props = ImmutableMap.of("key10", "value10", "key11", "value11");
    tags = ImmutableSet.of("tag11");
    addProperties(datasetId, props);
    addTags(datasetId, tags);
    expectedUserMetadata.put(datasetId, new Metadata(props, tags));
    Set<MetadataSearchResultRecord> results = searchMetadata(NamespaceId.DEFAULT, "value*").getResults();
    // Verify results
    Assert.assertEquals(expectedUserMetadata.keySet(), extractEntityIds(results));
    for (MetadataSearchResultRecord result : results) {
        // User metadata has to match exactly since we know what we have set
        Assert.assertEquals(expectedUserMetadata.get(result.getEntityId()), result.getMetadata().get(MetadataScope.USER));
        // Make sure system metadata is returned, we cannot check for exact match since we haven't set it
        Metadata systemMetadata = result.getMetadata().get(MetadataScope.SYSTEM);
        Assert.assertNotNull(systemMetadata);
        Assert.assertFalse(systemMetadata.getProperties().isEmpty());
        Assert.assertFalse(systemMetadata.getTags().isEmpty());
    }
    // add metadata to field (custom entity)
    props = ImmutableMap.of("fKey1", "fValue1", "fKey2", "fValue2");
    tags = ImmutableSet.of("fTag1");
    MetadataEntity metadataEntity = MetadataEntity.builder(datasetId.toMetadataEntity()).appendAsType("field", "someField").build();
    addProperties(metadataEntity, props);
    addTags(metadataEntity, tags);
    Map<MetadataEntity, Metadata> expectedUserMetadataV2 = new HashMap<>();
    expectedUserMetadataV2.put(metadataEntity, new Metadata(props, tags));
    Set<MetadataSearchResultRecord> resultsV2 = super.searchMetadata(ImmutableList.of(NamespaceId.DEFAULT), "fValue*", ImmutableSet.of(), null, 0, Integer.MAX_VALUE, 0, null, false).getResults();
    // Verify results
    Assert.assertEquals(expectedUserMetadataV2.keySet(), ImmutableSet.copyOf(extractMetadataEntities(resultsV2)));
    for (MetadataSearchResultRecord result : resultsV2) {
        // User metadata has to match exactly since we know what we have set
        Assert.assertEquals(expectedUserMetadataV2.get(result.getMetadataEntity()), result.getMetadata().get(MetadataScope.USER));
        // Make sure system metadata is returned, we cannot check for exact match since we haven't set it
        Metadata systemMetadata = result.getMetadata().get(MetadataScope.SYSTEM);
        // custom entity should not have any system metadata for it
        Assert.assertNull(systemMetadata);
    }
}

Example 97 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class MetadataDataset method parseRow.

// there may not be a MetadataEntry in the row or it may for a different targetType (entityFilter),
// so return an Optional
private Optional<MetadataEntry> parseRow(Row rowToProcess, String indexColumn, Set<String> entityFilter, boolean showHidden) {
    String rowValue = rowToProcess.getString(indexColumn);
    if (rowValue == null) {
        return Optional.empty();
    }
    final byte[] rowKey = rowToProcess.getRow();
    String targetType = MetadataKey.extractTargetType(rowKey);
    // Filter on target type if not set to include all types
    if (!entityFilter.isEmpty() && !entityFilter.contains(targetType)) {
        return Optional.empty();
    }
    MetadataEntity metadataEntity = MetadataKey.extractMetadataEntityFromKey(rowKey);
    try {
        NamespacedEntityId namespacedEntityId = EntityId.fromMetadataEntity(metadataEntity);
        // if the entity starts with _ then skip it unless the caller choose to showHidden.
        if (!showHidden && namespacedEntityId != null && namespacedEntityId.getEntityName().startsWith("_")) {
            return Optional.empty();
        }
    } catch (IllegalArgumentException e) {
    // ignore. For custom entities we don't really want to hide them if they start with _
    }
    String key = MetadataKey.extractMetadataKey(rowKey);
    MetadataEntry entry = getMetadata(metadataEntity, key);
    return Optional.ofNullable(entry);
}

Example 98 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataDocumentTest method testInvalidSchema.

@Test
public void testInvalidSchema() {
    Schema schema = Schema.recordOf("mystruct", Schema.Field.of("x", Schema.of(Schema.Type.STRING)), Schema.Field.of("y", Schema.of(Schema.Type.INT)), Schema.Field.of("z", Schema.of(Schema.Type.DOUBLE)));
    String schemaString = schema.toString();
    MetadataEntity entity = MetadataEntity.ofDataset("ds");
    String[] results = MetadataDocument.Builder.parseSchema(entity, schemaString).split(" ");
    String[] expected = { "mystruct", "mystruct:RECORD", "x", "x:STRING", "y", "y:INT", "z", "z:DOUBLE" };
    Assert.assertArrayEquals(expected, results);
    String schemaWithInvalidTypes = schemaString.replace("string", "nosuchtype");
    Assert.assertEquals(schemaWithInvalidTypes, MetadataDocument.Builder.parseSchema(entity, schemaWithInvalidTypes));
    String truncatedSchema = schemaString.substring(10, schemaString.length() - 10);
    Assert.assertEquals(truncatedSchema, MetadataDocument.Builder.parseSchema(entity, truncatedSchema));
}

Example 99 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class DataPipelineConnectionTest method assertMetadataSearch.

private void assertMetadataSearch(Set<MetadataEntity> appsRelated, String query) throws Exception {
    SearchResponse search = getMetadataAdmin().search(SearchRequest.of(query).build());
    Set<MetadataEntity> actual = search.getResults().stream().map(record -> record.getEntity()).collect(Collectors.toSet());
    Assert.assertEquals(appsRelated, actual);
}

Example 100 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataDatasetTest method testSearchIncludesSystemEntities.

@Test
public void testSearchIncludesSystemEntities() throws InterruptedException, TransactionFailureException {
    // Use the same artifact in two different namespaces - system and ns2
    final MetadataEntity sysArtifact = NamespaceId.SYSTEM.artifact("artifact", "1.0").toMetadataEntity();
    final MetadataEntity ns2Artifact = new ArtifactId("ns2", "artifact", "1.0").toMetadataEntity();
    final String multiWordKey = "multiword";
    final String multiWordValue = "aV1 av2 ,  -  ,  av3 - av4_av5 av6";
    txnl.execute(() -> {
        dataset.addProperty(program1, multiWordKey, multiWordValue);
        dataset.addProperty(sysArtifact, multiWordKey, multiWordValue);
        dataset.addProperty(ns2Artifact, multiWordKey, multiWordValue);
    });
    // perform the exact same multiword search in the 'ns1' namespace. It should return the system artifact along with
    // matched entities in the 'ns1' namespace
    final MetadataEntry flowMultiWordEntry = new MetadataEntry(program1, multiWordKey, multiWordValue);
    final MetadataEntry systemArtifactEntry = new MetadataEntry(sysArtifact, multiWordKey, multiWordValue);
    final MetadataEntry ns2ArtifactEntry = new MetadataEntry(ns2Artifact, multiWordKey, multiWordValue);
    txnl.execute(() -> {
        List<MetadataEntry> results = searchByDefaultIndex("ns1", "aV5", ALL_TYPES);
        Assert.assertEquals(Sets.newHashSet(flowMultiWordEntry, systemArtifactEntry), Sets.newHashSet(results));
        // search only programs - should only return flow
        results = searchByDefaultIndex("ns1", multiWordKey + MetadataConstants.KEYVALUE_SEPARATOR + "aV5", ImmutableSet.of(MetadataEntity.PROGRAM));
        Assert.assertEquals(ImmutableList.of(flowMultiWordEntry), results);
        // search only artifacts - should only return system artifact
        results = searchByDefaultIndex("ns1", multiWordKey + MetadataConstants.KEYVALUE_SEPARATOR + multiWordValue, ImmutableSet.of(MetadataEntity.ARTIFACT));
        // this query returns the system artifact 4 times, since the dataset returns a list with duplicates for scoring
        // purposes. Convert to a Set for comparison.
        Assert.assertEquals(Sets.newHashSet(systemArtifactEntry), Sets.newHashSet(results));
        // search all entities in namespace 'ns2' - should return the system artifact and the same artifact in ns2
        results = searchByDefaultIndex("ns2", multiWordKey + MetadataConstants.KEYVALUE_SEPARATOR + "aV4", ALL_TYPES);
        Assert.assertEquals(Sets.newHashSet(systemArtifactEntry, ns2ArtifactEntry), Sets.newHashSet(results));
        // search only programs in a namespace 'ns2'. Should return empty
        results = searchByDefaultIndex("ns2", "aV*", ImmutableSet.of(MetadataEntity.PROGRAM));
        Assert.assertTrue(results.isEmpty());
        // search all entities in namespace 'ns3'. Should return only the system artifact
        results = searchByDefaultIndex("ns3", "av*", ALL_TYPES);
        Assert.assertEquals(Sets.newHashSet(systemArtifactEntry), Sets.newHashSet(results));
        // search the system namespace for all entities. Should return only the system artifact
        results = searchByDefaultIndex(NamespaceId.SYSTEM.getEntityName(), "av*", ALL_TYPES);
        Assert.assertEquals(Sets.newHashSet(systemArtifactEntry), Sets.newHashSet(results));
    });
    // clean up
    txnl.execute(() -> {
        dataset.removeProperties(program1);
        dataset.removeProperties(sysArtifact);
    });
}