Examples with MetadataEntity io.cdap.cdap.api.metadata.MetadataEntity used on opensource projects

Example 86 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class EntityIdTest method testFromMetadataEntity.

@Test
public void testFromMetadataEntity() {
    ApplicationId applicationId = new ApplicationId("testNs", "app1");
    Assert.assertEquals(applicationId, EntityId.fromMetadataEntity(applicationId.toMetadataEntity()));
    MetadataEntity metadataEntity = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "testNs").appendAsType(MetadataEntity.APPLICATION, "app1").build();
    Assert.assertEquals(applicationId, EntityId.fromMetadataEntity(metadataEntity));
    metadataEntity = MetadataEntity.builder(MetadataEntity.ofDataset("testNs", "testDs")).appendAsType("field", "testField").build();
    try {
        EntityId.fromMetadataEntity(metadataEntity);
        Assert.fail("Should have failed to get create an EntityId from MetadataEntity");
    } catch (IllegalArgumentException e) {
    // expected
    }
    ProgramId programId = new ProgramId(applicationId, ProgramType.WORKER, "testWorker");
    Assert.assertEquals(programId, EntityId.fromMetadataEntity(programId.toMetadataEntity()));
    // artifact
    ArtifactId artifactId = new ArtifactId("testNs", "testArtifact-1.0.0.jar");
    Assert.assertEquals(artifactId, EntityId.fromMetadataEntity(artifactId.toMetadataEntity()));
    // schedule
    ScheduleId scheduleId = applicationId.schedule("sch");
    metadataEntity = MetadataEntity.builder().append(MetadataEntity.NAMESPACE, "testNs").append(MetadataEntity.APPLICATION, "app1").appendAsType(MetadataEntity.SCHEDULE, "sch").build();
    Assert.assertEquals(scheduleId, EntityId.fromMetadataEntity(metadataEntity));
}

Example 87 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class EntityId method getSelfOrParentEntityId.

/**
 * Creates a valid known CDAP entity which can be considered as the parent for the MetadataEntity by walking up the
 * key-value hierarchy of the MetadataEntity till a known CDAP {@link EntityType} is found. If the last node itself
 * is known type then that will be considered as the parent.
 *
 * @param metadataEntity whose parent entityId needs to be found
 * @return {@link EntityId} of the given metadataEntity
 * @throws IllegalArgumentException if the metadataEntity does not have any know entity type in it's hierarchy or if
 * it does not have all the required key-value pairs to construct the identified EntityId.
 */
public static <T extends EntityId> T getSelfOrParentEntityId(MetadataEntity metadataEntity) {
    EntityType entityType = findParentType(metadataEntity);
    if (entityType == null) {
        throw new IllegalArgumentException(String.format("No known type found in the hierarchy of %s", metadataEntity));
    }
    List<String> values = new LinkedList<>();
    // get the key-value pair till the known entity-type. Note: for application the version comes after application
    // key-value pair and needs to be included too
    List<MetadataEntity.KeyValue> extractedParts = metadataEntity.head(entityType.toString());
    // if a version was specified extract that else use the default version
    String version = metadataEntity.containsKey(MetadataEntity.VERSION) ? metadataEntity.getValue(MetadataEntity.VERSION) : ApplicationId.DEFAULT_VERSION;
    if (entityType == EntityType.APPLICATION) {
        // if the entity is an application our extractParts will not contain the version info since we extracted till
        // application so append it
        extractedParts.add(new MetadataEntity.KeyValue(MetadataEntity.VERSION, version));
    }
    if (entityType == EntityType.PROGRAM || entityType == EntityType.SCHEDULE || entityType == EntityType.PROGRAM_RUN) {
        // (namespace, application, version) if the version information is not present
        if (!metadataEntity.containsKey(MetadataEntity.VERSION)) {
            extractedParts.add(2, new MetadataEntity.KeyValue(MetadataEntity.VERSION, version));
        }
    }
    // for artifacts get till version (artifacts always have version
    if (entityType == EntityType.ARTIFACT) {
        extractedParts = metadataEntity.head(MetadataEntity.VERSION);
    }
    // for plugins get till plugin name
    if (entityType == EntityType.PLUGIN) {
        extractedParts = metadataEntity.head(MetadataEntity.PLUGIN);
    }
    extractedParts.iterator().forEachRemaining(keyValue -> values.add(keyValue.getValue()));
    return entityType.fromIdParts(values);
}

Example 88 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class DataPipelineConnectionTest method testConnectionsRegistry.

@Test
public void testConnectionsRegistry() throws Exception {
    // source -> sink
    ETLBatchConfig conf1 = ETLBatchConfig.builder().addStage(new ETLStage("source", MockSource.getPluginUsingConnection("conn 1"))).addStage(new ETLStage("sink", MockSink.getPluginUsingConnection("conn 3"))).addConnection("source", "sink").build();
    // 3 sources -> identity -> 2 sinks
    ETLBatchConfig conf2 = ETLBatchConfig.builder().addStage(new ETLStage("src1", MockSource.getPluginUsingConnection("conn 1"))).addStage(new ETLStage("src2", MockSource.getPluginUsingConnection("conn 2"))).addStage(new ETLStage("src3", MockSource.getPluginUsingConnection("conn 3"))).addStage(new ETLStage("sink1", MockSink.getPluginUsingConnection("conn 4"))).addStage(new ETLStage("sink2", MockSink.getPluginUsingConnection("conn 5"))).addStage(new ETLStage("identity", IdentityTransform.getPlugin())).addConnection("src1", "identity").addConnection("src2", "identity").addConnection("src3", "identity").addConnection("identity", "sink1").addConnection("identity", "sink2").build();
    // deploy apps
    AppRequest<ETLBatchConfig> appRequest1 = new AppRequest<>(APP_ARTIFACT, conf1);
    ApplicationId appId1 = NamespaceId.DEFAULT.app("app1");
    ApplicationManager appManager1 = deployApplication(appId1, appRequest1);
    AppRequest<ETLBatchConfig> appRequest2 = new AppRequest<>(APP_ARTIFACT, conf2);
    ApplicationId appId2 = NamespaceId.DEFAULT.app("app2");
    ApplicationManager appManager2 = deployApplication(appId2, appRequest2);
    // Assert metadata
    Metadata app1Actual = getMetadataAdmin().getMetadata(appId1.toMetadataEntity(), MetadataScope.SYSTEM);
    Set<String> app1ExpectedTags = ImmutableSet.of("_conn_1", "_conn_3");
    // here assert actual tags contain all the tags about connections
    Assert.assertTrue(app1Actual.getTags(MetadataScope.SYSTEM).containsAll(app1ExpectedTags));
    // user metadata should be empty
    Assert.assertEquals(Metadata.EMPTY, getMetadataAdmin().getMetadata(appId1.toMetadataEntity(), MetadataScope.USER));
    Metadata app2Actual = getMetadataAdmin().getMetadata(appId2.toMetadataEntity(), MetadataScope.SYSTEM);
    Set<String> app2ExpectedTags = ImmutableSet.of("_conn_1", "_conn_2", "_conn_3", "_conn_4", "_conn_5");
    // here assert actual tags contain all the tags about connections
    Assert.assertTrue(app2Actual.getTags(MetadataScope.SYSTEM).containsAll(app2ExpectedTags));
    // user metadata should be empty
    Assert.assertEquals(Metadata.EMPTY, getMetadataAdmin().getMetadata(appId2.toMetadataEntity(), MetadataScope.USER));
    // using search query to find out the related apps
    Set<MetadataEntity> appsRelated = ImmutableSet.of(appId1.toMetadataEntity(), appId2.toMetadataEntity());
    assertMetadataSearch(appsRelated, "tags:_conn_1");
    assertMetadataSearch(Collections.singleton(appId2.toMetadataEntity()), "tags:_conn_2");
    assertMetadataSearch(appsRelated, "tags:_conn_3");
    assertMetadataSearch(Collections.singleton(appId2.toMetadataEntity()), "tags:_conn_4");
    assertMetadataSearch(Collections.singleton(appId2.toMetadataEntity()), "tags:_conn_5");
}

Example 89 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class GetMetadataCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    MetadataEntity metadataEntity = MetadataCommandHelper.toMetadataEntity(arguments.get(ArgumentName.ENTITY.toString()));
    String scope = arguments.getOptional(ArgumentName.METADATA_SCOPE.toString());
    Set<MetadataRecord> metadata = scope == null ? client.getMetadata(metadataEntity) : client.getMetadata(metadataEntity, MetadataScope.valueOf(scope.toUpperCase()));
    Table table = getTableBuilder().setRows(metadata.stream().map(record -> Lists.newArrayList(record.toString(), Joiner.on("\n").join(record.getTags()), Joiner.on("\n").withKeyValueSeparator(":").join(record.getProperties()), record.getScope().name())).collect(Collectors.toList())).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
}

Example 90 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by cdapio.

the class GetMetadataPropertiesCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    MetadataEntity metadataEntity = MetadataCommandHelper.toMetadataEntity(arguments.get(ArgumentName.ENTITY.toString()));
    String scope = arguments.getOptional(ArgumentName.METADATA_SCOPE.toString());
    Map<String, String> properties = scope == null ? client.getProperties(metadataEntity) : client.getProperties(metadataEntity, MetadataScope.valueOf(scope.toUpperCase()));
    Table table = Table.builder().setHeader("key", "value").setRows(Iterables.transform(properties.entrySet(), new Function<Map.Entry<String, String>, List<String>>() {

        @Nullable
        @Override
        public List<String> apply(@Nullable Map.Entry<String, String> entry) {
            return Lists.newArrayList(entry.getKey(), entry.getValue());
        }
    })).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
}