Examples with MetadataOperation io.cdap.cdap.data2.metadata.writer.MetadataOperation used on opensource projects

Example 1 with MetadataOperation

use of io.cdap.cdap.data2.metadata.writer.MetadataOperation in project cdap by caskdata.

the class DataPipelineTest method testMetadata.

@Test
public void testMetadata() throws Exception {
    ImmutableSet<String> inputTagsToAdd = ImmutableSet.of("tOne", "tTwo");
    ImmutableMap<String, String> inputPropToAdd = ImmutableMap.of("kOne", "vOne", "kTwo", "vTwo");
    MetadataOperation op = new MetadataOperation.Put(MetadataEntity.ofDataset(NamespaceId.DEFAULT.getNamespace(), "singleInput"), inputPropToAdd, inputTagsToAdd);
    Set<MetadataOperation> operations = new HashSet<>(Collections.singletonList(op));
    // run pipeline with the metadata operations which need to be performed
    MetadataAdmin metadataAdmin = getMetadataAdmin();
    runPipelineForMetadata(metadataAdmin, operations);
    waitForMetadataProcessing(metadataAdmin, 2);
    // verify metadata written by the pipeline
    Metadata actual = metadataAdmin.getMetadata(MetadataEntity.ofDataset(NamespaceId.DEFAULT.getNamespace(), "singleInput"), MetadataScope.USER);
    Assert.assertNotNull(actual);
    Assert.assertTrue(!actual.isEmpty());
    // verify the user properties
    Assert.assertEquals(inputPropToAdd, actual.getProperties(MetadataScope.USER));
    Assert.assertEquals(inputTagsToAdd, actual.getTags(MetadataScope.USER));
    // delete some properties and tag
    op = new MetadataOperation.Delete(MetadataEntity.ofDataset(NamespaceId.DEFAULT.getNamespace(), "singleInput"), ImmutableSet.of("kOne"), ImmutableSet.of("tOne"));
    operations = new HashSet<>(Collections.singleton(op));
    runPipelineForMetadata(metadataAdmin, operations);
    waitForMetadataProcessing(metadataAdmin, 1);
    actual = metadataAdmin.getMetadata(MetadataEntity.ofDataset(NamespaceId.DEFAULT.getNamespace(), "singleInput"), MetadataScope.USER);
    Assert.assertNotNull(actual);
    Assert.assertTrue(!actual.isEmpty());
    // verify the user properties
    Assert.assertEquals(Collections.singletonMap("kTwo", "vTwo"), actual.getProperties(MetadataScope.USER));
    Assert.assertEquals(Collections.singleton("tTwo"), actual.getTags(MetadataScope.USER));
}

Example 2 with MetadataOperation

use of io.cdap.cdap.data2.metadata.writer.MetadataOperation in project cdap by caskdata.

the class MockSource method processsMetadata.

/**
 * Processes metadata operations
 */
private void processsMetadata(BatchSourceContext context) throws MetadataException {
    MetadataEntity metadataEntity = MetadataEntity.ofDataset(context.getNamespace(), config.connectionConfig.tableName);
    Map<MetadataScope, Metadata> currentMetadata = context.getMetadata(metadataEntity);
    Set<MetadataOperation> operations = GSON.fromJson(config.metadataOperations, SET_METADATA_OPERATION_TYPE);
    // must be to fetch metadata and there should be system metadata
    if (currentMetadata.get(MetadataScope.SYSTEM).getProperties().isEmpty() || currentMetadata.get(MetadataScope.SYSTEM).getProperties().isEmpty()) {
        throw new IllegalArgumentException(String.format("System properties or tags for '%s' is empty. " + "Expected to have system metadata.", metadataEntity));
    }
    LOG.trace("Metadata operations {} will be applied. Current Metadata Record is {}", operations, currentMetadata);
    // noinspection ConstantConditions
    for (MetadataOperation curOperation : operations) {
        switch(curOperation.getType()) {
            case PUT:
                // noinspection ConstantConditions
                context.addTags(curOperation.getEntity(), ((MetadataOperation.Put) curOperation).getTags());
                context.addProperties(curOperation.getEntity(), ((MetadataOperation.Put) curOperation).getProperties());
                break;
            case DELETE:
                // noinspection ConstantConditions
                context.removeTags(curOperation.getEntity(), ((MetadataOperation.Delete) curOperation).getTags().toArray(new String[0]));
                context.removeProperties(curOperation.getEntity(), ((MetadataOperation.Delete) curOperation).getProperties().toArray(new String[0]));
                break;
            case DELETE_ALL:
                context.removeMetadata(curOperation.getEntity());
                break;
            case DELETE_ALL_TAGS:
                context.removeTags(curOperation.getEntity());
                break;
            case DELETE_ALL_PROPERTIES:
                context.removeProperties(curOperation.getEntity());
                break;
            default:
                throw new IllegalArgumentException(String.format("Invalid metadata operation '%s'", curOperation.getType()));
        }
    }
}