Examples with TAG io.cdap.cdap.spi.metadata.MetadataKind.TAG used on opensource projects

Example 1 with TAG

use of io.cdap.cdap.spi.metadata.MetadataKind.TAG in project cdap by caskdata.

the class DatasetMetadataStorage method replaceInScope.

private MetadataDataset.Change replaceInScope(MetadataDatasetContext context, MetadataScope scope, MetadataEntity entity, Set<String> newTags, Map<String, String> newProperties, Map<ScopedNameOfKind, MetadataDirective> directives) {
    MetadataDataset dataset = context.getDataset(scope);
    MetadataDataset.Record before = dataset.getMetadata(entity);
    if (newTags.isEmpty() && newProperties.isEmpty()) {
        // this scope remains unchanged
        return new MetadataDataset.Change(before, before);
    }
    Set<String> existingTags = before.getTags();
    Set<String> tagsToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == TAG && (entry.getValue() == MetadataDirective.KEEP || entry.getValue() == MetadataDirective.PRESERVE)).map(Map.Entry::getKey).map(ScopedName::getName).filter(existingTags::contains).collect(Collectors.toSet());
    newTags = Sets.union(newTags, tagsToKeepOrPreserve);
    Map<String, String> existingProperties = before.getProperties();
    Map<String, String> propertiesToKeepOrPreserve = directives.entrySet().stream().filter(entry -> entry.getKey().getScope() == scope && entry.getKey().getKind() == PROPERTY).filter(entry -> existingProperties.containsKey(entry.getKey().getName())).filter(entry -> entry.getValue() == MetadataDirective.PRESERVE || entry.getValue() == MetadataDirective.KEEP && !newProperties.containsKey(entry.getKey().getName())).map(Map.Entry::getKey).map(ScopedName::getName).collect(Collectors.toMap(name -> name, existingProperties::get));
    newProperties.putAll(propertiesToKeepOrPreserve);
    Set<String> tagsToRemove = Sets.difference(before.getTags(), newTags);
    Set<String> tagsToAdd = Sets.difference(newTags, before.getTags());
    Set<String> propertiesToRemove = Sets.difference(before.getProperties().keySet(), newProperties.keySet());
    @SuppressWarnings("ConstantConditions") Map<String, String> propertiesToAdd = Maps.filterEntries(newProperties, entry -> !entry.getValue().equals(existingProperties.get(entry.getKey())));
    MetadataDataset.Record after = before;
    if (!tagsToRemove.isEmpty()) {
        after = dataset.removeTags(entity, tagsToRemove).getLatest();
    }
    if (!tagsToAdd.isEmpty()) {
        after = dataset.addTags(entity, tagsToAdd).getLatest();
    }
    if (!propertiesToRemove.isEmpty()) {
        after = dataset.removeProperties(entity, propertiesToRemove).getLatest();
    }
    if (!propertiesToAdd.isEmpty()) {
        after = dataset.addProperties(entity, propertiesToAdd).getLatest();
    }
    return new MetadataDataset.Change(before, after);
}

Example 2 with TAG

use of io.cdap.cdap.spi.metadata.MetadataKind.TAG in project cdap by caskdata.

the class DatasetMetadataStorage method readScope.

private MetadataDataset.Record readScope(MetadataDatasetContext context, MetadataScope scope, Read read) {
    MetadataEntity entity = read.getEntity();
    if (read.getSelection() == null && (!read.getScopes().contains(scope) || read.getKinds().isEmpty())) {
        return new MetadataDataset.Record(entity);
    }
    Set<ScopedNameOfKind> selectionForScope = null;
    if (read.getSelection() != null) {
        // noinspection ConstantConditions
        selectionForScope = Sets.filter(read.getSelection(), entry -> entry.getScope() == scope);
        if (selectionForScope.isEmpty()) {
            return new MetadataDataset.Record(entity);
        }
    }
    // now we know we must read from the dataset
    MetadataDataset dataset = context.getDataset(scope);
    if (selectionForScope != null) {
        // request is for a specific set of tags and properties
        Set<String> tagsToRead = selectionForScope.stream().filter(entry -> TAG == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
        Set<String> propertiesToRead = selectionForScope.stream().filter(entry -> PROPERTY == entry.getKind()).map(ScopedName::getName).collect(Collectors.toSet());
        Set<String> tags = tagsToRead.isEmpty() ? Collections.emptySet() : Sets.intersection(tagsToRead, dataset.getTags(entity));
        Map<String, String> properties = propertiesToRead.isEmpty() ? Collections.emptyMap() : Maps.filterKeys(dataset.getProperties(entity), propertiesToRead::contains);
        return new MetadataDataset.Record(entity, properties, tags);
    }
    if (MetadataKind.ALL.equals(read.getKinds())) {
        // all metadata kinds requested
        return dataset.getMetadata(entity);
    }
    // exactly one kind is requested
    MetadataKind requestKind = read.getKinds().iterator().next();
    if (requestKind == TAG) {
        return new MetadataDataset.Record(entity, Collections.emptyMap(), dataset.getTags(entity));
    }
    if (requestKind == PROPERTY) {
        return new MetadataDataset.Record(entity, dataset.getProperties(entity), Collections.emptySet());
    }
    throw new IllegalStateException("Encountered metadata read request for unknown kind " + requestKind);
}

Example 3 with TAG

use of io.cdap.cdap.spi.metadata.MetadataKind.TAG in project cdap by caskdata.

the class MetadataStorageTest method testBatchConcurrency.

@Test
public void testBatchConcurrency() throws IOException {
    // T threads
    int numThreads = 10;
    // N entities
    int numEntities = 20;
    MetadataStorage mds = getMetadataStorage();
    ExecutorService executor = Executors.newFixedThreadPool(numThreads);
    CompletionService<List<MetadataChange>> completionService = new ExecutorCompletionService<>(executor);
    // Set up N entities with T tags (one to be removed per thread)
    Set<String> allRTags = IntStream.range(0, numThreads).mapToObj(t -> "r" + t).collect(Collectors.toSet());
    Map<Integer, MetadataEntity> entities = IntStream.range(0, numEntities).boxed().collect(Collectors.toMap(i -> i, i -> MetadataEntity.ofDataset("myds" + i)));
    mds.batch(entities.values().stream().map(entity -> new Update(entity, new Metadata(USER, allRTags))).collect(Collectors.toList()), MutationOptions.DEFAULT);
    // Generate a random but conflicting set of batches of mutations, one for each thread.
    // Construct the expected results for each entity along with the mutations
    // Also, because some threads will perform a Create, create a set of directives to preserve all other tags
    Map<Integer, Set<String>> expected = IntStream.range(0, numEntities).boxed().collect(Collectors.toMap(i -> i, i -> new HashSet<>(allRTags)));
    Map<Integer, List<MetadataMutation>> mutations = IntStream.range(0, numThreads).boxed().collect(Collectors.toMap(i -> i, i -> new ArrayList<>()));
    Map<ScopedNameOfKind, MetadataDirective> directives = new HashMap<>();
    Random rand = new Random(System.currentTimeMillis());
    IntStream.range(0, numThreads).forEach(t -> {
        directives.put(new ScopedNameOfKind(TAG, USER, "t" + t), MetadataDirective.KEEP);
        directives.put(new ScopedNameOfKind(TAG, USER, "r" + t), MetadataDirective.KEEP);
        directives.put(new ScopedNameOfKind(TAG, USER, "c" + t), MetadataDirective.KEEP);
        IntStream.range(0, numEntities).forEach(e -> {
            int random = rand.nextInt(100);
            if (random < 30) {
                expected.get(e).add("t" + t);
                mutations.get(t).add(new Update(entities.get(e), new Metadata(USER, tags("t" + t))));
            } else if (random < 60) {
                expected.get(e).add("c" + t);
                mutations.get(t).add(new Create(entities.get(e), new Metadata(USER, tags("c" + t)), directives));
            } else if (random < 90) {
                expected.get(e).remove("r" + t);
                mutations.get(t).add(new Remove(entities.get(e), Collections.singleton(new ScopedNameOfKind(TAG, USER, "r" + t))));
            }
        });
    });
    // submit all tasks and wait for their completion
    IntStream.range(0, numThreads).forEach(t -> completionService.submit(() -> mds.batch(mutations.get(t), MutationOptions.DEFAULT)));
    IntStream.range(0, numThreads).forEach(t -> {
        try {
            completionService.take();
        } catch (InterruptedException e) {
            throw Throwables.propagate(e);
        }
    });
    // validate that all "r" tags were removed and all "c" and "t" tags were added
    IntStream.range(0, numEntities).forEach(e -> {
        try {
            Assert.assertEquals("For entity " + entities.get(e), expected.get(e), mds.read(new Read(entities.get(e))).getTags(USER));
        } catch (Exception ex) {
            throw Throwables.propagate(ex);
        }
    });
    // clean up
    mds.batch(entities.values().stream().map(Drop::new).collect(Collectors.toList()), MutationOptions.DEFAULT);
}

Example 4 with TAG

use of io.cdap.cdap.spi.metadata.MetadataKind.TAG in project cdap by caskdata.

the class MetadataStorageTest method testUpdateDropConflictInBatch.

/**
 * See {@link #testUpdateDropConflict()} for a description. The difference in this test is that
 * we issue batches of mutations over a collection of entities. The same assumptions apply,
 * however, for each entity.
 */
@Test
@Category(SlowTests.class)
public void testUpdateDropConflictInBatch() throws IOException {
    int numTests = 10;
    int numThreads = 2;
    int numEntities = 20;
    MetadataStorage mds = getMetadataStorage();
    ExecutorService executor = Executors.newFixedThreadPool(numThreads);
    CompletionService<List<MetadataChange>> completionService = new ExecutorCompletionService<>(executor);
    Map<Integer, MetadataEntity> entities = IntStream.range(0, numEntities).boxed().collect(Collectors.toMap(i -> i, i -> MetadataEntity.ofDataset("myds" + i)));
    List<MetadataMutation> creates = entities.values().stream().map(e -> new Create(e, new Metadata(USER, tags("a")), Collections.emptyMap())).collect(Collectors.toList());
    Random rand = new Random(System.currentTimeMillis());
    IntStream.range(0, numTests).forEach(x -> {
        try {
            mds.batch(creates, MutationOptions.DEFAULT);
            Map<Integer, List<MetadataMutation>> mutations = IntStream.range(0, numThreads).boxed().collect(Collectors.toMap(i -> i, i -> new ArrayList<>()));
            IntStream.range(0, numEntities).forEach(e -> {
                // ensure that at least one thread attempts to drop this entity
                int dropThread = rand.nextInt(numThreads);
                IntStream.range(0, numThreads).forEach(t -> {
                    if (t == dropThread || rand.nextInt(100) < 50) {
                        mutations.get(t).add(new Drop(entities.get(e)));
                    } else {
                        mutations.get(t).add(new Update(entities.get(e), new Metadata(USER, tags("b"))));
                    }
                });
            });
            IntStream.range(0, numThreads).forEach(t -> completionService.submit(() -> mds.batch(mutations.get(t), MutationOptions.DEFAULT)));
            IntStream.range(0, numThreads).forEach(t -> {
                try {
                    completionService.take();
                } catch (InterruptedException e) {
                    throw Throwables.propagate(e);
                }
            });
            IntStream.range(0, numEntities).forEach(e -> {
                try {
                    // each entity is either dropped then updated (and then it has tag "b" only)
                    // or it first update and then dropped (and then it has empty metadata)
                    Assert.assertTrue(ImmutableSet.of(Metadata.EMPTY, new Metadata(USER, tags("b"))).contains(mds.read(new Read(entities.get(e)))));
                } catch (Exception ex) {
                    throw Throwables.propagate(ex);
                }
            });
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    });
    mds.batch(entities.values().stream().map(Drop::new).collect(Collectors.toList()), MutationOptions.DEFAULT);
}

Example 5 with TAG

use of io.cdap.cdap.spi.metadata.MetadataKind.TAG in project cdap by caskdata.

the class MetadataStorageTest method testBatch.

@Test
public void testBatch() throws IOException {
    MetadataEntity entity = MetadataEntity.ofDataset("a", "b");
    Map<ScopedNameOfKind, MetadataDirective> directives = ImmutableMap.of(new ScopedNameOfKind(PROPERTY, SYSTEM, CREATION_TIME_KEY), MetadataDirective.PRESERVE, new ScopedNameOfKind(PROPERTY, SYSTEM, DESCRIPTION_KEY), MetadataDirective.KEEP);
    MetadataStorage mds = getMetadataStorage();
    Create create = new Create(entity, new Metadata(SYSTEM, tags("batch"), props(CREATION_TIME_KEY, "12345678", DESCRIPTION_KEY, "hello", "other", "value")), directives);
    MetadataChange change = mds.apply(create, MutationOptions.DEFAULT);
    Assert.assertEquals(Metadata.EMPTY, change.getBefore());
    Assert.assertEquals(create.getMetadata(), change.getAfter());
    List<MetadataMutation> mutations = ImmutableList.of(new Update(entity, new Metadata(USER, tags("tag1", "tag2"))), new Drop(entity), new Create(entity, new Metadata(SYSTEM, tags("batch"), props(CREATION_TIME_KEY, "23456789", "other", "different")), directives), new Update(entity, new Metadata(USER, tags("tag3"), props("key", "value"))), new Remove(entity, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, SYSTEM, "other"), new ScopedNameOfKind(TAG, USER, "tag2"))), new Create(entity, new Metadata(SYSTEM, tags("realtime"), props(CREATION_TIME_KEY, "33456789", DESCRIPTION_KEY, "new description", "other", "yet other")), directives));
    // apply all mutations in sequence
    List<MetadataChange> changes = mutations.stream().map(mutation -> {
        try {
            return mds.apply(mutation, MutationOptions.DEFAULT);
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }).collect(Collectors.toList());
    // drop and recreate the entity
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
    change = mds.apply(create, MutationOptions.DEFAULT);
    Assert.assertEquals(Metadata.EMPTY, change.getBefore());
    Assert.assertEquals(create.getMetadata(), change.getAfter());
    // apply all mutations in batch
    List<MetadataChange> batchChanges = mds.batch(mutations, MutationOptions.DEFAULT);
    // make sure the same mutations were applied
    Assert.assertEquals(changes, batchChanges);
    // clean up
    mds.apply(new Drop(entity), MutationOptions.DEFAULT);
}