Examples with Tasks io.cdap.cdap.common.utils.Tasks used on opensource projects

Example 1 with Tasks

use of io.cdap.cdap.common.utils.Tasks in project cdap by caskdata.

the class MetadataStorageTest method testBatchConcurrency.

@Test
public void testBatchConcurrency() throws IOException {
    // T threads
    int numThreads = 10;
    // N entities
    int numEntities = 20;
    MetadataStorage mds = getMetadataStorage();
    ExecutorService executor = Executors.newFixedThreadPool(numThreads);
    CompletionService<List<MetadataChange>> completionService = new ExecutorCompletionService<>(executor);
    // Set up N entities with T tags (one to be removed per thread)
    Set<String> allRTags = IntStream.range(0, numThreads).mapToObj(t -> "r" + t).collect(Collectors.toSet());
    Map<Integer, MetadataEntity> entities = IntStream.range(0, numEntities).boxed().collect(Collectors.toMap(i -> i, i -> MetadataEntity.ofDataset("myds" + i)));
    mds.batch(entities.values().stream().map(entity -> new Update(entity, new Metadata(USER, allRTags))).collect(Collectors.toList()), MutationOptions.DEFAULT);
    // Generate a random but conflicting set of batches of mutations, one for each thread.
    // Construct the expected results for each entity along with the mutations
    // Also, because some threads will perform a Create, create a set of directives to preserve all other tags
    Map<Integer, Set<String>> expected = IntStream.range(0, numEntities).boxed().collect(Collectors.toMap(i -> i, i -> new HashSet<>(allRTags)));
    Map<Integer, List<MetadataMutation>> mutations = IntStream.range(0, numThreads).boxed().collect(Collectors.toMap(i -> i, i -> new ArrayList<>()));
    Map<ScopedNameOfKind, MetadataDirective> directives = new HashMap<>();
    Random rand = new Random(System.currentTimeMillis());
    IntStream.range(0, numThreads).forEach(t -> {
        directives.put(new ScopedNameOfKind(TAG, USER, "t" + t), MetadataDirective.KEEP);
        directives.put(new ScopedNameOfKind(TAG, USER, "r" + t), MetadataDirective.KEEP);
        directives.put(new ScopedNameOfKind(TAG, USER, "c" + t), MetadataDirective.KEEP);
        IntStream.range(0, numEntities).forEach(e -> {
            int random = rand.nextInt(100);
            if (random < 30) {
                expected.get(e).add("t" + t);
                mutations.get(t).add(new Update(entities.get(e), new Metadata(USER, tags("t" + t))));
            } else if (random < 60) {
                expected.get(e).add("c" + t);
                mutations.get(t).add(new Create(entities.get(e), new Metadata(USER, tags("c" + t)), directives));
            } else if (random < 90) {
                expected.get(e).remove("r" + t);
                mutations.get(t).add(new Remove(entities.get(e), Collections.singleton(new ScopedNameOfKind(TAG, USER, "r" + t))));
            }
        });
    });
    // submit all tasks and wait for their completion
    IntStream.range(0, numThreads).forEach(t -> completionService.submit(() -> mds.batch(mutations.get(t), MutationOptions.DEFAULT)));
    IntStream.range(0, numThreads).forEach(t -> {
        try {
            completionService.take();
        } catch (InterruptedException e) {
            throw Throwables.propagate(e);
        }
    });
    // validate that all "r" tags were removed and all "c" and "t" tags were added
    IntStream.range(0, numEntities).forEach(e -> {
        try {
            Assert.assertEquals("For entity " + entities.get(e), expected.get(e), mds.read(new Read(entities.get(e))).getTags(USER));
        } catch (Exception ex) {
            throw Throwables.propagate(ex);
        }
    });
    // clean up
    mds.batch(entities.values().stream().map(Drop::new).collect(Collectors.toList()), MutationOptions.DEFAULT);
}