Examples with MetadataEntity io.cdap.cdap.api.metadata.MetadataEntity used on opensource projects

Example 26 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class ElasticsearchMetadataStorage method batch.

@Override
public List<MetadataChange> batch(List<? extends MetadataMutation> mutations, MutationOptions options) throws IOException {
    if (mutations.isEmpty()) {
        return Collections.emptyList();
    }
    if (mutations.size() == 1) {
        return Collections.singletonList(apply(mutations.get(0), options));
    }
    // first detect whether there are duplicate entity ids. If so, execute in sequence
    Set<MetadataEntity> entities = new HashSet<>();
    LinkedHashMap<MetadataEntity, MetadataMutation> mutationMap = new LinkedHashMap<>(mutations.size());
    boolean duplicate = false;
    for (MetadataMutation mutation : mutations) {
        if (!entities.add(mutation.getEntity())) {
            duplicate = true;
            break;
        }
        mutationMap.put(mutation.getEntity(), mutation);
    }
    if (duplicate) {
        // if there are multiple mutations for the same entity, execute all in sequence
        List<MetadataChange> changes = new ArrayList<>(mutations.size());
        for (MetadataMutation mutation : mutations) {
            changes.add(apply(mutation, options));
        }
        return changes;
    }
    // collect all changes in an order-preserving map. The first time doBatch() is called, it will
    // enter all entities in the map. Every time it is retried, the change may get updated, but that
    // will not change the order of the map.
    LinkedHashMap<MetadataEntity, MetadataChange> changes = new LinkedHashMap<>(mutations.size());
    try {
        // repeatedly try to read current metadata, apply the mutations and reindex, until there is no conflict
        return Retries.callWithRetries(() -> doBatch(mutationMap, changes, options), RetryStrategies.limit(50, RetryStrategies.fixDelay(100, TimeUnit.MILLISECONDS)), e -> e instanceof MetadataConflictException);
    } catch (MetadataConflictException e) {
        throw new MetadataConflictException("After retries: " + e.getRawMessage(), e.getConflictingEntities());
    }
}

Example 27 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class InvalidMetadataExceptionTest method testMesssages.

@Test
public void testMesssages() {
    // test dataset
    InvalidMetadataException invalidMetadataException = new InvalidMetadataException(NamespaceId.DEFAULT.dataset("ds").toMetadataEntity(), "error");
    String expectedMessage = "Unable to set metadata for dataset: ds " + "which exists in namespace: default. error";
    Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage());
    // test program
    invalidMetadataException = new InvalidMetadataException(NamespaceId.DEFAULT.app("app").program(ProgramType.WORKER, "wk").toMetadataEntity(), "error");
    expectedMessage = "Unable to set metadata for worker: wk in application: app of version: -SNAPSHOT deployed in " + "namespace: default. error";
    Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage());
    // test custom entity
    MetadataEntity customEntity = MetadataEntity.builder(NamespaceId.DEFAULT.dataset("ds").toMetadataEntity()).appendAsType("field", "empName").build();
    invalidMetadataException = new InvalidMetadataException(customEntity, "error");
    expectedMessage = "Unable to set metadata for namespace=default,dataset=ds,field=empName of type 'field'. error";
    Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage());
}

Example 28 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataDatasetTest method testCrossNamespaceCustomSearch.

@Test
public void testCrossNamespaceCustomSearch() throws Exception {
    String appName = "app";
    MetadataEntity ns1App = new NamespaceId("ns1").app(appName).toMetadataEntity();
    MetadataEntity ns2App = new NamespaceId("ns2").app(appName).toMetadataEntity();
    txnl.execute(() -> {
        dataset.addProperty(ns2App, MetadataConstants.ENTITY_NAME_KEY, appName);
        dataset.addProperty(ns1App, MetadataConstants.ENTITY_NAME_KEY, appName);
    });
    SortInfo nameAsc = new SortInfo(MetadataConstants.ENTITY_NAME_KEY, SortInfo.SortOrder.ASC);
    SearchRequest request = new SearchRequest(null, "*", ALL_TYPES, nameAsc, 0, 10, 0, null, false, EnumSet.allOf(EntityScope.class));
    SearchResults results = txnl.execute(() -> dataset.search(request));
    List<MetadataEntry> actual = results.getResults();
    List<MetadataEntry> expected = new ArrayList<>();
    expected.add(new MetadataEntry(ns1App, MetadataConstants.ENTITY_NAME_KEY, appName));
    expected.add(new MetadataEntry(ns2App, MetadataConstants.ENTITY_NAME_KEY, appName));
    Assert.assertEquals(expected, actual);
}

Example 29 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataDatasetTest method testCrossNamespaceDefaultSearch.

@Test
public void testCrossNamespaceDefaultSearch() throws Exception {
    MetadataEntity ns1App = new NamespaceId("ns1").app("a").toMetadataEntity();
    MetadataEntity ns2App = new NamespaceId("ns2").app("a").toMetadataEntity();
    txnl.execute(() -> {
        dataset.addProperty(ns1App, "k1", "v1");
        dataset.addProperty(ns1App, "k2", "v2");
        dataset.addProperty(ns2App, "k1", "v1");
    });
    SearchRequest request1 = new SearchRequest(null, "v1", ALL_TYPES, SortInfo.DEFAULT, 0, 10, 0, null, false, EnumSet.allOf(EntityScope.class));
    SearchResults results = txnl.execute(() -> dataset.search(request1));
    Set<MetadataEntry> actual = new HashSet<>(results.getResults());
    Set<MetadataEntry> expected = new HashSet<>();
    expected.add(new MetadataEntry(ns1App, "k1", "v1"));
    expected.add(new MetadataEntry(ns2App, "k1", "v1"));
    Assert.assertEquals(expected, actual);
    SearchRequest request2 = new SearchRequest(null, "v2", ALL_TYPES, SortInfo.DEFAULT, 0, 10, 0, null, false, EnumSet.allOf(EntityScope.class));
    results = txnl.execute(() -> dataset.search(request2));
    Assert.assertEquals(Collections.singletonList(new MetadataEntry(ns1App, "k2", "v2")), results.getResults());
    SearchRequest star = new SearchRequest(null, "*", ALL_TYPES, SortInfo.DEFAULT, 0, 10, 0, null, false, EnumSet.allOf(EntityScope.class));
    results = txnl.execute(() -> dataset.search(star));
    expected.add(new MetadataEntry(ns1App, "k2", "v2"));
    Assert.assertEquals(expected, new HashSet<>(results.getResults()));
}

Example 30 with MetadataEntity

use of io.cdap.cdap.api.metadata.MetadataEntity in project cdap by caskdata.

the class MetadataDatasetTest method testSearchOnTypes.

@Test
public void testSearchOnTypes() throws Exception {
    MetadataEntity myField1 = MetadataEntity.builder(MetadataEntity.ofDataset(NamespaceId.DEFAULT.getEntityName(), "myDs")).appendAsType("field", "myField1").build();
    MetadataEntity myField2 = MetadataEntity.builder(MetadataEntity.ofDataset(NamespaceId.DEFAULT.getEntityName(), "myDs")).appendAsType("field", "myField2").build();
    final MetadataEntry myFieldEntry1 = new MetadataEntry(myField1, "testKey1", "testValue1");
    final MetadataEntry myFieldEntry2 = new MetadataEntry(myField2, "testKey2", "testValue2");
    txnl.execute(() -> {
        dataset.addProperty(myField1, "testKey1", "testValue1");
        dataset.addProperty(myField2, "testKey2", "testValue2");
    });
    // Search for it based on value
    txnl.execute(() -> {
        List<MetadataEntry> results = searchByDefaultIndex("default", "field:myField1", ALL_TYPES);
        Assert.assertEquals(ImmutableList.of(myFieldEntry1), results);
        // should return both fields
        results = searchByDefaultIndex("default", "field:myFie*", ALL_TYPES);
        Assert.assertEquals(ImmutableList.of(myFieldEntry1, myFieldEntry2), results);
        results = searchByDefaultIndex("default", "field*", ALL_TYPES);
        Assert.assertEquals(ImmutableList.of(myFieldEntry1, myFieldEntry2), results);
    });
}