Examples with Category com.thinkbiganalytics.metadata.api.category.Category used on opensource projects

Example 1 with Category

use of com.thinkbiganalytics.metadata.api.category.Category in project kylo by Teradata.

the class FeedOpsUpgradeAction method upgradeTo.

/* (non-Javadoc)
     * @see com.thinkbiganalytics.metadata.upgrade.UpgradeState#upgradeFrom(com.thinkbiganalytics.metadata.api.app.KyloVersion)
     */
@Override
public void upgradeTo(KyloVersion startingVersion) {
    log.info("Upgrading from version: " + startingVersion);
    for (Category category : categoryProvider.findAll()) {
        // Ensure each category has an allowedActions (gets create if not present.)
        category.getAllowedActions();
    }
    // get all feeds defined in feed manager
    List<Feed> domainFeeds = feedProvider.findAll();
    Map<String, Feed> feedManagerFeedMap = new HashMap<>();
    if (domainFeeds != null && !domainFeeds.isEmpty()) {
        List<OpsManagerFeed.ID> opsManagerFeedIds = new ArrayList<OpsManagerFeed.ID>();
        for (Feed feedManagerFeed : domainFeeds) {
            opsManagerFeedIds.add(opsManagerFeedProvider.resolveId(feedManagerFeed.getId().toString()));
            feedManagerFeedMap.put(feedManagerFeed.getId().toString(), feedManagerFeed);
            // Ensure each feed has an allowedActions (gets create if not present.)
            feedManagerFeed.getAllowedActions();
        }
        // find those that match
        List<? extends OpsManagerFeed> opsManagerFeeds = opsManagerFeedProvider.findByFeedIds(opsManagerFeedIds);
        if (opsManagerFeeds != null) {
            for (OpsManagerFeed opsManagerFeed : opsManagerFeeds) {
                feedManagerFeedMap.remove(opsManagerFeed.getId().toString());
            }
        }
        List<OpsManagerFeed> feedsToAdd = new ArrayList<>();
        for (Feed feed : feedManagerFeedMap.values()) {
            String fullName = FeedNameUtil.fullName(feed.getCategory().getSystemName(), feed.getName());
            OpsManagerFeed.ID opsManagerFeedId = opsManagerFeedProvider.resolveId(feed.getId().toString());
            OpsManagerFeed opsManagerFeed = new JpaOpsManagerFeed(opsManagerFeedId, fullName);
            feedsToAdd.add(opsManagerFeed);
        }
        log.info("Synchronizing Feeds from Feed Manager. About to insert {} feed ids/names into Operations Manager", feedsToAdd.size());
        opsManagerFeedProvider.save(feedsToAdd);
    }
}

Example 2 with Category

use of com.thinkbiganalytics.metadata.api.category.Category in project kylo by Teradata.

the class UserFieldsUpgradeAction method upgradeCategoryFeedFields.

private void upgradeCategoryFeedFields(ExtensibleType type) {
    // tba:category:<category name>:feed
    String[] nameParts = type.getName().split(":");
    String catName = nameParts[2];
    Category category = this.categoryProvider.findBySystemName(catName);
    if (category != null) {
        this.categoryProvider.setFeedUserFields(category.getId(), type.getUserFieldDescriptors());
        this.extensibleTypeProvider.deleteType(type.getId());
    }
}

Example 3 with Category

use of com.thinkbiganalytics.metadata.api.category.Category in project kylo by Teradata.

the class JcrPropertyTest method testFeed.

/**
 * Creates a new Category Creates a new Feed Updates the Feed Get Feed and its versions Validates Feed is versioned along with its properties and can successfully return a version
 */
@Test
public void testFeed() {
    String categorySystemName = "my_category";
    Category category = metadata.commit(() -> {
        JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(categorySystemName);
        cat.setDescription("my category desc");
        cat.setTitle("my category");
        categoryProvider.update(cat);
        return cat;
    }, MetadataAccess.SERVICE);
    final JcrFeed.FeedId createdFeedId = metadata.commit(() -> {
        String sysName = "my_category";
        JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(sysName);
        cat.setDescription("my category desc");
        cat.setTitle("my category");
        categoryProvider.update(cat);
        JcrDerivedDatasource datasource1 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HiveDatasource", "mysql.table1", "mysql.table1", "mysql table source 1", null);
        JcrDerivedDatasource datasource2 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HiveDatasource", "mysql.table2", "mysql.table2", "mysql table source 2", null);
        JcrDerivedDatasource emptySource1 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "", "", "empty hdfs source", null);
        JcrDerivedDatasource emptySource2 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "", "", "empty hdfs source", null);
        Assert.assertEquals(emptySource1.getId(), emptySource2.getId());
        JcrDerivedDatasource emptySource3 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", null, null, "empty hdfs source", null);
        JcrDerivedDatasource emptySource4 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", null, null, "empty hdfs source", null);
        Assert.assertEquals(emptySource3.getId(), emptySource4.getId());
        JcrDerivedDatasource datasource3 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "/etl/customers/swert/012320342", "/etl/customers/swert/012320342", "mysql hdfs source", null);
        JcrDerivedDatasource datasource4 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "/etl/customers/swert/012320342", "/etl/customers/swert/012320342", "mysql hdfs source", null);
        Assert.assertEquals(datasource3.getId(), datasource4.getId());
        String feedSystemName = "my_feed_" + UUID.randomUUID();
        // JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(categorySystemName, feedSystemName, "my feed desc", datasource1.getId(), null);
        JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(sysName, feedSystemName, "my feed desc");
        feedProvider.ensureFeedSource(feed.getId(), datasource1.getId());
        feedProvider.ensureFeedSource(feed.getId(), datasource2.getId());
        feed.setTitle("my feed");
        feed.addTag("my tag");
        feed.addTag("my second tag");
        feed.addTag("feedTag");
        Map<String, Object> otherProperties = new HashMap<String, Object>();
        otherProperties.put("prop1", "my prop1");
        otherProperties.put("prop2", "my prop2");
        feed.setProperties(otherProperties);
        return feed.getId();
    }, MetadataAccess.SERVICE);
    // read and find feed verisons and ensure props
    JcrFeed.FeedId readFeedId = metadata.read(() -> {
        Session s = null;
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(createdFeedId);
        // TODO: Feed vesioning disabled for Kylo v0.5.0
        // int versions = printVersions(f);
        // Assert.assertTrue(versions > 1, "Expecting more than 1 version: jcr:rootVersion, 1.0");
        @SuppressWarnings("unchecked") List<? extends FeedSource> sources = f.getSources();
        Assert.assertTrue(sources.size() > 0);
        if (sources != null) {
            for (FeedSource source : sources) {
                Map<String, Object> dataSourceProperties = ((JcrDatasource) source.getDatasource()).getAllProperties();
                String type = (String) dataSourceProperties.get(JcrDerivedDatasource.TYPE_NAME);
                Assert.assertEquals(type, "HiveDatasource");
            }
        }
        List<JcrObject> taggedObjects = tagProvider.findByTag("my tag");
        // assert we got 1 feed back
        Assert.assertTrue(taggedObjects.size() >= 1);
        return f.getId();
    }, MetadataAccess.SERVICE);
    // update the feed again
    JcrFeed.FeedId updatedFeed = metadata.commit(() -> {
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(createdFeedId);
        f.setDescription("My Feed Updated Description");
        Map<String, Object> otherProperties = new HashMap<String, Object>();
        otherProperties.put("prop1", "my updated prop1");
        f.setProperties(otherProperties);
        ((JcrFeedProvider) feedProvider).update(f);
        return f.getId();
    }, MetadataAccess.SERVICE);
    // read it again and find the versions
    readFeedId = metadata.read(() -> {
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(updatedFeed);
        // Assert.assertEquals(v1Id, baseId);
        return f.getId();
    }, MetadataAccess.SERVICE);
}

Example 4 with Category

use of com.thinkbiganalytics.metadata.api.category.Category in project kylo by Teradata.

the class JcrPropertyTest method queryTest.

@Test
public void queryTest() {
    // final String query = "select e.* from [tba:feed] as e  join [tba:category] c on e.[tba:category].[tba:systemName] = c.[tba:systemName] where  c.[tba:systemName] = $category ";
    final String query = "select e.* from [tba:feed] as e join [tba:category] as c on e.[tba:category] = c.[jcr:uuid]";
    metadata.read(() -> {
        List<Node> feeds = ((JcrFeedProvider) feedProvider).findNodes(query);
    }, MetadataAccess.SERVICE);
    metadata.read(() -> {
        List<Category> c = categoryProvider.findAll();
        if (c != null) {
            for (Category cat : c) {
                Category jcrCategory = (Category) cat;
                List<? extends Feed> categoryFeeds = jcrCategory.getFeeds();
                if (categoryFeeds != null) {
                    for (Feed feed : categoryFeeds) {
                        log.info("Feed for category {} is {}", cat.getSystemName(), feed.getName());
                    }
                }
            }
        }
    }, MetadataAccess.SERVICE);
}

Example 5 with Category

use of com.thinkbiganalytics.metadata.api.category.Category in project kylo by Teradata.

the class JcrCategoryAllowedActionsTest method testSeeOwnContentOnly.

@Test
public void testSeeOwnContentOnly() {
    metadata.read(() -> {
        Category catA = this.categoryProvider.findById(idA);
        assertThat(catA.getDisplayName()).isNotNull().isEqualTo("Test A");
        assertThat(catA.getDescription()).isNotNull().isEqualTo("Test A descr");
        assertThat(catA.getSecurityGroups()).isNotNull();
        assertThat(catA.getUserProperties()).isNotNull();
        Category catB = this.categoryProvider.findById(idB);
        assertThat(catB).isNull();
    }, TEST_USER1);
}