Search in sources :

Example 1 with FeedSource

use of com.thinkbiganalytics.metadata.api.feed.FeedSource in project kylo by Teradata.

the class JcrPropertyTest method testFeed.

/**
 * Creates a new Category Creates a new Feed Updates the Feed Get Feed and its versions Validates Feed is versioned along with its properties and can successfully return a version
 */
@Test
public void testFeed() {
    String categorySystemName = "my_category";
    Category category = metadata.commit(() -> {
        JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(categorySystemName);
        cat.setDescription("my category desc");
        cat.setTitle("my category");
        categoryProvider.update(cat);
        return cat;
    }, MetadataAccess.SERVICE);
    final JcrFeed.FeedId createdFeedId = metadata.commit(() -> {
        String sysName = "my_category";
        JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(sysName);
        cat.setDescription("my category desc");
        cat.setTitle("my category");
        categoryProvider.update(cat);
        JcrDerivedDatasource datasource1 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HiveDatasource", "mysql.table1", "mysql.table1", "mysql table source 1", null);
        JcrDerivedDatasource datasource2 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HiveDatasource", "mysql.table2", "mysql.table2", "mysql table source 2", null);
        JcrDerivedDatasource emptySource1 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "", "", "empty hdfs source", null);
        JcrDerivedDatasource emptySource2 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "", "", "empty hdfs source", null);
        Assert.assertEquals(emptySource1.getId(), emptySource2.getId());
        JcrDerivedDatasource emptySource3 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", null, null, "empty hdfs source", null);
        JcrDerivedDatasource emptySource4 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", null, null, "empty hdfs source", null);
        Assert.assertEquals(emptySource3.getId(), emptySource4.getId());
        JcrDerivedDatasource datasource3 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "/etl/customers/swert/012320342", "/etl/customers/swert/012320342", "mysql hdfs source", null);
        JcrDerivedDatasource datasource4 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "/etl/customers/swert/012320342", "/etl/customers/swert/012320342", "mysql hdfs source", null);
        Assert.assertEquals(datasource3.getId(), datasource4.getId());
        String feedSystemName = "my_feed_" + UUID.randomUUID();
        // JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(categorySystemName, feedSystemName, "my feed desc", datasource1.getId(), null);
        JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(sysName, feedSystemName, "my feed desc");
        feedProvider.ensureFeedSource(feed.getId(), datasource1.getId());
        feedProvider.ensureFeedSource(feed.getId(), datasource2.getId());
        feed.setTitle("my feed");
        feed.addTag("my tag");
        feed.addTag("my second tag");
        feed.addTag("feedTag");
        Map<String, Object> otherProperties = new HashMap<String, Object>();
        otherProperties.put("prop1", "my prop1");
        otherProperties.put("prop2", "my prop2");
        feed.setProperties(otherProperties);
        return feed.getId();
    }, MetadataAccess.SERVICE);
    // read and find feed verisons and ensure props
    JcrFeed.FeedId readFeedId = metadata.read(() -> {
        Session s = null;
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(createdFeedId);
        // TODO: Feed vesioning disabled for Kylo v0.5.0
        // int versions = printVersions(f);
        // Assert.assertTrue(versions > 1, "Expecting more than 1 version: jcr:rootVersion, 1.0");
        @SuppressWarnings("unchecked") List<? extends FeedSource> sources = f.getSources();
        Assert.assertTrue(sources.size() > 0);
        if (sources != null) {
            for (FeedSource source : sources) {
                Map<String, Object> dataSourceProperties = ((JcrDatasource) source.getDatasource()).getAllProperties();
                String type = (String) dataSourceProperties.get(JcrDerivedDatasource.TYPE_NAME);
                Assert.assertEquals(type, "HiveDatasource");
            }
        }
        List<JcrObject> taggedObjects = tagProvider.findByTag("my tag");
        // assert we got 1 feed back
        Assert.assertTrue(taggedObjects.size() >= 1);
        return f.getId();
    }, MetadataAccess.SERVICE);
    // update the feed again
    JcrFeed.FeedId updatedFeed = metadata.commit(() -> {
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(createdFeedId);
        f.setDescription("My Feed Updated Description");
        Map<String, Object> otherProperties = new HashMap<String, Object>();
        otherProperties.put("prop1", "my updated prop1");
        f.setProperties(otherProperties);
        ((JcrFeedProvider) feedProvider).update(f);
        return f.getId();
    }, MetadataAccess.SERVICE);
    // read it again and find the versions
    readFeedId = metadata.read(() -> {
        JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(updatedFeed);
        // Assert.assertEquals(v1Id, baseId);
        return f.getId();
    }, MetadataAccess.SERVICE);
}
Also used : Category(com.thinkbiganalytics.metadata.api.category.Category) JcrCategory(com.thinkbiganalytics.metadata.modeshape.category.JcrCategory) JcrFeedProvider(com.thinkbiganalytics.metadata.modeshape.feed.JcrFeedProvider) HashMap(java.util.HashMap) JcrCategory(com.thinkbiganalytics.metadata.modeshape.category.JcrCategory) JcrObject(com.thinkbiganalytics.metadata.modeshape.common.JcrObject) JcrDatasource(com.thinkbiganalytics.metadata.modeshape.datasource.JcrDatasource) JcrFeed(com.thinkbiganalytics.metadata.modeshape.feed.JcrFeed) FeedSource(com.thinkbiganalytics.metadata.api.feed.FeedSource) JcrDerivedDatasource(com.thinkbiganalytics.metadata.modeshape.datasource.JcrDerivedDatasource) JcrObject(com.thinkbiganalytics.metadata.modeshape.common.JcrObject) Session(javax.jcr.Session) Test(org.junit.Test)

Example 2 with FeedSource

use of com.thinkbiganalytics.metadata.api.feed.FeedSource in project kylo by Teradata.

the class JcrDatasource method setSources.

public void setSources(List<FeedSource> sources) {
    JcrPropertyUtil.setProperty(this.node, SOURCE_NAME, null);
    for (FeedSource src : sources) {
        Node destNode = ((JcrFeedSource) src).getNode();
        addSourceNode(destNode);
    }
}
Also used : FeedSource(com.thinkbiganalytics.metadata.api.feed.FeedSource) JcrFeedSource(com.thinkbiganalytics.metadata.modeshape.feed.JcrFeedSource) Node(javax.jcr.Node) JcrFeedSource(com.thinkbiganalytics.metadata.modeshape.feed.JcrFeedSource)

Example 3 with FeedSource

use of com.thinkbiganalytics.metadata.api.feed.FeedSource in project kylo by Teradata.

the class BaseFeed method addSource.

public FeedSource addSource(Datasource ds, ServiceLevelAgreement agreement) {
    Source src = new Source(ds, agreement);
    this.sources.add(src);
    return src;
}
Also used : FeedSource(com.thinkbiganalytics.metadata.api.feed.FeedSource)

Example 4 with FeedSource

use of com.thinkbiganalytics.metadata.api.feed.FeedSource in project kylo by Teradata.

the class InMemoryFeedProvider method ensureFeedSource.

private FeedSource ensureFeedSource(BaseFeed feed, Datasource ds, ServiceLevelAgreement.ID slaId) {
    Map<Datasource.ID, FeedSource> srcIds = new HashMap<>();
    for (FeedSource src : feed.getSources()) {
        srcIds.put(src.getDatasource().getId(), src);
    }
    if (srcIds.containsKey(ds.getId())) {
        return srcIds.get(ds.getId());
    } else {
        ServiceLevelAgreement sla = this.slaProvider.getAgreement(slaId);
        FeedSource src = feed.addSource(ds, sla);
        return src;
    }
}
Also used : HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) FeedSource(com.thinkbiganalytics.metadata.api.feed.FeedSource) ServiceLevelAgreement(com.thinkbiganalytics.metadata.sla.api.ServiceLevelAgreement) ID(com.thinkbiganalytics.metadata.api.datasource.Datasource.ID)

Example 5 with FeedSource

use of com.thinkbiganalytics.metadata.api.feed.FeedSource in project kylo by Teradata.

the class JcrFeedProvider method ensureFeedSource.

@Override
public FeedSource ensureFeedSource(Feed.ID feedId, Datasource.ID dsId) {
    JcrFeed feed = (JcrFeed) findById(feedId);
    FeedSource source = feed.getSource(dsId);
    if (source == null) {
        JcrDatasource datasource = (JcrDatasource) datasourceProvider.getDatasource(dsId);
        if (datasource != null) {
            JcrFeedSource jcrSrc = feed.ensureFeedSource(datasource);
            save();
            return jcrSrc;
        } else {
            throw new DatasourceNotFoundException(dsId);
        }
    } else {
        return source;
    }
}
Also used : JcrDatasource(com.thinkbiganalytics.metadata.modeshape.datasource.JcrDatasource) FeedSource(com.thinkbiganalytics.metadata.api.feed.FeedSource) DatasourceNotFoundException(com.thinkbiganalytics.metadata.api.datasource.DatasourceNotFoundException)

Aggregations

FeedSource (com.thinkbiganalytics.metadata.api.feed.FeedSource)6 HashMap (java.util.HashMap)3 Category (com.thinkbiganalytics.metadata.api.category.Category)2 Stopwatch (com.google.common.base.Stopwatch)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 Sets (com.google.common.collect.Sets)1 HadoopAuthorizationService (com.thinkbiganalytics.datalake.authorization.service.HadoopAuthorizationService)1 CreateFeedBuilder (com.thinkbiganalytics.feedmgr.nifi.CreateFeedBuilder)1 PropertyExpressionResolver (com.thinkbiganalytics.feedmgr.nifi.PropertyExpressionResolver)1 TemplateConnectionUtil (com.thinkbiganalytics.feedmgr.nifi.TemplateConnectionUtil)1 NifiFlowCache (com.thinkbiganalytics.feedmgr.nifi.cache.NifiFlowCache)1 EntityVersion (com.thinkbiganalytics.feedmgr.rest.model.EntityVersion)1 EntityVersionDifference (com.thinkbiganalytics.feedmgr.rest.model.EntityVersionDifference)1 FeedMetadata (com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata)1 FeedSummary (com.thinkbiganalytics.feedmgr.rest.model.FeedSummary)1 FeedVersions (com.thinkbiganalytics.feedmgr.rest.model.FeedVersions)1 NifiFeed (com.thinkbiganalytics.feedmgr.rest.model.NifiFeed)1 RegisteredTemplate (com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate)1 RegisteredTemplateRequest (com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplateRequest)1 ReusableTemplateConnectionInfo (com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo)1