use of com.thinkbiganalytics.metadata.modeshape.category.JcrCategory in project kylo by Teradata.
the class CategoryAndFeedReindexingUpgradeAction method upgradeTo.
@Override
public void upgradeTo(KyloVersion startingVersion) {
log.info("****************** Start: Category and Feed Reindexing Support Upgrade Action ****************");
log.info("[REF.A] Upgrading feeds to support (1) allowing/stopping metadata indexing, AND (2) reindexing of history data for version: {}...", startingVersion);
log.info("Total feeds found for upgrading: {}", feedProvider.getFeeds().size());
if (feedProvider.getFeeds().isEmpty()) {
log.warn("No feeds have been found for upgrade!");
}
feedProvider.getFeeds().forEach(feed -> {
JcrFeed jcrFeed = (JcrFeed) feed;
if ((jcrFeed.getFeedSummary().isPresent()) && (jcrFeed.getFeedData().isPresent())) {
Node summaryNode = jcrFeed.getFeedSummary().get().getNode();
Node feedDataNode = jcrFeed.getFeedData().get().getNode();
try {
summaryNode.addMixin("tba:indexControlled");
summaryNode.setProperty("tba:allowIndexing", "Y");
if (!JcrUtil.hasNode(feedDataNode, "tba:historyReindexing")) {
log.info("Feed with id [{}] in category [{}] having name [{}] requires history reindexing support. Adding it.", jcrFeed.getId(), jcrFeed.getCategory().getDisplayName(), jcrFeed.getName());
feedDataNode.addNode("tba:historyReindexing", "tba:historyReindexing");
} else {
log.info("Feed with id [{}] in category [{}] having name [{}] already has history reindexing support. Skipping step to add again.", jcrFeed.getId(), jcrFeed.getCategory().getDisplayName(), jcrFeed.getName());
}
log.info("Upgraded feed with id [{}] in category [{}] having name [{}]", jcrFeed.getId(), jcrFeed.getCategory().getDisplayName(), jcrFeed.getName());
} catch (Exception e) {
log.error("Failed to configure feed {} to (1) support allowing/stopping metadata indexing, AND/OR (2) support reindexing of history data: {}", ((JcrFeed) feed).getName(), e);
throw new UpgradeException("Failed to configure feed to (1) support allowing/stopping metadata indexing, AND/OR (2) support reindexing of history data: " + summaryNode, e);
}
} else {
log.error("Failed to get summary and/or data node for feed: {}", ((JcrFeed) feed).getName());
throw new UpgradeException("Failed to get summary and/or data node for feed:" + ((JcrFeed) feed).getName());
}
});
log.info("Completed [REF.A]: Configured feeds to support (1) allowing/stopping metadata indexing, AND (2) reindexing of history data for version: {}", startingVersion);
log.info("[REF.B] Upgrading categories to support allowing/stopping metadata indexing for version: {}...", startingVersion);
log.info("Total categories found for upgrading: {}", categoryProvider.findAll().size());
if (categoryProvider.findAll().isEmpty()) {
log.warn("No categories have been found for for upgrade!");
}
categoryProvider.findAll().forEach(category -> {
JcrCategory jcrCategory = (JcrCategory) category;
Node categoryNode = jcrCategory.getNode();
try {
categoryNode.addMixin("tba:indexControlled");
categoryNode.setProperty("tba:allowIndexing", "Y");
log.info("Upgraded category with id [{}] having name [{}]", jcrCategory.getId(), jcrCategory.getDisplayName());
} catch (Exception e) {
log.error("Failed to configure category {} to support allowing/stopping metadata indexing: {}", ((JcrCategory) category).getDisplayName(), e);
throw new UpgradeException("Failed to configure category to support allowing/stopping metadata indexing: " + categoryNode, e);
}
});
log.info("Completed [REF.B]: Configured categories to support allowing/stopping metadata indexing for version: {}", startingVersion);
log.info("****************** End: Category and Feed Reindexing Support Upgrade Action ****************");
}
use of com.thinkbiganalytics.metadata.modeshape.category.JcrCategory in project kylo by Teradata.
the class JcrPropertyTest method testFeed.
/**
* Creates a new Category Creates a new Feed Updates the Feed Get Feed and its versions Validates Feed is versioned along with its properties and can successfully return a version
*/
@Test
public void testFeed() {
String categorySystemName = "my_category";
Category category = metadata.commit(() -> {
JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(categorySystemName);
cat.setDescription("my category desc");
cat.setTitle("my category");
categoryProvider.update(cat);
return cat;
}, MetadataAccess.SERVICE);
final JcrFeed.FeedId createdFeedId = metadata.commit(() -> {
String sysName = "my_category";
JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(sysName);
cat.setDescription("my category desc");
cat.setTitle("my category");
categoryProvider.update(cat);
JcrDerivedDatasource datasource1 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HiveDatasource", "mysql.table1", "mysql.table1", "mysql table source 1", null);
JcrDerivedDatasource datasource2 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HiveDatasource", "mysql.table2", "mysql.table2", "mysql table source 2", null);
JcrDerivedDatasource emptySource1 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "", "", "empty hdfs source", null);
JcrDerivedDatasource emptySource2 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "", "", "empty hdfs source", null);
Assert.assertEquals(emptySource1.getId(), emptySource2.getId());
JcrDerivedDatasource emptySource3 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", null, null, "empty hdfs source", null);
JcrDerivedDatasource emptySource4 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", null, null, "empty hdfs source", null);
Assert.assertEquals(emptySource3.getId(), emptySource4.getId());
JcrDerivedDatasource datasource3 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "/etl/customers/swert/012320342", "/etl/customers/swert/012320342", "mysql hdfs source", null);
JcrDerivedDatasource datasource4 = (JcrDerivedDatasource) datasourceProvider.ensureDerivedDatasource("HDFSDatasource", "/etl/customers/swert/012320342", "/etl/customers/swert/012320342", "mysql hdfs source", null);
Assert.assertEquals(datasource3.getId(), datasource4.getId());
String feedSystemName = "my_feed_" + UUID.randomUUID();
// JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(categorySystemName, feedSystemName, "my feed desc", datasource1.getId(), null);
JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(sysName, feedSystemName, "my feed desc");
feedProvider.ensureFeedSource(feed.getId(), datasource1.getId());
feedProvider.ensureFeedSource(feed.getId(), datasource2.getId());
feed.setTitle("my feed");
feed.addTag("my tag");
feed.addTag("my second tag");
feed.addTag("feedTag");
Map<String, Object> otherProperties = new HashMap<String, Object>();
otherProperties.put("prop1", "my prop1");
otherProperties.put("prop2", "my prop2");
feed.setProperties(otherProperties);
return feed.getId();
}, MetadataAccess.SERVICE);
// read and find feed verisons and ensure props
JcrFeed.FeedId readFeedId = metadata.read(() -> {
Session s = null;
JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(createdFeedId);
// TODO: Feed vesioning disabled for Kylo v0.5.0
// int versions = printVersions(f);
// Assert.assertTrue(versions > 1, "Expecting more than 1 version: jcr:rootVersion, 1.0");
@SuppressWarnings("unchecked") List<? extends FeedSource> sources = f.getSources();
Assert.assertTrue(sources.size() > 0);
if (sources != null) {
for (FeedSource source : sources) {
Map<String, Object> dataSourceProperties = ((JcrDatasource) source.getDatasource()).getAllProperties();
String type = (String) dataSourceProperties.get(JcrDerivedDatasource.TYPE_NAME);
Assert.assertEquals(type, "HiveDatasource");
}
}
List<JcrObject> taggedObjects = tagProvider.findByTag("my tag");
// assert we got 1 feed back
Assert.assertTrue(taggedObjects.size() >= 1);
return f.getId();
}, MetadataAccess.SERVICE);
// update the feed again
JcrFeed.FeedId updatedFeed = metadata.commit(() -> {
JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(createdFeedId);
f.setDescription("My Feed Updated Description");
Map<String, Object> otherProperties = new HashMap<String, Object>();
otherProperties.put("prop1", "my updated prop1");
f.setProperties(otherProperties);
((JcrFeedProvider) feedProvider).update(f);
return f.getId();
}, MetadataAccess.SERVICE);
// read it again and find the versions
readFeedId = metadata.read(() -> {
JcrFeed f = (JcrFeed) ((JcrFeedProvider) feedProvider).findById(updatedFeed);
// Assert.assertEquals(v1Id, baseId);
return f.getId();
}, MetadataAccess.SERVICE);
}
use of com.thinkbiganalytics.metadata.modeshape.category.JcrCategory in project kylo by Teradata.
the class JcrPropertyTest method queryTest.
@Test
public void queryTest() {
// final String query = "select e.* from [tba:feed] as e join [tba:category] c on e.[tba:category].[tba:systemName] = c.[tba:systemName] where c.[tba:systemName] = $category ";
final String query = "select e.* from [tba:feed] as e join [tba:category] as c on e.[tba:category] = c.[jcr:uuid]";
metadata.read(() -> {
List<Node> feeds = ((JcrFeedProvider) feedProvider).findNodes(query);
}, MetadataAccess.SERVICE);
metadata.read(() -> {
List<Category> c = categoryProvider.findAll();
if (c != null) {
for (Category cat : c) {
Category jcrCategory = (Category) cat;
List<? extends Feed> categoryFeeds = jcrCategory.getFeeds();
if (categoryFeeds != null) {
for (Feed feed : categoryFeeds) {
log.info("Feed for category {} is {}", cat.getSystemName(), feed.getName());
}
}
}
}
}, MetadataAccess.SERVICE);
}
use of com.thinkbiganalytics.metadata.modeshape.category.JcrCategory in project kylo by Teradata.
the class FeedTestUtil method createCategory.
public Category createCategory(String categorySystemName) {
JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(categorySystemName);
cat.setDescription(categorySystemName + " desc");
cat.setTitle(categorySystemName);
categoryProvider.update(cat);
return cat;
}
use of com.thinkbiganalytics.metadata.modeshape.category.JcrCategory in project kylo by Teradata.
the class JcrFeedSlaTest method createFeed.
public JcrFeed.FeedId createFeed(final String categorySystemName, final String feedSystemName) {
Category category = metadata.commit(() -> {
JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(categorySystemName);
cat.setDescription(categorySystemName + " desc");
cat.setTitle(categorySystemName);
categoryProvider.update(cat);
return cat;
}, MetadataAccess.ADMIN);
return metadata.commit(() -> {
JcrCategory cat = (JcrCategory) categoryProvider.ensureCategory(categorySystemName);
JcrFeed feed = (JcrFeed) feedProvider.ensureFeed(categorySystemName, feedSystemName, feedSystemName + " desc");
feed.setTitle(feedSystemName);
return feed.getId();
}, MetadataAccess.ADMIN);
}
Aggregations