use of com.thinkbiganalytics.feedmgr.rest.model.FeedCategory in project kylo by Teradata.
the class TableSetupTest method test.
@Test
public void test() throws Exception {
ObjectMapper mapper = new ObjectMapper();
FeedMetadata feedMetadata = new FeedMetadata();
feedMetadata.setCategory(new FeedCategory());
feedMetadata.setTable(new TableSetup());
feedMetadata.getTable().setTableSchema(new DefaultTableSchema());
feedMetadata.getTable().getTableSchema().setName("test");
DefaultField f1 = new DefaultField();
f1.setName("field1");
feedMetadata.getTable().getTableSchema().getFields().add(f1);
String json = mapper.writeValueAsString(feedMetadata);
FeedMetadata feedMetadata2 = mapper.readValue(json, FeedMetadata.class);
assertEquals(feedMetadata2.getTable().getTableSchema().getName(), feedMetadata.getTable().getTableSchema().getName());
}
use of com.thinkbiganalytics.feedmgr.rest.model.FeedCategory in project kylo by Teradata.
the class CategoryModelTransform method domainToFeedCategory.
/**
* Transforms the specified Metadata category into a Feed Manager category.
*
* @param domainCategory the Metadata category
* @param userFields the user-defined fields
* @return the Feed Manager category
*/
@Nullable
private FeedCategory domainToFeedCategory(@Nullable final Category domainCategory, @Nonnull final Set<UserFieldDescriptor> userFields, boolean includeFeedDetails) {
if (domainCategory != null) {
FeedCategory category = new FeedCategory();
category.setId(domainCategory.getId().toString());
if (includeFeedDetails && domainCategory.getFeeds() != null) {
List<FeedSummary> summaries = feedModelTransform.domainToFeedSummary(domainCategory.getFeeds());
category.setFeeds(summaries);
category.setRelatedFeeds(summaries.size());
}
category.setIconColor(domainCategory.getIconColor());
category.setIcon(domainCategory.getIcon());
category.setName(domainCategory.getDisplayName());
// in pre-0.8.4 version of Kylo there was no system name stored for domain categories
category.setSystemName(domainCategory.getSystemName() == null ? domainCategory.getDisplayName() : domainCategory.getSystemName());
category.setDescription(domainCategory.getDescription());
category.setAllowIndexing(domainCategory.isAllowIndexing());
category.setCreateDate(domainCategory.getCreatedTime() != null ? domainCategory.getCreatedTime().toDate() : null);
category.setUpdateDate(domainCategory.getModifiedTime() != null ? domainCategory.getModifiedTime().toDate() : null);
// Transform user-defined fields and properties
category.setUserFields(UserPropertyTransform.toUserFields(categoryProvider.getFeedUserFields(domainCategory.getId()).orElse(Collections.emptySet())));
category.setUserProperties(UserPropertyTransform.toUserProperties(domainCategory.getUserProperties(), userFields));
// Convert JCR securitygroup to DTO
List<com.thinkbiganalytics.feedmgr.rest.model.HadoopSecurityGroup> restSecurityGroups = new ArrayList<>();
if (domainCategory.getSecurityGroups() != null && domainCategory.getSecurityGroups().size() > 0) {
for (Object group : domainCategory.getSecurityGroups()) {
HadoopSecurityGroup hadoopSecurityGroup = (HadoopSecurityGroup) group;
com.thinkbiganalytics.feedmgr.rest.model.HadoopSecurityGroup restSecurityGroup = new com.thinkbiganalytics.feedmgr.rest.model.HadoopSecurityGroup();
restSecurityGroup.setDescription(hadoopSecurityGroup.getDescription());
restSecurityGroup.setId(hadoopSecurityGroup.getGroupId());
restSecurityGroup.setName(hadoopSecurityGroup.getName());
restSecurityGroups.add(restSecurityGroup);
}
}
category.setSecurityGroups(restSecurityGroups);
securityTransform.applyAccessControl(domainCategory, category);
return category;
} else {
return null;
}
}
use of com.thinkbiganalytics.feedmgr.rest.model.FeedCategory in project kylo by Teradata.
the class DefaultFeedManagerCategoryService method getCategoryBySystemName.
@Override
public FeedCategory getCategoryBySystemName(final String name) {
return metadataAccess.read(() -> {
this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_CATEGORIES);
final Category domainCategory = categoryProvider.findBySystemName(name);
return categoryModelTransform.domainToFeedCategory(domainCategory, true);
});
}
use of com.thinkbiganalytics.feedmgr.rest.model.FeedCategory in project kylo by Teradata.
the class InMemoryFeedManagerCategoryService method saveCategory.
@Override
public void saveCategory(final FeedCategory category) {
if (category.getId() == null) {
category.setId(UUID.randomUUID().toString());
category.setSystemName(SystemNamingService.generateSystemName(category.getName()));
} else {
FeedCategory oldCategory = categories.get(category.getId());
if (oldCategory != null && !oldCategory.getName().equalsIgnoreCase(category.getName())) {
// only regenerate the system name if there are no related feeds
if (oldCategory.getRelatedFeeds() == 0) {
category.setSystemName(SystemNamingService.generateSystemName(category.getName()));
}
}
List<FeedSummary> feeds = categories.get(category.getId()).getFeeds();
category.setFeeds(feeds);
}
categories.put(category.getId(), category);
FileObjectPersistence.getInstance().writeCategoriesToFile(categories.values());
}
use of com.thinkbiganalytics.feedmgr.rest.model.FeedCategory in project kylo by Teradata.
the class FeedImporter method importFeed.
// Import
/**
* Import a feed zip file
*/
private ImportFeed importFeed() throws Exception {
// read the JSON into the Feed object
FeedMetadata metadata = importFeed.getFeedToImport();
// query for this feed.
String feedCategory = StringUtils.isNotBlank(importFeedOptions.getCategorySystemName()) ? importFeedOptions.getCategorySystemName() : metadata.getSystemCategoryName();
FeedMetadata existingFeed = metadataAccess.read(() -> metadataService.getFeedByName(feedCategory, metadata.getSystemFeedName()));
metadata.getCategory().setSystemName(feedCategory);
ImportTemplateOptions importTemplateOptions = new ImportTemplateOptions();
importTemplateOptions.setImportComponentOptions(importFeedOptions.getImportComponentOptions());
importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setContinueIfExists(true);
ImportTemplate importTemplate = importFeed.getTemplate();
importTemplate.setImportOptions(importTemplateOptions);
importTemplateOptions.setUploadKey(importFeedOptions.getUploadKey());
importTemplate.setValid(true);
importTemplateOptions.setDeferCleanup(true);
// Import the Template
ImportTemplateRoutine importTemplateRoutine = importTemplateRoutineFactory.apply(importTemplate, importTemplateOptions, ImportTemplate.TYPE.ARCHIVE);
importTemplateRoutine.importTemplate();
if (importTemplate.isSuccess()) {
// import the feed
importFeed.setTemplate(importTemplate);
// now that we have the Feed object we need to create the instance of the feed
UploadProgressMessage uploadProgressMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Saving and creating feed instance in NiFi");
metadata.setIsNew(existingFeed == null ? true : false);
metadata.setFeedId(existingFeed != null ? existingFeed.getFeedId() : null);
metadata.setId(existingFeed != null ? existingFeed.getId() : null);
// reassign the templateId to the newly registered template id
metadata.setTemplateId(importTemplate.getTemplateId());
if (metadata.getRegisteredTemplate() != null) {
metadata.getRegisteredTemplate().setNifiTemplateId(importTemplate.getNifiTemplateId());
metadata.getRegisteredTemplate().setId(importTemplate.getTemplateId());
}
// get/create category
FeedCategory category = metadataService.getCategoryBySystemName(metadata.getCategory().getSystemName());
if (category == null) {
metadata.getCategory().setId(null);
metadataService.saveCategory(metadata.getCategory());
} else {
metadata.setCategory(category);
}
if (importFeedOptions.isDisableUponImport()) {
metadata.setActive(false);
metadata.setState(FeedMetadata.STATE.DISABLED.name());
}
// remap any preconditions to this new feed/category name.
if (metadata.getSchedule().hasPreconditions()) {
metadata.getSchedule().getPreconditions().stream().flatMap(preconditionRule -> preconditionRule.getProperties().stream()).filter(fieldRuleProperty -> PolicyPropertyTypes.PROPERTY_TYPE.currentFeed.name().equals(fieldRuleProperty.getType())).forEach(fieldRuleProperty -> fieldRuleProperty.setValue(metadata.getCategoryAndFeedName()));
}
// //for all those properties where the template value is != userEditable and the template value has a metadata. property, remove that property from the feed properties so it can be imported and assigned correctly
RegisteredTemplate template1 = registeredTemplateService.findRegisteredTemplateById(importTemplate.getTemplateId());
if (template1 != null) {
// Find all the properties in the template that have ${metadata. and are not userEditable.
// These are the properties we need to replace on the feed metadata
List<NifiProperty> metadataProperties = template1.getProperties().stream().filter(nifiProperty -> {
return nifiProperty != null && StringUtils.isNotBlank(nifiProperty.getValue()) && !nifiProperty.isUserEditable() && nifiProperty.getValue().contains("${" + MetadataFieldAnnotationFieldNameResolver.metadataPropertyPrefix);
}).collect(Collectors.toList());
// Replace the Feed Metadata properties with those that match the template ones from above.
List<NifiProperty> updatedProperties = metadata.getProperties().stream().map(nifiProperty -> {
NifiProperty p = NifiPropertyUtil.findPropertyByProcessorName(metadataProperties, nifiProperty);
return p != null ? p : nifiProperty;
}).collect(Collectors.toList());
metadata.setProperties(updatedProperties);
}
NifiFeed nifiFeed = metadataService.createFeed(metadata);
if (nifiFeed != null) {
importFeed.setFeedName(nifiFeed.getFeedMetadata().getCategoryAndFeedName());
if (nifiFeed.isSuccess()) {
uploadProgressMessage.update("Successfully saved the feed " + importFeed.getFeedName(), true);
} else {
if (nifiFeed.getFeedProcessGroup() != null && nifiFeed.getFeedProcessGroup().isRolledBack()) {
if (importTemplateRoutine != null) {
importTemplateRoutine.rollback();
}
}
uploadProgressMessage.update("Errors were found importing the feed " + importFeed.getFeedName(), false);
}
importTemplateRoutine.cleanup();
}
importFeed.setNifiFeed(nifiFeed);
importFeed.setSuccess(nifiFeed != null && nifiFeed.isSuccess());
} else {
importFeed.setSuccess(false);
importFeed.setTemplate(importTemplate);
importFeed.addErrorMessage(existingFeed, "The feed " + FeedNameUtil.fullName(feedCategory, metadata.getSystemFeedName()) + " needs additional properties to be supplied before importing.");
}
uploadProgressService.completeSection(importFeedOptions, ImportSection.Section.IMPORT_FEED_DATA);
return importFeed;
}
Aggregations