use of com.thinkbiganalytics.metadata.api.feed.Feed.ID in project kylo by Teradata.
the class JcrFeedProvider method findByCategoryId.
@Override
public List<? extends Feed> findByCategoryId(Category.ID categoryId) {
String query = "SELECT e.* from " + EntityUtil.asQueryProperty(JcrFeed.NODE_TYPE) + " as e " + "INNER JOIN [" + CategoryDetails.NODE_TYPE + "] as det on ISCHILDNODE(e, det)" + "INNER JOIN [" + JcrCategory.NODE_TYPE + "] as cat on ISCHILDNODE(det, cat)" + "WHERE cat.[mode:id] = $id";
Map<String, String> bindParams = new HashMap<>();
bindParams.put("id", categoryId.toString());
try {
QueryResult result = JcrQueryUtil.query(getSession(), query, bindParams);
// not accessible. For now filter the result based on the feed summary access.
return JcrQueryUtil.queryResultStream(result, JcrFeed.class).filter(feed -> feed.getFeedSummary().isPresent()).collect(Collectors.toList());
} catch (RepositoryException e) {
throw new MetadataRepositoryException("Unable to getFeeds for Category ", e);
}
}
use of com.thinkbiganalytics.metadata.api.feed.Feed.ID in project kylo by Teradata.
the class JcrFeedProvider method addPostFeedChangeAction.
/**
* Registers an action that produces a feed change event upon a successful transaction commit.
*
* @param feed the feed to being created
*/
private void addPostFeedChangeAction(Feed feed, ChangeType changeType) {
Feed.State state = feed.getState();
Feed.ID id = feed.getId();
String feedName = feed.getQualifiedName();
final Principal principal = SecurityContextHolder.getContext().getAuthentication();
Consumer<Boolean> action = (success) -> {
if (success) {
FeedChange change = new FeedChange(changeType, feedName, feedName, id, state);
FeedChangeEvent event = new FeedChangeEvent(change, DateTime.now(), principal);
metadataEventService.notify(event);
}
};
JcrMetadataAccess.addPostTransactionAction(action);
}
use of com.thinkbiganalytics.metadata.api.feed.Feed.ID in project kylo by Teradata.
the class DefaultFeedManagerFeedService method createFeed.
/**
* Create/Update a Feed in NiFi. Save the metadata to Kylo meta store.
*
* @param feedMetadata the feed metadata
* @return an object indicating if the feed creation was successful or not
*/
public NifiFeed createFeed(final FeedMetadata feedMetadata) {
// functional access to be able to create a feed
this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.EDIT_FEEDS);
feedHistoryDataReindexingService.checkAndConfigureNiFi(feedMetadata);
feedHistoryDataReindexingService.checkAndEnsureFeedHistoryDataReindexingRequestIsAcceptable(feedMetadata);
if (feedMetadata.getState() == null) {
if (feedMetadata.isActive()) {
feedMetadata.setState(Feed.State.ENABLED.name());
} else {
feedMetadata.setState(Feed.State.DISABLED.name());
}
}
if (StringUtils.isBlank(feedMetadata.getId())) {
feedMetadata.setIsNew(true);
}
// Read all the feeds as System Service account to ensure the feed name is unique
if (feedMetadata.isNew()) {
metadataAccess.read(() -> {
Feed existing = feedProvider.findBySystemName(feedMetadata.getCategory().getSystemName(), feedMetadata.getSystemFeedName());
if (existing != null) {
throw new DuplicateFeedNameException(feedMetadata.getCategoryName(), feedMetadata.getFeedName());
}
}, MetadataAccess.SERVICE);
}
NifiFeed feed = createAndSaveFeed(feedMetadata);
// register the audit for the update event
if (feed.isSuccess() && !feedMetadata.isNew()) {
Feed.State state = Feed.State.valueOf(feedMetadata.getState());
Feed.ID id = feedProvider.resolveId(feedMetadata.getId());
notifyFeedStateChange(feedMetadata, id, state, MetadataChange.ChangeType.UPDATE);
} else if (feed.isSuccess() && feedMetadata.isNew()) {
// update the access control
feedMetadata.toRoleMembershipChangeList().stream().forEach(roleMembershipChange -> securityService.changeFeedRoleMemberships(feed.getFeedMetadata().getId(), roleMembershipChange));
}
feedHistoryDataReindexingService.updateHistoryDataReindexingFeedsAvailableCache(feedMetadata);
return feed;
}
use of com.thinkbiganalytics.metadata.api.feed.Feed.ID in project kylo by Teradata.
the class DefaultFeedManagerFeedService method getFeedById.
@Override
public FeedMetadata getFeedById(final String id, final boolean refreshTargetTableSchema) {
return metadataAccess.read(() -> {
this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_FEEDS);
FeedMetadata feedMetadata = null;
Feed.ID domainId = feedProvider.resolveId(id);
Feed domainFeed = feedProvider.findById(domainId);
if (domainFeed != null) {
feedMetadata = feedModelTransform.domainToFeedMetadata(domainFeed);
}
if (refreshTargetTableSchema && feedMetadata != null) {
// commented out for now as some issues were found with feeds with TEXTFILE as their output
// this will attempt to sync the schema stored in modeshape with that in Hive
// feedModelTransform.refreshTableSchemaFromHive(feedMetadata);
}
return feedMetadata;
});
}
use of com.thinkbiganalytics.metadata.api.feed.Feed.ID in project kylo by Teradata.
the class DefaultFeedManagerFeedService method createAndSaveFeed.
/**
* Create/Update a Feed in NiFi. Save the metadata to Kylo meta store.
*
* @param feedMetadata the feed metadata
* @return an object indicating if the feed creation was successful or not
*/
private NifiFeed createAndSaveFeed(FeedMetadata feedMetadata) {
Stopwatch stopwatch = Stopwatch.createStarted();
NifiFeed feed = null;
if (StringUtils.isBlank(feedMetadata.getId())) {
feedMetadata.setIsNew(true);
// If the feed is New we need to ensure the user has CREATE_FEED entity permission
if (accessController.isEntityAccessControlled()) {
metadataAccess.read(() -> {
// ensure the user has rights to create feeds under the category
Category domainCategory = categoryProvider.findById(categoryProvider.resolveId(feedMetadata.getCategory().getId()));
if (domainCategory == null) {
// throw exception
throw new MetadataRepositoryException("Unable to find the category " + feedMetadata.getCategory().getSystemName());
}
domainCategory.getAllowedActions().checkPermission(CategoryAccessControl.CREATE_FEED);
// ensure the user has rights to create feeds using the template
FeedManagerTemplate domainTemplate = templateProvider.findById(templateProvider.resolveId(feedMetadata.getTemplateId()));
if (domainTemplate == null) {
throw new MetadataRepositoryException("Unable to find the template " + feedMetadata.getTemplateId());
}
// domainTemplate.getAllowedActions().checkPermission(TemplateAccessControl.CREATE_FEED);
});
}
} else if (accessController.isEntityAccessControlled()) {
metadataAccess.read(() -> {
// perform explict entity access check here as we dont want to modify the NiFi flow unless user has access to edit the feed
Feed.ID domainId = feedProvider.resolveId(feedMetadata.getId());
Feed domainFeed = feedProvider.findById(domainId);
if (domainFeed != null) {
domainFeed.getAllowedActions().checkPermission(FeedAccessControl.EDIT_DETAILS);
} else {
throw new NotFoundException("Feed not found for id " + feedMetadata.getId());
}
});
}
// replace expressions with values
if (feedMetadata.getTable() != null) {
feedMetadata.getTable().updateMetadataFieldValues();
}
if (feedMetadata.getProperties() == null) {
feedMetadata.setProperties(new ArrayList<NifiProperty>());
}
// store ref to the originalFeedProperties before resolving and merging with the template
List<NifiProperty> originalFeedProperties = feedMetadata.getProperties();
// get all the properties for the metadata
RegisteredTemplate registeredTemplate = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(feedMetadata.getTemplateId()).templateName(feedMetadata.getTemplateName()).isFeedEdit(true).includeSensitiveProperties(true).build());
// copy the registered template properties it a new list so it doest get updated
List<NifiProperty> templateProperties = registeredTemplate.getProperties().stream().map(nifiProperty -> new NifiProperty(nifiProperty)).collect(Collectors.toList());
// update the template properties with the feedMetadata properties
List<NifiProperty> matchedProperties = NifiPropertyUtil.matchAndSetPropertyByProcessorName(templateProperties, feedMetadata.getProperties(), NifiPropertyUtil.PROPERTY_MATCH_AND_UPDATE_MODE.UPDATE_ALL_PROPERTIES);
registeredTemplate.setProperties(templateProperties);
feedMetadata.setProperties(registeredTemplate.getProperties());
feedMetadata.setRegisteredTemplate(registeredTemplate);
// skip any properties that the user supplied which are not ${ values
List<NifiProperty> propertiesToSkip = originalFeedProperties.stream().filter(property -> !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
List<NifiProperty> templatePropertiesToSkip = registeredTemplate.getProperties().stream().filter(property -> property.isSelected() && !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
if (templatePropertiesToSkip != null && !templatePropertiesToSkip.isEmpty()) {
propertiesToSkip.addAll(templatePropertiesToSkip);
}
// resolve any ${metadata.} properties
List<NifiProperty> resolvedProperties = propertyExpressionResolver.resolvePropertyExpressions(feedMetadata, propertiesToSkip);
// decrypt the metadata
feedModelTransform.decryptSensitivePropertyValues(feedMetadata);
FeedMetadata.STATE state = FeedMetadata.STATE.NEW;
try {
state = FeedMetadata.STATE.valueOf(feedMetadata.getState());
} catch (Exception e) {
// if the string isnt valid, disregard as it will end up disabling the feed.
}
boolean enabled = (FeedMetadata.STATE.NEW.equals(state) && feedMetadata.isActive()) || FeedMetadata.STATE.ENABLED.equals(state);
// flag to indicate to enable the feed later
// if this is the first time for this feed and it is set to be enabled, mark it to be enabled after we commit to the JCR store
boolean enableLater = false;
if (enabled && feedMetadata.isNew()) {
enableLater = true;
enabled = false;
feedMetadata.setState(FeedMetadata.STATE.DISABLED.name());
}
CreateFeedBuilder feedBuilder = CreateFeedBuilder.newFeed(nifiRestClient, nifiFlowCache, feedMetadata, registeredTemplate.getNifiTemplateId(), propertyExpressionResolver, propertyDescriptorTransform, niFiObjectCache, templateConnectionUtil).enabled(enabled).removeInactiveVersionedProcessGroup(removeInactiveNifiVersionedFeedFlows).autoAlign(nifiAutoFeedsAlignAfterSave).withNiFiTemplateCache(niFiTemplateCache);
if (registeredTemplate.isReusableTemplate()) {
feedBuilder.setReusableTemplate(true);
feedMetadata.setIsReusableFeed(true);
} else {
feedBuilder.inputProcessorType(feedMetadata.getInputProcessorType()).feedSchedule(feedMetadata.getSchedule()).properties(feedMetadata.getProperties());
if (registeredTemplate.usesReusableTemplate()) {
for (ReusableTemplateConnectionInfo connection : registeredTemplate.getReusableTemplateConnections()) {
feedBuilder.addInputOutputPort(new InputOutputPort(connection.getReusableTemplateInputPortName(), connection.getFeedOutputPortName()));
}
}
}
stopwatch.stop();
log.debug("Time to prepare data for saving feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
stopwatch.start();
NifiProcessGroup entity = feedBuilder.build();
stopwatch.stop();
log.debug("Time to save feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
feed = new NifiFeed(feedMetadata, entity);
// set the original feedProperties back to the feed
feedMetadata.setProperties(originalFeedProperties);
// encrypt the metadata properties
feedModelTransform.encryptSensitivePropertyValues(feedMetadata);
if (entity.isSuccess()) {
feedMetadata.setNifiProcessGroupId(entity.getProcessGroupEntity().getId());
try {
stopwatch.start();
saveFeed(feedMetadata);
// tell NiFi if this is a streaming feed or not
if (feedMetadata.getRegisteredTemplate().isStream()) {
streamingFeedJmsNotificationService.updateNiFiStatusJMSTopic(entity, feedMetadata);
}
feed.setEnableAfterSave(enableLater);
feed.setSuccess(true);
stopwatch.stop();
log.debug("Time to saveFeed in Kylo: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
stopwatch.start();
feedBuilder.checkAndRemoveVersionedProcessGroup();
} catch (Exception e) {
feed.setSuccess(false);
feed.addErrorMessage(e);
}
} else {
feed.setSuccess(false);
}
if (!feed.isSuccess()) {
if (!entity.isRolledBack()) {
try {
feedBuilder.rollback();
} catch (FeedRollbackException rollbackException) {
log.error("Error rolling back feed {}. {} ", feedMetadata.getCategoryAndFeedName(), rollbackException.getMessage());
feed.addErrorMessage("Error occurred in rolling back the Feed.");
}
entity.setRolledBack(true);
}
}
return feed;
}
Aggregations