use of com.thinkbiganalytics.nifi.feedmgr.FeedRollbackException in project kylo by Teradata.
the class CreateFeedBuilder method rollback.
public ProcessGroupDTO rollback() throws FeedRollbackException {
if (newProcessGroup != null) {
try {
removeProcessGroup(newProcessGroup.getProcessGroupEntity());
} catch (NifiClientRuntimeException e) {
log.error("Unable to delete the ProcessGroup on rollback {} ", e.getMessage());
}
}
String parentGroupId = newProcessGroup != null ? newProcessGroup.getProcessGroupEntity().getParentGroupId() : (previousFeedProcessGroup != null ? previousFeedProcessGroup.getParentGroupId() : null);
try {
if (StringUtils.isNotBlank(parentGroupId)) {
ProcessGroupDTO feedGroup = restClient.getProcessGroupByName(parentGroupId, feedName);
// rename this group to be something else if for some reason we were not able to delete it
if (feedGroup != null) {
feedGroup.setName(feedGroup.getName() + ".rollback - " + new Date().getTime());
restClient.updateProcessGroup(feedGroup);
feedGroup = restClient.getProcessGroupByName(parentGroupId, feedName);
}
// there shouldn't be as we should have deleted it above
if (feedGroup == null) {
if (previousFeedProcessGroup != null) {
ProcessGroupDTO entity = restClient.getProcessGroup(previousFeedProcessGroup.getId(), false, false);
if (entity != null) {
entity.setName(feedName);
entity = restClient.updateProcessGroup(entity);
ProcessGroupDTO categoryGroup = restClient.getProcessGroup(entity.getParentGroupId(), false, false);
updatePortConnectionsForProcessGroup(entity, categoryGroup);
// disable all inputs
restClient.disableInputProcessors(entity.getId());
// mark everything else as running
restClient.markProcessorGroupAsRunning(entity);
if (hasConnectionPorts()) {
templateCreationHelper.markConnectionPortsAsRunning(entity);
}
// Set the state correctly for the inputs
if (enabled) {
restClient.setInputProcessorState(entity.getId(), inputProcessorType, NifiProcessUtil.PROCESS_STATE.RUNNING);
} else {
restClient.setInputProcessorState(entity.getId(), inputProcessorType, NifiProcessUtil.PROCESS_STATE.STOPPED);
}
return entity;
}
}
}
}
} catch (Exception e) {
throw new FeedRollbackException("Unable to rollback feed [" + feedName + "] with Parent Group Id of [" + parentGroupId + "] " + e.getMessage(), e);
}
return null;
}
use of com.thinkbiganalytics.nifi.feedmgr.FeedRollbackException in project kylo by Teradata.
the class DefaultFeedManagerFeedService method deployFeed.
private NifiFeed deployFeed(final FeedMetadata feedMetadata, com.thinkbiganalytics.metadata.api.versioning.EntityVersion<Feed.ID, Feed> version) throws DeployFeedException {
Stopwatch stopwatch = Stopwatch.createStarted();
boolean enabled = false;
if (feedMetadata.isActive()) {
feedMetadata.setState(Feed.State.ENABLED.name());
enabled = true;
} else {
feedMetadata.setState(Feed.State.DISABLED.name());
}
// Store ref to the originalFeedProperties before resolving and merging with the template
List<NifiProperty> originalFeedProperties = feedMetadata.getProperties();
// Get all the properties for the metadata
RegisteredTemplate registeredTemplate = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(feedMetadata.getTemplateId()).templateName(feedMetadata.getTemplateName()).isFeedEdit(true).includeSensitiveProperties(true).build());
// Copy the registered template properties it a new list so it doest get updated
List<NifiProperty> templateProperties = registeredTemplate.getProperties().stream().map(nifiProperty -> new NifiProperty(nifiProperty)).collect(Collectors.toList());
// Update the template properties with the feedMetadata properties
NifiPropertyUtil.matchAndSetPropertyByProcessorName(templateProperties, feedMetadata.getProperties(), NifiPropertyUtil.PropertyUpdateMode.UPDATE_ALL_PROPERTIES);
registeredTemplate.setProperties(templateProperties);
feedMetadata.setProperties(registeredTemplate.getProperties());
feedMetadata.setRegisteredTemplate(registeredTemplate);
// Skip any properties that the user supplied which are not ${ values
List<NifiProperty> propertiesToSkip = originalFeedProperties.stream().filter(property -> !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
List<NifiProperty> templatePropertiesToSkip = registeredTemplate.getProperties().stream().filter(property -> property.isSelected() && !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
if (templatePropertiesToSkip != null && !templatePropertiesToSkip.isEmpty()) {
propertiesToSkip.addAll(templatePropertiesToSkip);
}
// Resolve any ${metadata.} properties
propertyExpressionResolver.resolvePropertyExpressions(feedMetadata, propertiesToSkip);
// decrypt the metadata
feedModelTransform.decryptSensitivePropertyValues(feedMetadata);
// if this is the very first version we need to enable it later (after the data has been sync'd with ops manager)
boolean enableLater = false;
if (enabled && version.isFirstVersion()) {
enabled = false;
enableLater = true;
feedMetadata.setState(FeedMetadata.STATE.DISABLED.name());
}
CreateFeedBuilder feedBuilder = CreateFeedBuilder.newFeed(nifiRestClient, nifiFlowCache, feedMetadata, registeredTemplate.getNifiTemplateId(), propertyExpressionResolver, propertyDescriptorTransform, niFiObjectCache, templateConnectionUtil).enabled(enabled).setOriginalFeedProperties(originalFeedProperties).removeInactiveVersionedProcessGroup(removeInactiveNifiVersionedFeedFlows).autoAlign(nifiAutoFeedsAlignAfterSave).withNiFiTemplateCache(niFiTemplateCache);
if (registeredTemplate.isReusableTemplate()) {
feedBuilder.setReusableTemplate(true);
feedMetadata.setIsReusableFeed(true);
} else {
feedBuilder.inputProcessorType(feedMetadata.getInputProcessorType()).feedSchedule(feedMetadata.getSchedule()).properties(feedMetadata.getProperties());
if (registeredTemplate.usesReusableTemplate()) {
for (ReusableTemplateConnectionInfo connection : registeredTemplate.getReusableTemplateConnections()) {
feedBuilder.addInputOutputPort(new InputOutputPort(connection.getReusableTemplateInputPortName(), connection.getFeedOutputPortName()));
}
}
}
stopwatch.stop();
log.debug("Time to prepare data for saving feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
stopwatch.start();
NifiProcessGroup entity = feedBuilder.build();
stopwatch.stop();
log.debug("Time to save feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
NifiFeed feed = new NifiFeed(feedMetadata, entity);
// Set the original feedProperties back to the feed
feedMetadata.setProperties(originalFeedProperties);
// Encrypt the metadata properties
feedModelTransform.encryptSensitivePropertyValues(feedMetadata);
if (entity.isSuccess()) {
feedMetadata.setNifiProcessGroupId(entity.getProcessGroupEntity().getId());
try {
stopwatch.start();
saveDeployedFeed(feedMetadata, version);
// tell NiFi if this is a streaming feed or not
if (feedMetadata.getRegisteredTemplate().isStream()) {
streamingFeedJmsNotificationService.updateNiFiStatusJMSTopic(entity, feedMetadata);
}
feed.setSuccess(true);
feed.setEnableAfterSave(enableLater);
stopwatch.stop();
log.debug("Time to saveFeed in Kylo: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
stopwatch.start();
feedBuilder.checkAndRemoveVersionedProcessGroup();
} catch (Exception e) {
feed.setSuccess(false);
feed.addErrorMessage(e);
}
} else {
feed.setSuccess(false);
}
if (!feed.isSuccess()) {
if (!entity.isRolledBack()) {
try {
feedBuilder.rollback();
} catch (FeedRollbackException rollbackException) {
log.error("Error rolling back feed {}. {} ", feedMetadata.getCategoryAndFeedName(), rollbackException.getMessage());
feed.addErrorMessage("Error occurred in rolling back the Feed.");
}
entity.setRolledBack(true);
}
throw new DeployFeedException(feed);
}
// Move to isSuccess block??
feedHistoryDataReindexingService.updateHistoryDataReindexingFeedsAvailableCache(feedMetadata);
return feed;
}
use of com.thinkbiganalytics.nifi.feedmgr.FeedRollbackException in project kylo by Teradata.
the class DefaultFeedManagerFeedService method createAndSaveFeed.
/**
* Create/Update a Feed in NiFi. Save the metadata to Kylo meta store.
*
* @param feedMetadata the feed metadata
* @return an object indicating if the feed creation was successful or not
* @deprecated
*/
@Deprecated
private NifiFeed createAndSaveFeed(final FeedMetadata feedMetadata) {
Stopwatch stopwatch = Stopwatch.createStarted();
NifiFeed feed = null;
feedMetadata.setIsNew(StringUtils.isBlank(feedMetadata.getId()));
metadataAccess.read(() -> {
if (feedMetadata.isNew()) {
// ensure the user has rights to create feeds under the category by checking if it is visible (has read access)
Category domainCategory = categoryProvider.findById(categoryProvider.resolveId(feedMetadata.getCategory().getId()));
if (domainCategory == null) {
// throw exception
throw new MetadataRepositoryException("Unable to find the category " + feedMetadata.getCategory().getSystemName());
}
accessController.checkPermission(domainCategory, CategoryAccessControl.CREATE_FEED);
// ensure the user has rights to create feeds using the template by checking if it is visible.
FeedManagerTemplate domainTemplate = templateProvider.findById(templateProvider.resolveId(feedMetadata.getTemplateId()));
if (domainTemplate == null) {
throw new MetadataRepositoryException("Unable to find the template " + feedMetadata.getTemplateId());
}
// accessController.checkPermission(domainTemplate, TemplateAccessControl.CREATE_FEED);
} else {
// perform explict entity access check here as we dont want to modify the NiFi flow unless user has access to edit the feed
Feed.ID domainId = feedProvider.resolveId(feedMetadata.getId());
Feed domainFeed = feedProvider.findById(domainId);
if (domainFeed != null) {
accessController.checkPermission(domainFeed, FeedAccessControl.EDIT_DETAILS);
} else {
throw new NotFoundException("Feed not found for id " + feedMetadata.getId());
}
}
});
FeedMetadata feedToSave = feedMetadata;
if (!feedToSave.isNew()) {
// create a new version if this is not a new feed
feedToSave = saveDraftFeed(feedToSave);
}
// replace expressions with values
if (feedToSave.getTable() != null) {
feedToSave.getTable().updateMetadataFieldValues();
}
if (feedToSave.getProperties() == null) {
feedToSave.setProperties(new ArrayList<NifiProperty>());
}
FeedMetadata.STATE state = FeedMetadata.STATE.NEW;
try {
state = FeedMetadata.STATE.valueOf(feedToSave.getState());
} catch (Exception e) {
// if the string isnt valid, disregard as it will end up disabling the feed.
}
boolean enabled = (FeedMetadata.STATE.NEW.equals(state) && feedToSave.isActive()) || FeedMetadata.STATE.ENABLED.equals(state);
// store ref to the originalFeedProperties before resolving and merging with the template
List<NifiProperty> originalFeedProperties = feedToSave.getProperties();
// get all the properties for the metadata
RegisteredTemplate registeredTemplate = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(feedToSave.getTemplateId()).templateName(feedToSave.getTemplateName()).isFeedEdit(true).includeSensitiveProperties(true).build());
// copy the registered template properties it a new list so it doest get updated
List<NifiProperty> templateProperties = registeredTemplate.getProperties().stream().map(nifiProperty -> new NifiProperty(nifiProperty)).collect(Collectors.toList());
// update the template properties with the feedToSave properties
NifiPropertyUtil.matchAndSetPropertyByProcessorName(templateProperties, feedToSave.getProperties(), NifiPropertyUtil.PropertyUpdateMode.UPDATE_ALL_PROPERTIES);
registeredTemplate.setProperties(templateProperties);
feedToSave.setProperties(registeredTemplate.getProperties());
feedToSave.setRegisteredTemplate(registeredTemplate);
// skip any properties that the user supplied which are not ${ values
List<NifiProperty> propertiesToSkip = originalFeedProperties.stream().filter(property -> !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
List<NifiProperty> templatePropertiesToSkip = registeredTemplate.getProperties().stream().filter(property -> property.isSelected() && !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
if (templatePropertiesToSkip != null && !templatePropertiesToSkip.isEmpty()) {
propertiesToSkip.addAll(templatePropertiesToSkip);
}
// resolve any ${metadata.} properties
List<NifiProperty> resolvedProperties = propertyExpressionResolver.resolvePropertyExpressions(feedToSave, propertiesToSkip);
// decrypt the metadata
feedModelTransform.decryptSensitivePropertyValues(feedToSave);
// flag to indicate to enable the feed later
// if this is the first time for this feed and it is set to be enabled, mark it to be enabled after we commit to the JCR store
boolean enableLater = false;
if (enabled && feedToSave.isNew()) {
enableLater = true;
enabled = false;
feedToSave.setState(FeedMetadata.STATE.DISABLED.name());
}
CreateFeedBuilder feedBuilder = CreateFeedBuilder.newFeed(nifiRestClient, nifiFlowCache, feedToSave, registeredTemplate.getNifiTemplateId(), propertyExpressionResolver, propertyDescriptorTransform, niFiObjectCache, templateConnectionUtil).enabled(enabled).setOriginalFeedProperties(originalFeedProperties).removeInactiveVersionedProcessGroup(removeInactiveNifiVersionedFeedFlows).autoAlign(nifiAutoFeedsAlignAfterSave).withNiFiTemplateCache(niFiTemplateCache);
if (registeredTemplate.isReusableTemplate()) {
feedBuilder.setReusableTemplate(true);
feedToSave.setIsReusableFeed(true);
} else {
feedBuilder.inputProcessorType(feedToSave.getInputProcessorType()).feedSchedule(feedToSave.getSchedule()).properties(feedToSave.getProperties());
if (registeredTemplate.usesReusableTemplate()) {
for (ReusableTemplateConnectionInfo connection : registeredTemplate.getReusableTemplateConnections()) {
feedBuilder.addInputOutputPort(new InputOutputPort(connection.getReusableTemplateInputPortName(), connection.getFeedOutputPortName()));
}
}
}
stopwatch.stop();
log.debug("Time to prepare data for saving feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
stopwatch.start();
NifiProcessGroup entity = feedBuilder.build();
stopwatch.stop();
log.debug("Time to save feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
feed = new NifiFeed(feedToSave, entity);
// set the original feedProperties back to the feed
feedToSave.setProperties(originalFeedProperties);
// encrypt the metadata properties
feedModelTransform.encryptSensitivePropertyValues(feedToSave);
if (entity.isSuccess()) {
feedToSave.setNifiProcessGroupId(entity.getProcessGroupEntity().getId());
try {
stopwatch.start();
saveFeed(feedToSave);
// tell NiFi if this is a streaming feed or not
if (feedToSave.getRegisteredTemplate().isStream()) {
streamingFeedJmsNotificationService.updateNiFiStatusJMSTopic(entity, feedToSave);
}
feed.setEnableAfterSave(enableLater);
feed.setSuccess(true);
stopwatch.stop();
log.debug("Time to saveFeed in Kylo: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
stopwatch.start();
feedBuilder.checkAndRemoveVersionedProcessGroup();
} catch (Exception e) {
feed.setSuccess(false);
feed.addErrorMessage(e);
}
} else {
feed.setSuccess(false);
}
if (!feed.isSuccess()) {
if (!entity.isRolledBack()) {
try {
feedBuilder.rollback();
} catch (FeedRollbackException rollbackException) {
log.error("Error rolling back feed {}. {} ", feedToSave.getCategoryAndFeedName(), rollbackException.getMessage());
feed.addErrorMessage("Error occurred in rolling back the Feed.");
}
entity.setRolledBack(true);
}
}
return feed;
}
Aggregations