use of com.thinkbiganalytics.nifi.feedmgr.FeedCreationException in project kylo by Teradata.
the class CreateFeedBuilder method build.
/**
* Build the NiFi flow instance
*
* @return an object indicating if the feed flow was successfully built or not
*/
public NifiProcessGroup build() throws FeedCreationException {
try {
log.info("Creating the feed {}.{} ", category, feedName);
newProcessGroup = null;
Stopwatch totalTime = Stopwatch.createStarted();
Stopwatch eventTime = Stopwatch.createStarted();
TemplateDTO template = getTemplate();
if (template != null) {
log.debug("Time to get Template {}. ElapsedTime: {} ms", template.getName(), eventTime(eventTime));
// create the encompassing process group
eventTime.start();
ProcessGroupDTO feedProcessGroup = createProcessGroupForFeed();
log.debug("Time to create process group. ElapsedTime: {} ms", eventTime(eventTime));
if (feedProcessGroup != null) {
String processGroupId = feedProcessGroup.getId();
// snapshot the existing controller services
eventTime.start();
templateCreationHelper.snapshotControllerServiceReferences();
templateCreationHelper.setOriginalFeedProperties(this.originalFeedProperties);
log.debug("Time to snapshotControllerServices. ElapsedTime: {} ms", eventTime(eventTime));
// create the flow from the template
eventTime.start();
TemplateInstance instance = templateCreationHelper.instantiateFlowFromTemplate(processGroupId, templateId);
FlowSnippetDTO feedInstance = instance.getFlowSnippetDTO();
feedProcessGroup.setContents(feedInstance);
log.debug("Time to instantiateFlowFromTemplate. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
String feedCategoryId = feedProcessGroup.getParentGroupId();
ProcessGroupDTO categoryGroup = this.categoryGroup;
if (categoryGroup == null) {
categoryGroup = this.categoryGroup = restClient.getProcessGroup(feedCategoryId, false, false);
}
// update the group with this template?
updatePortConnectionsForProcessGroup(feedProcessGroup, categoryGroup);
log.debug("Time to updatePortConnectionsForProcessGroup. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// mark the new services that were created as a result of creating the new flow from the template
templateCreationHelper.identifyNewlyCreatedControllerServiceReferences(instance);
log.debug("Time to identifyNewlyCreatedControllerServiceReferences. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// match the properties incoming to the defined properties
updateProcessGroupProperties(processGroupId, feedProcessGroup.getName());
log.debug("Time to updateProcessGroupProperties. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// Fetch the Feed Group now that it has the flow in it
ProcessGroupDTO entity = restClient.getProcessGroup(processGroupId, true, true);
log.debug("Time to getProcessGroup. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
ProcessorDTO input = fetchInputProcessorForProcessGroup(entity);
ProcessorDTO cleanupProcessor = NifiProcessUtil.findFirstProcessorsByType(NifiProcessUtil.getInputProcessors(entity), "com.thinkbiganalytics.nifi.v2.metadata.TriggerCleanup");
List<ProcessorDTO> nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
log.debug("Time to fetchInputProcessorForProcessGroup. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
List<NifiProperty> updatedControllerServiceProperties = new ArrayList<>();
// update any references to the controller services and try to assign the value to an enabled service if it is not already
if (input != null) {
updatedControllerServiceProperties.addAll(templateCreationHelper.updateControllerServiceReferences(Lists.newArrayList(input), instance));
}
if (cleanupProcessor != null) {
updatedControllerServiceProperties.addAll(templateCreationHelper.updateControllerServiceReferences(Collections.singletonList(cleanupProcessor), instance));
}
updatedControllerServiceProperties.addAll(templateCreationHelper.updateControllerServiceReferences(nonInputProcessors, instance));
log.debug("Time to updatedControllerServiceProperties. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// refetch processors for updated errors
entity = restClient.getProcessGroup(processGroupId, true, true);
input = fetchInputProcessorForProcessGroup(entity);
nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
RemoteProcessGroupValidator remoteProcessGroupValidator = new RemoteProcessGroupValidator(restClient, templateConnectionUtil, modifiedProperties);
RemoteProcessGroupValidator.RemoteProcessGroupValidation remoteProcessGroupValidation = remoteProcessGroupValidator.validateAndFixRemoteProcessGroups(entity);
newProcessGroup = new NifiProcessGroup(entity, input, nonInputProcessors);
if (!remoteProcessGroupValidation.isValid()) {
log.error("Invalid Remote Process Group's were found.");
// add the errors
remoteProcessGroupValidation.getAllInvalidConnections().stream().forEach(connectionDTO -> {
String errorMsg = "Invalid Remote Process Group. Unable set find valid remote input port for " + connectionDTO.getDestination().getName() + ". Please ensure you have a remote input port matching this name";
newProcessGroup.addError(processGroupId, connectionDTO.getId(), NifiError.SEVERITY.FATAL, errorMsg, "Remote Process Group");
newProcessGroup.setSuccess(false);
});
}
log.debug("Time to re-fetchInputProcessorForProcessGroup. ElapsedTime: {} ms", eventTime(eventTime));
// Validate and if invalid Delete the process group
if (newProcessGroup.hasFatalErrors()) {
eventTime.start();
removeProcessGroup(entity);
// cleanupControllerServices();
newProcessGroup.setSuccess(false);
log.debug("Time to removeProcessGroup. Errors found. ElapsedTime: {} ms", eventTime(eventTime));
} else {
eventTime.start();
// update the input schedule
updateFeedSchedule(newProcessGroup, input);
log.debug("Time to update feed schedule. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// just need to update for this processgroup
Collection<ProcessorDTO> processors = NifiProcessUtil.getProcessors(entity);
Collection<ConnectionDTO> connections = NifiConnectionUtil.getAllConnections(entity);
nifiFlowCache.updateFlowForFeed(feedMetadata, entity.getId(), processors, connections);
log.debug("Time to build flow graph with {} processors and {} connections. ElapsedTime: {} ms", processors.size(), connections.size(), eventTime(eventTime));
/*
//Cache the processorIds to the respective flowIds for availability in the ProvenanceReportingTask
NifiVisitableProcessGroup group = nifiFlowCache.getFlowOrder(newProcessGroup.getProcessGroupEntity(), true);
log.debug("Time to get the flow order. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
NifiFlowProcessGroup
flow =
new NifiFlowBuilder().build(
group);
log.debug("Time to build flow graph with {} processors. ElapsedTime: {} ms", flow.getProcessorMap().size(), eventTime(eventTime));
eventTime.start();
nifiFlowCache.updateFlow(feedMetadata, flow);
log.debug("Time to update NiFiFlowCache with {} processors. ElapsedTime: {} ms", flow.getProcessorMap().size(), eventTime(eventTime));
*/
eventTime.start();
// disable all inputs
restClient.disableInputProcessors(newProcessGroup.getProcessGroupEntity().getId());
log.debug("Time to disableInputProcessors. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// mark everything else as running
templateCreationHelper.markProcessorsAsRunning(newProcessGroup);
log.debug("Time to markNonInputsAsRunning. ElapsedTime: {} ms", eventTime(eventTime));
// if desired start the input processor
if (input != null) {
eventTime.start();
if (enabled) {
markInputAsRunning(newProcessGroup, input);
// /make the input/output ports in the category group as running
if (hasConnectionPorts()) {
templateCreationHelper.markConnectionPortsAsRunning(entity);
}
} else {
// /make the input/output ports in the category group as running
if (hasConnectionPorts()) {
templateCreationHelper.markConnectionPortsAsRunning(entity);
}
markInputAsStopped(newProcessGroup, input);
}
log.debug("Time to mark input as {}. ElapsedTime: {} ms", (enabled ? "Running" : "Stopped"), eventTime(eventTime));
}
if (newProcessGroup.hasFatalErrors()) {
eventTime.start();
rollback();
newProcessGroup.setRolledBack(true);
// cleanupControllerServices();
newProcessGroup.setSuccess(false);
log.debug("Time to rollback on Fatal Errors. ElapsedTime: {} ms", eventTime(eventTime));
}
List<NifiError> templateCreationErrors = templateCreationHelper.getErrors();
if (templateCreationErrors != null) {
errors.addAll(templateCreationErrors);
}
// add any global errors to the object
if (errors != null && !errors.isEmpty()) {
for (NifiError error : errors) {
newProcessGroup.addError(error);
if (error.isFatal()) {
newProcessGroup.setSuccess(false);
if (!newProcessGroup.isRolledBack()) {
rollback();
newProcessGroup.setRolledBack(true);
}
}
}
}
}
eventTime.start();
templateCreationHelper.cleanupControllerServices();
// fix the feed metadata controller service references
updateFeedMetadataControllerServiceReferences(updatedControllerServiceProperties);
log.debug("Time cleanup controller services. ElapsedTime: {} ms", eventTime(eventTime));
// align items
if (this.autoAlign) {
eventTime.start();
log.info("Aligning Feed flows in NiFi ");
AlignProcessGroupComponents alignProcessGroupComponents = new AlignProcessGroupComponents(restClient.getNiFiRestClient(), entity.getParentGroupId());
alignProcessGroupComponents.autoLayout();
// fetch the parent to get that id to align
if (newCategory) {
log.info("This is the first feed created in the category {}. Aligning the categories. ", feedMetadata.getCategory().getSystemName());
new AlignProcessGroupComponents(restClient.getNiFiRestClient(), this.categoryGroup.getParentGroupId()).autoLayout();
}
log.info("Time align feed process groups. ElapsedTime: {} ms", eventTime(eventTime));
} else {
log.info("Skipping auto alignment in NiFi. You can always manually align this category and all of its feeds by using the rest api: /v1/feedmgr/nifi/auto-align/{}", entity.getParentGroupId());
if (newCategory) {
log.info("To re align the categories: /v1/feedmgr/nifi/auto-align/{}", this.categoryGroup.getParentGroupId());
}
}
}
} else {
log.error("Unable to create/save the feed {}. Unable to find a template for id {}", feedName, templateId);
throw new FeedCreationException("Unable to create the feed [" + feedName + "]. Unable to find a template with id " + templateId);
}
log.info("Time save Feed flow in NiFi. ElapsedTime: {} ms", eventTime(totalTime));
return newProcessGroup;
} catch (NifiClientRuntimeException e) {
throw new FeedCreationException("Unable to create the feed [" + feedName + "]. " + e.getMessage(), e);
}
}
use of com.thinkbiganalytics.nifi.feedmgr.FeedCreationException in project kylo by Teradata.
the class CreateFeedBuilder method createProcessGroupForFeed.
private ProcessGroupDTO createProcessGroupForFeed() throws FeedCreationException {
Stopwatch stopwatch = Stopwatch.createStarted();
// create Category Process group
this.categoryGroup = niFiObjectCache.getCategoryProcessGroup(category);
if (categoryGroup == null) {
try {
ProcessGroupDTO group = restClient.createProcessGroup(category);
this.categoryGroup = group;
this.newCategory = true;
if (this.categoryGroup != null) {
niFiObjectCache.addCategoryProcessGroup(this.categoryGroup);
}
} catch (Exception e) {
// Swallow exception... it will be handled later
}
}
if (this.categoryGroup == null) {
throw new FeedCreationException("Unable to get or create the Process group for the Category " + category + ". Error occurred while creating instance of template " + templateId + " for Feed " + feedName);
}
stopwatch.stop();
log.debug("Time to get/create Category Process Group:{} was: {} ms", category, stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
stopwatch.start();
// 1 create the processGroup
// check to see if the feed exists... if so version off the old group and create a new group with this feed
ProcessGroupDTO feedGroup = restClient.getProcessGroupByName(this.categoryGroup.getId(), feedName);
stopwatch.stop();
log.debug("Time to find feed Process Group: {} was: {} ms", feedName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
if (feedGroup != null) {
try {
previousFeedProcessGroup = feedGroup;
templateCreationHelper.versionProcessGroup(feedGroup, versionIdentifier());
} catch (Exception e) {
throw new FeedCreationException("Previous version of the feed " + feedName + " was found. Error in attempting to version the previous feed. Please go into Nifi and address any issues with the Feeds Process Group", e);
}
}
ProcessGroupDTO group = restClient.createProcessGroup(this.categoryGroup.getId(), feedName);
return group;
}
Aggregations