use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.
the class TemplateInstanceCreator method createTemplate.
public NifiProcessGroup createTemplate() throws TemplateCreationException {
try {
NifiProcessGroup newProcessGroup = null;
TemplateDTO template = restClient.getTemplateById(templateId);
VersionedProcessGroup versionedProcessGroup = null;
if (template != null) {
TemplateCreationHelper templateCreationHelper = new TemplateCreationHelper(this.restClient);
templateCreationHelper.setTemplateProperties(templateProperties);
String processGroupId = null;
ProcessGroupDTO group = null;
if (isCreateReusableFlow()) {
log.info("Creating a new Reusable flow instance for template: {} ", template.getName());
// 1 get/create the parent "reusable_templates" processgroup
ProcessGroupDTO reusableParentGroup = restClient.getProcessGroupByName("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
if (reusableParentGroup == null) {
reusableParentGroup = restClient.createProcessGroup("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
}
ProcessGroupDTO thisGroup = restClient.getProcessGroupByName(reusableParentGroup.getId(), template.getName());
if (thisGroup != null) {
// version the group
log.info("A previous Process group of with this name {} was found. Versioning it before continuing", thisGroup.getName());
versionedProcessGroup = templateCreationHelper.versionProcessGroup(thisGroup, this.getVersionIdentifier());
}
group = restClient.createProcessGroup(reusableParentGroup.getId(), template.getName());
} else {
String tmpName = template.getName() + "_" + System.currentTimeMillis();
group = restClient.createProcessGroup(tmpName);
log.info("Creating a temporary process group with name {} for template {} ", tmpName, template.getName());
}
processGroupId = group.getId();
if (StringUtils.isNotBlank(processGroupId)) {
// snapshot the existing controller services
templateCreationHelper.snapshotControllerServiceReferences();
log.info("Successfully Snapshot of controller services");
// create the flow from the template
TemplateInstance instance = templateCreationHelper.instantiateFlowFromTemplate(processGroupId, templateId);
FlowSnippetDTO flowSnippetDTO = instance.getFlowSnippetDTO();
log.info("Successfully created the temp flow");
if (this.createReusableFlow) {
ensureInputPortsForReuseableTemplate(processGroupId);
log.info("Reusable flow, input ports created successfully.");
}
// mark the new services that were created as a result of creating the new flow from the template
templateCreationHelper.identifyNewlyCreatedControllerServiceReferences(instance);
ProcessGroupDTO entity = restClient.getProcessGroup(processGroupId, true, true);
// replace static properties and inject values into the flow
List<NifiProperty> processorProperties = NifiPropertyUtil.getProperties(entity, restClient.getPropertyDescriptorTransform());
if (processorProperties != null) {
boolean didReplace = false;
for (NifiProperty property : processorProperties) {
boolean replaced = ConfigurationPropertyReplacer.resolveStaticConfigurationProperty(property, staticConfigPropertyMap);
if (replaced) {
// update the properties that are replaced
if (property.getPropertyDescriptor() != null && StringUtils.isNotBlank(property.getPropertyDescriptor().getIdentifiesControllerService())) {
// verify the property is a valid cs property
String value = property.getValue();
if (templateCreationHelper.getEnabledServiceNameMap().containsKey(value)) {
property.setValue(templateCreationHelper.getEnabledServiceNameMap().get(value).get(0).getId());
}
}
restClient.updateProcessorProperty(property.getProcessGroupId(), property.getProcessorId(), property);
didReplace = true;
}
}
// if we replaced any properties, refetch to get the latest data
if (didReplace) {
entity = restClient.getProcessGroup(processGroupId, true, true);
}
}
// identify the various processors (first level initial processors)
List<ProcessorDTO> inputProcessors = NifiProcessUtil.getInputProcessors(entity);
ProcessorDTO input = null;
List<ProcessorDTO> nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
// if the input is null attempt to get the first input available on the template
if (input == null && inputProcessors != null && !inputProcessors.isEmpty()) {
input = inputProcessors.get(0);
}
log.info("Attempt to update/validate controller services for template.");
// update any references to the controller services and try to assign the value to an enabled service if it is not already
boolean updatedControllerServices = false;
if (input != null) {
log.info("attempt to update controllerservices on {} input processor ", input.getName());
List<NifiProperty> updatedProperties = templateCreationHelper.updateControllerServiceReferences(Lists.newArrayList(inputProcessors), staticConfigPropertyStringMap, instance);
updatedControllerServices = !updatedProperties.isEmpty();
}
log.info("attempt to update controllerservices on {} processors ", (nonInputProcessors != null ? nonInputProcessors.size() : 0));
List<NifiProperty> updatedProperties = templateCreationHelper.updateControllerServiceReferences(nonInputProcessors, staticConfigPropertyStringMap, instance);
if (!updatedControllerServices) {
updatedControllerServices = !updatedProperties.isEmpty();
}
log.info("Controller service validation complete");
// refetch processors for updated errors
entity = restClient.getProcessGroup(processGroupId, true, true);
nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
inputProcessors = NifiProcessUtil.getInputProcessors(entity);
if (inputProcessors != null && !inputProcessors.isEmpty()) {
input = inputProcessors.get(0);
}
// /make the input/output ports in the category group as running
if (isCreateReusableFlow()) {
log.info("Reusable flow, attempt to mark the connection ports as running.");
templateCreationHelper.startProcessGroupAndParentInputPorts(entity);
// templateCreationHelper.markConnectionPortsAsRunning(entity);
log.info("Reusable flow. Successfully marked the ports as running.");
}
newProcessGroup = new NifiProcessGroup(entity, input, nonInputProcessors);
newProcessGroup.setVersionedProcessGroup(versionedProcessGroup);
newProcessGroup.setReusableFlowInstance(isCreateReusableFlow());
if (isCreateReusableFlow()) {
// call listeners notify of before mark as running processing
if (creationCallback != null) {
try {
creationCallback.beforeMarkAsRunning(template.getName(), entity);
} catch (Exception e) {
log.error("Error calling callback beforeMarkAsRunning ", e);
}
}
log.info("Reusable flow, attempt to mark the Processors as running.");
templateCreationHelper.markProcessorsAsRunning(newProcessGroup);
log.info("Reusable flow. Successfully marked the Processors as running.");
// align items
AlignProcessGroupComponents alignProcessGroupComponents = new AlignProcessGroupComponents(restClient.getNiFiRestClient(), entity.getParentGroupId());
alignProcessGroupComponents.autoLayout();
}
templateCreationHelper.cleanupControllerServices();
log.info("Controller service cleanup complete");
List<NifiError> errors = templateCreationHelper.getErrors();
// add any global errors to the object
if (errors != null && !errors.isEmpty()) {
for (NifiError error : errors) {
newProcessGroup.addError(error);
}
}
newProcessGroup.setSuccess(!newProcessGroup.hasFatalErrors());
if (!newProcessGroup.isSuccess()) {
log.info("Errors while importing the template. {} errors found. {}", (errors != null ? errors.size() : 0), (errors != null ? " - " + StringUtils.join(errors, ",") : ""));
} else {
log.info("Success. Finished importing template ");
}
return newProcessGroup;
}
}
} catch (final Exception e) {
if (log.isDebugEnabled() && e instanceof WebApplicationException) {
final Response response = ((WebApplicationException) e).getResponse();
log.debug("NiFi server returned error: {}", response.readEntity(String.class), e);
}
throw new TemplateCreationException("Unable to create the template for the Id of [" + templateId + "]. " + e.getMessage(), e);
}
return null;
}
use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.
the class LegacyNifiRestClient method getTemplatesAsXmlMatchingInputPortName.
/**
* Returns a Map of the Template Name and the Template XML that has a matching Input Port in the template.
*/
@Deprecated
public Map<String, String> getTemplatesAsXmlMatchingInputPortName(final String inputPortName) {
final Set<TemplateDTO> templates = client.templates().findByInputPortName(inputPortName);
final Map<String, String> result = new HashMap<>(templates.size());
for (TemplateDTO template : templates) {
client.templates().download(template.getId()).ifPresent(xml -> result.put(template.getId(), xml));
}
return result;
}
use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.
the class TemplatesRestController method getTemplates.
/**
* This will list all the templates registered in Kylo
*/
@GET
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets the list of all templates.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the templates.", response = TemplateDtoWrapper.class, responseContainer = "Set"), @ApiResponse(code = 500, message = "NiFi is unavailable.", response = RestResponseStatus.class) })
public Response getTemplates(@QueryParam("includeDetails") boolean includeDetails) {
Set<TemplateDTO> nifiTemplates = nifiRestClient.getTemplates(includeDetails);
Set<TemplateDtoWrapper> dtos = new HashSet<>();
List<RegisteredTemplate> registeredTemplates = registeredTemplateService.getRegisteredTemplates();
for (final TemplateDTO dto : nifiTemplates) {
RegisteredTemplate match = registeredTemplates.stream().filter(template -> template.getNifiTemplateId().equalsIgnoreCase(dto.getId()) || template.getTemplateName().equalsIgnoreCase(dto.getName())).findFirst().orElse(null);
TemplateDtoWrapper wrapper = new TemplateDtoWrapper(dto);
if (match != null) {
wrapper.setRegisteredTemplateId(match.getId());
}
dtos.add(wrapper);
}
return Response.ok(dtos).build();
}
use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.
the class CreateFeedBuilder method build.
/**
* Build the NiFi flow instance
*
* @return an object indicating if the feed flow was successfully built or not
*/
public NifiProcessGroup build() throws FeedCreationException {
try {
log.info("Creating the feed {}.{} ", category, feedName);
newProcessGroup = null;
Stopwatch totalTime = Stopwatch.createStarted();
Stopwatch eventTime = Stopwatch.createStarted();
TemplateDTO template = getTemplate();
if (template != null) {
log.debug("Time to get Template {}. ElapsedTime: {} ms", template.getName(), eventTime(eventTime));
// create the encompassing process group
eventTime.start();
ProcessGroupDTO feedProcessGroup = createProcessGroupForFeed();
log.debug("Time to create process group. ElapsedTime: {} ms", eventTime(eventTime));
if (feedProcessGroup != null) {
String processGroupId = feedProcessGroup.getId();
// snapshot the existing controller services
eventTime.start();
templateCreationHelper.snapshotControllerServiceReferences();
log.debug("Time to snapshotControllerServices. ElapsedTime: {} ms", eventTime(eventTime));
// create the flow from the template
eventTime.start();
TemplateInstance instance = templateCreationHelper.instantiateFlowFromTemplate(processGroupId, templateId);
FlowSnippetDTO feedInstance = instance.getFlowSnippetDTO();
feedProcessGroup.setContents(feedInstance);
log.debug("Time to instantiateFlowFromTemplate. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
String feedCategoryId = feedProcessGroup.getParentGroupId();
ProcessGroupDTO categoryGroup = this.categoryGroup;
if (categoryGroup == null) {
categoryGroup = this.categoryGroup = restClient.getProcessGroup(feedCategoryId, false, false);
}
// update the group with this template?
updatePortConnectionsForProcessGroup(feedProcessGroup, categoryGroup);
log.debug("Time to updatePortConnectionsForProcessGroup. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// mark the new services that were created as a result of creating the new flow from the template
templateCreationHelper.identifyNewlyCreatedControllerServiceReferences(instance);
log.debug("Time to identifyNewlyCreatedControllerServiceReferences. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// match the properties incoming to the defined properties
updateProcessGroupProperties(processGroupId, feedProcessGroup.getName());
log.debug("Time to updateProcessGroupProperties. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// Fetch the Feed Group now that it has the flow in it
ProcessGroupDTO entity = restClient.getProcessGroup(processGroupId, true, true);
log.debug("Time to getProcessGroup. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
ProcessorDTO input = fetchInputProcessorForProcessGroup(entity);
ProcessorDTO cleanupProcessor = NifiProcessUtil.findFirstProcessorsByType(NifiProcessUtil.getInputProcessors(entity), "com.thinkbiganalytics.nifi.v2.metadata.TriggerCleanup");
List<ProcessorDTO> nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
log.debug("Time to fetchInputProcessorForProcessGroup. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
List<NifiProperty> updatedControllerServiceProperties = new ArrayList<>();
// update any references to the controller services and try to assign the value to an enabled service if it is not already
if (input != null) {
updatedControllerServiceProperties.addAll(templateCreationHelper.updateControllerServiceReferences(Lists.newArrayList(input), instance));
}
if (cleanupProcessor != null) {
updatedControllerServiceProperties.addAll(templateCreationHelper.updateControllerServiceReferences(Collections.singletonList(cleanupProcessor), instance));
}
updatedControllerServiceProperties.addAll(templateCreationHelper.updateControllerServiceReferences(nonInputProcessors, instance));
log.debug("Time to updatedControllerServiceProperties. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// refetch processors for updated errors
entity = restClient.getProcessGroup(processGroupId, true, true);
input = fetchInputProcessorForProcessGroup(entity);
nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
newProcessGroup = new NifiProcessGroup(entity, input, nonInputProcessors);
log.debug("Time to re-fetchInputProcessorForProcessGroup. ElapsedTime: {} ms", eventTime(eventTime));
// Validate and if invalid Delete the process group
if (newProcessGroup.hasFatalErrors()) {
eventTime.start();
removeProcessGroup(entity);
// cleanupControllerServices();
newProcessGroup.setSuccess(false);
log.debug("Time to removeProcessGroup. Errors found. ElapsedTime: {} ms", eventTime(eventTime));
} else {
eventTime.start();
// update the input schedule
updateFeedSchedule(newProcessGroup, input);
log.debug("Time to update feed schedule. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// just need to update for this processgroup
Collection<ProcessorDTO> processors = NifiProcessUtil.getProcessors(entity);
Collection<ConnectionDTO> connections = NifiConnectionUtil.getAllConnections(entity);
nifiFlowCache.updateFlowForFeed(feedMetadata, entity.getId(), processors, connections);
log.debug("Time to build flow graph with {} processors and {} connections. ElapsedTime: {} ms", processors.size(), connections.size(), eventTime(eventTime));
/*
//Cache the processorIds to the respective flowIds for availability in the ProvenanceReportingTask
NifiVisitableProcessGroup group = nifiFlowCache.getFlowOrder(newProcessGroup.getProcessGroupEntity(), true);
log.debug("Time to get the flow order. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
NifiFlowProcessGroup
flow =
new NifiFlowBuilder().build(
group);
log.debug("Time to build flow graph with {} processors. ElapsedTime: {} ms", flow.getProcessorMap().size(), eventTime(eventTime));
eventTime.start();
nifiFlowCache.updateFlow(feedMetadata, flow);
log.debug("Time to update NiFiFlowCache with {} processors. ElapsedTime: {} ms", flow.getProcessorMap().size(), eventTime(eventTime));
*/
eventTime.start();
// disable all inputs
restClient.disableInputProcessors(newProcessGroup.getProcessGroupEntity().getId());
log.debug("Time to disableInputProcessors. ElapsedTime: {} ms", eventTime(eventTime));
eventTime.start();
// mark everything else as running
templateCreationHelper.markProcessorsAsRunning(newProcessGroup);
log.debug("Time to markNonInputsAsRunning. ElapsedTime: {} ms", eventTime(eventTime));
// if desired start the input processor
if (input != null) {
eventTime.start();
if (enabled) {
markInputAsRunning(newProcessGroup, input);
// /make the input/output ports in the category group as running
if (hasConnectionPorts()) {
templateCreationHelper.markConnectionPortsAsRunning(entity);
}
} else {
// /make the input/output ports in the category group as running
if (hasConnectionPorts()) {
templateCreationHelper.markConnectionPortsAsRunning(entity);
}
markInputAsStopped(newProcessGroup, input);
}
log.debug("Time to mark input as {}. ElapsedTime: {} ms", (enabled ? "Running" : "Stopped"), eventTime(eventTime));
}
if (newProcessGroup.hasFatalErrors()) {
eventTime.start();
rollback();
newProcessGroup.setRolledBack(true);
// cleanupControllerServices();
newProcessGroup.setSuccess(false);
log.debug("Time to rollback on Fatal Errors. ElapsedTime: {} ms", eventTime(eventTime));
}
List<NifiError> templateCreationErrors = templateCreationHelper.getErrors();
if (templateCreationErrors != null) {
errors.addAll(templateCreationErrors);
}
// add any global errors to the object
if (errors != null && !errors.isEmpty()) {
for (NifiError error : errors) {
newProcessGroup.addError(error);
if (error.isFatal()) {
newProcessGroup.setSuccess(false);
if (!newProcessGroup.isRolledBack()) {
rollback();
newProcessGroup.setRolledBack(true);
}
}
}
}
}
eventTime.start();
templateCreationHelper.cleanupControllerServices();
// fix the feed metadata controller service references
updateFeedMetadataControllerServiceReferences(updatedControllerServiceProperties);
log.debug("Time cleanup controller services. ElapsedTime: {} ms", eventTime(eventTime));
// align items
if (this.autoAlign) {
eventTime.start();
log.info("Aligning Feed flows in NiFi ");
AlignProcessGroupComponents alignProcessGroupComponents = new AlignProcessGroupComponents(restClient.getNiFiRestClient(), entity.getParentGroupId());
alignProcessGroupComponents.autoLayout();
// fetch the parent to get that id to align
if (newCategory) {
log.info("This is the first feed created in the category {}. Aligning the categories. ", feedMetadata.getCategory().getSystemName());
new AlignProcessGroupComponents(restClient.getNiFiRestClient(), this.categoryGroup.getParentGroupId()).autoLayout();
}
log.info("Time align feed process groups. ElapsedTime: {} ms", eventTime(eventTime));
} else {
log.info("Skipping auto alignment in NiFi. You can always manually align this category and all of its feeds by using the rest api: /v1/feedmgr/nifi/auto-align/{}", entity.getParentGroupId());
if (newCategory) {
log.info("To re align the categories: /v1/feedmgr/nifi/auto-align/{}", this.categoryGroup.getParentGroupId());
}
}
}
} else {
log.error("Unable to create/save the feed {}. Unable to find a template for id {}", feedName, templateId);
throw new FeedCreationException("Unable to create the feed [" + feedName + "]. Unable to find a template with id " + templateId);
}
log.info("Time save Feed flow in NiFi. ElapsedTime: {} ms", eventTime(totalTime));
return newProcessGroup;
} catch (NifiClientRuntimeException e) {
throw new FeedCreationException("Unable to create the feed [" + feedName + "]. " + e.getMessage(), e);
}
}
use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.
the class NiFiTemplateCache method ensureNifiTemplate.
/**
* Return the NiFi {@link TemplateDTO} object fully populated and sets this to the incoming {@link RegisteredTemplate#nifiTemplate}
* If at first looking at the {@link RegisteredTemplate#nifiTemplateId} it is unable to find the template it will then fallback and attempt to find the template by its name
*
* @param registeredTemplate a registered template object
* @return the NiFi template
*/
public TemplateDTO ensureNifiTemplate(RegisteredTemplate registeredTemplate) {
if (registeredTemplate.getNifiTemplate() == null) {
TemplateDTO templateDTO = geTemplate(registeredTemplate.getNifiTemplateId(), registeredTemplate.getTemplateName());
if (templateDTO != null) {
registeredTemplate.setNifiTemplate(templateDTO);
registeredTemplate.setNifiTemplateId(registeredTemplate.getNifiTemplate().getId());
}
}
return registeredTemplate.getNifiTemplate();
}
Aggregations