Examples with TemplateDTO org.apache.nifi.web.api.dto.TemplateDTO used on opensource projects

Example 31 with TemplateDTO

use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.

the class DefaultTemplateExporter method export.

private ExportTemplate export(String templateId) {
    RegisteredTemplate template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(templateId).nifiTemplateId(templateId).includeSensitiveProperties(true).build());
    if (template != null) {
        List<String> connectingReusableTemplates = new ArrayList<>();
        Set<String> connectedTemplateIds = new HashSet<>();
        Set<ReusableTemplateConnectionInfo> outputPortConnectionMetadata = new HashSet<>();
        Set<RemoteProcessGroupInputPort> templateRemoteInputPorts = new HashSet<>();
        if (template.usesReusableTemplate()) {
            ProcessGroupFlowDTO reusableTemplateFlow = templateConnectionUtil.getReusableTemplateCategoryProcessGroupFlow();
            List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos = template.getReusableTemplateConnections();
            Map<String, PortDTOWithGroupInfo> reusableTemplatePorts = templateConnectionUtil.getReusableFeedInputPorts(reusableTemplateFlow).stream().collect(Collectors.toMap(port -> port.getName(), port -> port));
            reusableTemplateConnectionInfos.stream().filter(connectionInfo -> StringUtils.isBlank(connectionInfo.getReusableTemplateProcessGroupName())).forEach(connectionInfo -> {
                PortDTOWithGroupInfo port = reusableTemplatePorts.get(connectionInfo.getReusableTemplateInputPortName());
                if (port != null) {
                    connectionInfo.setReusableTemplateProcessGroupName(port.getDestinationProcessGroupName());
                }
            });
            // Get flow information for the 'reusable_templates' process group in NiFi
            if (reusableTemplateFlow != null) {
                gatherConnectedReusableTemplates(connectingReusableTemplates, connectedTemplateIds, outputPortConnectionMetadata, reusableTemplateConnectionInfos, reusableTemplateFlow);
            }
            // Only gather remote input ports on the reusable templates if we are clustered
            NiFiClusterSummary clusterSummary = nifiRestClient.getNiFiRestClient().clusterSummary();
            if (clusterSummary.getClustered()) {
                // for all the reusable templates used gather any that have remote input ports
                reusableTemplateConnectionInfos.stream().forEach(connectionInfo -> {
                    Set<RemoteProcessGroupInputPort> remoteProcessGroupInputPorts = findReusableTemplateRemoteInputPorts(reusableTemplateFlow, connectionInfo.getReusableTemplateProcessGroupName());
                    templateRemoteInputPorts.addAll(remoteProcessGroupInputPorts);
                });
            }
        }
        String templateXml = null;
        try {
            if (template != null) {
                try {
                    templateXml = nifiRestClient.getTemplateXml(template.getNifiTemplateId());
                } catch (NifiClientRuntimeException e) {
                    TemplateDTO templateDTO = nifiRestClient.getTemplateByName(template.getTemplateName());
                    if (templateDTO != null) {
                        templateXml = nifiRestClient.getTemplateXml(templateDTO.getId());
                    }
                }
            }
        } catch (Exception e) {
            throw new UnsupportedOperationException("Unable to find Nifi Template for " + templateId);
        }
        // create a zip file with the template and xml
        byte[] zipFile = zip(template, templateXml, connectingReusableTemplates, outputPortConnectionMetadata, templateRemoteInputPorts);
        return new ExportTemplate(SystemNamingService.generateSystemName(template.getTemplateName()) + ".template.zip", zipFile);
    } else {
        throw new UnsupportedOperationException("Unable to find Template for " + templateId);
    }
}

Example 32 with TemplateDTO

use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.

the class AbstractImportTemplateRoutine method restoreOldTemplateXml.

protected UploadProgressMessage restoreOldTemplateXml() {
    UploadProgressMessage rollbackMessage = uploadProgressService.addUploadStatus(importTemplate.getImportOptions().getUploadKey(), "Attempting to rollback the template xml in NiFi " + importTemplate.getTemplateName());
    if (niFiTemplateImport != null) {
        TemplateDTO dto = niFiTemplateImport.getDto();
        String oldTemplateXml = niFiTemplateImport.getOldTemplateXml();
        log.error("ERROR! This template is NOT VALID Nifi Template: {} for file {}.  Errors are: {} ", importTemplate.getTemplateName(), importTemplate.getFileName(), importTemplate.getTemplateResults().getAllErrors());
        // delete this template
        importTemplate.setSuccess(false);
        // delete the template from NiFi
        nifiRestClient.deleteTemplate(dto.getId());
        // restore old template
        if (oldTemplateXml != null) {
            UploadProgressMessage progressMessage = uploadProgressService.addUploadStatus(importTemplate.getImportOptions().getUploadKey(), "Attempting to restore old template xml for: " + importTemplate.getTemplateName());
            log.info("Rollback Nifi: Attempt to restore old template xml ");
            nifiRestClient.importTemplate(oldTemplateXml);
            log.info("Rollback Nifi: restored old template xml ");
            progressMessage.update("Rollback Status: Restored old template xml for " + importTemplate.getTemplateName(), true);
        }
    }
    return rollbackMessage;
}

Example 33 with TemplateDTO

use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.

the class ImportFeedTemplateXml method create.

@Override
public NifiProcessGroup create(NiFiTemplateImport niFiTemplateImport, UploadProgressMessage importStatusMessage) {
    TemplateDTO dto = niFiTemplateImport.getDto();
    String templateName = importTemplate.getTemplateName();
    String fileName = importTemplate.getFileName();
    importStatusMessage.update("Importing the NiFi flow, " + templateName);
    log.info("validate NiFi Flow by creating a template instance in Nifi. Template: {} for file {}", templateName, fileName);
    Map<String, Object> configProperties = propertyExpressionResolver.getStaticConfigProperties();
    List<NifiProperty> templateProperties = importTemplate.getTemplateToImport() != null ? importTemplate.getTemplateToImport().getProperties() : Collections.emptyList();
    NifiProcessGroup newTemplateInstance = nifiRestClient.createNewTemplateInstance(dto.getId(), templateProperties, configProperties, false, null, importTemplate.getVersionIdentifier());
    importTemplate.setTemplateResults(newTemplateInstance);
    return newTemplateInstance;
}

Example 34 with TemplateDTO

use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.

the class AbstractValidateImportTemplate method validateNiFiTemplateImport.

/**
 * Validate the NiFi template is valid.  This method will validate the template can be created/overwritten based upon the user supplied properties
 */
public void validateNiFiTemplateImport() {
    // ImportOptions options = template.getImportOptions();
    ImportComponentOption nifiTemplateOption = this.importTemplateOptions.findImportComponentOption(ImportComponent.NIFI_TEMPLATE);
    // if the options of the TEMPLATE_DATA are marked to import and overwrite this should be as well
    ImportComponentOption templateData = this.importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA);
    if (templateData.isUserAcknowledged()) {
        nifiTemplateOption.setUserAcknowledged(true);
    }
    if (templateData.isShouldImport()) {
        nifiTemplateOption.setShouldImport(true);
    }
    if (templateData.isOverwrite()) {
        nifiTemplateOption.setOverwrite(true);
    }
    if (nifiTemplateOption.isShouldImport()) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Validating the NiFi template");
        String templateName = null;
        TemplateDTO dto = null;
        try {
            templateName = NifiTemplateParser.getTemplateName(this.importTemplate.getNifiTemplateXml());
            this.importTemplate.setTemplateName(templateName);
            dto = nifiRestClient.getNiFiRestClient().templates().findByName(templateName).orElse(null);
            if (dto != null) {
                this.importTemplate.setNifiTemplateId(dto.getId());
                // if the template incoming is an XML template and it already exists, or if its a zip file and it exists and the user has not acknowledge to overwrite then error out
                if ((!this.importTemplateOptions.isUserAcknowledged(ImportComponent.NIFI_TEMPLATE) || this.importTemplateOptions.isUserAcknowledged(ImportComponent.NIFI_TEMPLATE) && !this.importTemplate.isZipFile()) && !this.importTemplateOptions.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE) && !this.importTemplateOptions.isContinueIfExists(ImportComponent.NIFI_TEMPLATE)) {
                    this.importTemplate.setValid(false);
                    String msg = "Unable to import Template " + templateName + ".  It already exists in NiFi.";
                    this.importTemplate.getImportOptions().addErrorMessage(ImportComponent.NIFI_TEMPLATE, msg);
                    statusMessage.update("Validation Error: Unable to import Template " + templateName + ".  It already exists in NiFi.");
                    statusMessage.complete(false);
                } else {
                    statusMessage.update("Validated the NiFi template. ");
                    statusMessage.complete(true);
                }
            } else {
                statusMessage.update("Validated the NiFi template.  The template " + templateName + " will be created in NiFi");
                statusMessage.complete(true);
            }
        } catch (ParserConfigurationException | XPathExpressionException | IOException | SAXException e) {
            getLogger().error("Error validating the file {} for import ", fileName, e);
            this.importTemplate.setValid(false);
            this.importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, "The xml file you are trying to import is not a valid NiFi template.  Please try again. " + e.getMessage(), "");
            statusMessage.complete(false);
        }
        nifiTemplateOption.setValidForImport(!nifiTemplateOption.hasErrorMessages());
    }
    uploadProgressService.completeSection(importTemplateOptions, ImportSection.Section.VALIDATE_NIFI_TEMPLATE);
}

Example 35 with TemplateDTO

use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.

the class NifiRestTest method testLoad.

// @Test
public void testLoad() {
    // setup constants for the test
    String templateName = "Data Ingest";
    int num = 10;
    String processGroupName = "LoadTest";
    String feedPrefix = "LT_";
    String inputType = "org.apache.nifi.processors.standard.GetFile";
    List<NifiProperty> templateProperties = new ArrayList<>();
    String schedulePeriod = "10 sec";
    String GET_FILE_PROCESSOR_NAME = "Poll filesystem";
    String UPDATE_PARAMETERS_PROCESSOR_NAME = "Update flow parameters";
    String INPUT_DIRECTORY_PROPERTY = "Input Directory";
    String SOURCE_PROPERTY = "source";
    String ENTITY_PROPERTY = "entity";
    try {
        TemplateDTO template = restClient.getTemplateByName(templateName);
        List<NifiProperty> propertyList = restClient.getPropertiesForTemplate(template.getId(), true);
        NifiProperty inputDirectory = NifiPropertyUtil.getProperty(GET_FILE_PROCESSOR_NAME, INPUT_DIRECTORY_PROPERTY, propertyList);
        NifiProperty entity = NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, SOURCE_PROPERTY, propertyList);
        NifiProperty source = NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, ENTITY_PROPERTY, propertyList);
        templateProperties.add(inputDirectory);
        templateProperties.add(entity);
        templateProperties.add(source);
        NifiProcessorSchedule schedule = new NifiProcessorSchedule();
        schedule.setSchedulingStrategy("TIMER_DRIVEN");
        schedule.setSchedulingPeriod(schedulePeriod);
        for (int i = 0; i < num; i++) {
            String feedName = feedPrefix + i;
            List<NifiProperty> instanceProperties = NifiPropertyUtil.copyProperties(templateProperties);
            // update the properties
            NifiPropertyUtil.getProperty(GET_FILE_PROCESSOR_NAME, INPUT_DIRECTORY_PROPERTY, instanceProperties).setValue("/tmp/" + feedName);
            NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, SOURCE_PROPERTY, instanceProperties).setValue(processGroupName);
            NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, ENTITY_PROPERTY, instanceProperties).setValue(feedName);
            FeedMetadata feedMetadata = new FeedMetadata();
            feedMetadata.setCategory(new FeedCategory());
            feedMetadata.getCategory().setSystemName(processGroupName);
            feedMetadata.setSystemFeedName("feedPrefix + i");
            CreateFeedBuilder.newFeed(restClient, nifiFlowCache, feedMetadata, template.getId(), new PropertyExpressionResolver(), propertyDescriptorTransform, createFeedBuilderCache, templateConnectionUtil).inputProcessorType(inputType).feedSchedule(schedule).properties(instanceProperties).build();
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}