Examples with ImportTemplate com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate used on opensource projects

Example 11 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class ImportTemplateArchive method importTemplate.

public boolean importTemplate() {
    UploadProgressMessage statusMessage = null;
    // Get information about the import
    RegisteredTemplate template = importTemplate.getTemplateToImport();
    // validateTemplateProperties(template, importTemplate, importOptions);
    // 1 ensure this template doesnt already exist
    importTemplate.setTemplateName(template.getTemplateName());
    RegisteredTemplate existingTemplate = registeredTemplateService.findRegisteredTemplate(RegisteredTemplateRequest.requestByTemplateName(template.getTemplateName()));
    if (existingTemplate != null) {
        template.setId(existingTemplate.getId());
    } else {
        template.setId(null);
    }
    // first we just import the reusable templates as flows
    this.importedReusableTemplates = importReusableTemplateInArchive(importTemplate, this.importTemplateOptions);
    // after the templates are created we then connect the templates
    if (!this.importTemplateOptions.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).hasErrorMessages()) {
        if (!importedReusableTemplates.isEmpty()) {
            connectReusableTemplates();
        } else {
            importTemplate.setSuccess(true);
        }
        if (importTemplate.isSuccess()) {
            RegisteredTemplate newTemplate = importFeedTemplate(existingTemplate);
            if (importTemplate.isSuccess()) {
                if (newTemplate != null) {
                    validateInstance();
                }
                if (!importedReusableTemplates.isEmpty()) {
                    importedReusableTemplates.stream().filter(importReusableTemplate -> importReusableTemplate.getImportTemplate().isSuccess()).map(t -> t.getImportTemplate()).forEach(connectingTemplate -> {
                        nifiRestClient.markConnectionPortsAsRunning(connectingTemplate.getTemplateResults().getProcessGroupEntity());
                    });
                }
            }
        }
    } else {
        rollback();
        // return if invalid
        return false;
    }
    // cleanup any temp process groups for this template
    if (!importTemplateOptions.isDeferCleanup()) {
        cleanup();
    }
    uploadProgressService.completeSection(importTemplateOptions, ImportSection.Section.IMPORT_REGISTERED_TEMPLATE);
    return importTemplate.isSuccess();
}

Example 12 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class ImportTemplateArchive method importReusableTemplateInArchive.

/**
 * Called when another Template or Feed uses a Reusable template
 */
private List<ImportReusableTemplate> importReusableTemplateInArchive(ImportTemplate importTemplate, ImportTemplateOptions importOptions) {
    List<ImportReusableTemplate> connectingTemplates = new ArrayList<>();
    // start the import
    ImportComponentOption reusableComponentOption = importOptions.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE);
    if (importTemplate.hasConnectingReusableTemplate() && reusableComponentOption.isShouldImport() && reusableComponentOption.isValidForImport()) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "Import Reusable Template. Starting import");
        // import the reusable templates
        boolean valid = true;
        ImportTemplate lastReusableTemplate = null;
        log.info("Importing Zip file template {}. first importing reusable flow from zip");
        try {
            for (String reusableTemplateXml : importTemplate.getNifiConnectingReusableTemplateXmls()) {
                String name = NifiTemplateParser.getTemplateName(reusableTemplateXml);
                byte[] content = reusableTemplateXml.getBytes("UTF-8");
                ImportReusableTemplate importReusableTemplate = importReusableTemplateFactory.apply(name, content, importOptions);
                boolean validReusableTemplate = importReusableTemplate.importIntoNiFiAndCreateInstance();
                connectingTemplates.add(importReusableTemplate);
                if (!validReusableTemplate) {
                    ImportTemplate connectingTemplate = importReusableTemplate.importTemplate;
                    // add in the error messages
                    connectingTemplate.getTemplateResults().getAllErrors().stream().forEach(nifiError -> {
                        importTemplate.getTemplateResults().addError(nifiError);
                    });
                    // error out
                    importTemplate.setSuccess(false);
                    reusableComponentOption.getErrorMessages().add("Error importing Reusable Template");
                    // exit
                    valid = false;
                    break;
                }
            }
            if (valid) {
                statusMessage.update("Successfully imported Reusable Templates " + (connectingTemplates.stream().map(t -> t.getImportTemplate().getTemplateName()).collect(Collectors.joining(","))), true);
            } else {
                statusMessage.update("Errors importing reusable template: Imported Reusable Template. " + lastReusableTemplate != null ? lastReusableTemplate.getTemplateName() : "");
            }
        } catch (Exception e) {
            log.error("Error importing reusable template from archive {}.  {} ", importTemplate.getFileName(), lastReusableTemplate != null ? lastReusableTemplate.getTemplateName() : "");
            importTemplate.setSuccess(false);
        }
    }
    uploadProgressService.completeSection(importOptions, ImportSection.Section.IMPORT_REUSABLE_TEMPLATE);
    return connectingTemplates;
}

Example 13 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class ImportUtil method getNewNiFiTemplateImport.

public static ImportTemplate getNewNiFiTemplateImport(String fileName, InputStream inputStream) throws IOException {
    ImportTemplate template = new ImportTemplate(fileName);
    template.setValid(true);
    StringWriter writer = new StringWriter();
    IOUtils.copy(inputStream, writer, "UTF-8");
    String xmlTemplate = writer.toString();
    template.setNifiTemplateXml(xmlTemplate);
    return template;
}

Example 14 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class ImportUtil method applyImportPropertiesToFeed.

public static boolean applyImportPropertiesToFeed(FeedMetadata metadata, ImportFeed importFeed, ImportComponent component) {
    ImportComponentOption option = importFeed.getImportOptions().findImportComponentOption(component);
    if (!option.getProperties().isEmpty() && option.getProperties().stream().anyMatch(importProperty -> StringUtils.isBlank(importProperty.getPropertyValue()))) {
        importFeed.setSuccess(false);
        if (importFeed.getTemplate() == null) {
            ImportTemplate importTemplate = new ImportTemplate(importFeed.getFileName());
            importFeed.setTemplate(importTemplate);
        }
        String feedCategory = importFeed.getImportOptions().getCategorySystemName() != null ? importFeed.getImportOptions().getCategorySystemName() : metadata.getSystemCategoryName();
        String msg = "The feed " + FeedNameUtil.fullName(feedCategory, metadata.getSystemFeedName()) + " needs additional properties to be supplied before importing.";
        importFeed.addErrorMessage(metadata, msg);
        option.getErrorMessages().add(msg);
        return false;
    } else {
        metadata.getSensitiveProperties().forEach(nifiProperty -> {
            ImportProperty userSuppliedValue = importFeed.getImportOptions().getProperties(ImportComponent.FEED_DATA).stream().filter(importFeedProperty -> {
                return nifiProperty.getProcessorId().equalsIgnoreCase(importFeedProperty.getProcessorId()) && nifiProperty.getKey().equalsIgnoreCase(importFeedProperty.getPropertyKey());
            }).findFirst().orElse(null);
            // deal with nulls?
            if (userSuppliedValue != null) {
                nifiProperty.setValue(userSuppliedValue.getPropertyValue());
            }
        });
        return true;
    }
}

Example 15 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class ImportUtil method openZip.

/**
 * Open the zip file and populate the {@link ImportTemplate} object with the components in the file/archive
 *
 * @param fileName    the file name
 * @param inputStream the file
 * @return the template data to import
 */
public static ImportTemplate openZip(String fileName, InputStream inputStream) throws IOException {
    byte[] buffer = new byte[1024];
    ZipInputStream zis = new ZipInputStream(inputStream);
    ZipEntry zipEntry;
    ImportTemplate importTemplate = new ImportTemplate(fileName);
    while ((zipEntry = zis.getNextEntry()) != null) {
        String zipEntryContents = ZipFileUtil.zipEntryToString(buffer, zis, zipEntry);
        if (zipEntry.getName().startsWith(ImportTemplate.NIFI_TEMPLATE_XML_FILE)) {
            importTemplate.setNifiTemplateXml(zipEntryContents);
        } else if (zipEntry.getName().startsWith(ImportTemplate.TEMPLATE_JSON_FILE)) {
            importTemplate.setTemplateJson(zipEntryContents);
        } else if (zipEntry.getName().startsWith(ImportTemplate.NIFI_CONNECTING_REUSABLE_TEMPLATE_XML_FILE)) {
            importTemplate.addNifiConnectingReusableTemplateXml(zipEntryContents);
        } else if (zipEntry.getName().startsWith(ImportTemplate.REUSABLE_TEMPLATE_REMOTE_INPUT_PORT_JSON_FILE)) {
            String json = zipEntryContents;
            List<RemoteProcessGroupInputPort> remoteProcessGroupInputPorts = ObjectMapperSerializer.deserialize(json, new TypeReference<List<RemoteProcessGroupInputPort>>() {
            });
            importTemplate.addRemoteProcessGroupInputPorts(remoteProcessGroupInputPorts);
        } else if (zipEntry.getName().startsWith(ImportTemplate.REUSABLE_TEMPLATE_OUTPUT_CONNECTION_FILE)) {
            String json = zipEntryContents;
            List<ReusableTemplateConnectionInfo> connectionInfos = ObjectMapperSerializer.deserialize(json, new TypeReference<List<ReusableTemplateConnectionInfo>>() {
            });
            importTemplate.addReusableTemplateConnectionInformation(connectionInfos);
        }
    }
    zis.closeEntry();
    zis.close();
    if (!importTemplate.hasValidComponents()) {
        throw new UnsupportedOperationException(" The file you uploaded is not a valid archive.  Please ensure the Zip file has been exported from the system and has 2 valid files named: " + ImportTemplate.NIFI_TEMPLATE_XML_FILE + ", and " + ImportTemplate.TEMPLATE_JSON_FILE);
    }
    importTemplate.setZipFile(true);
    return importTemplate;
}