Examples with UploadProgressMessage com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage used on opensource projects

Example 16 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ImportTemplateArchive method registerTemplate.

/**
 * Register the template with the metadata and save it
 *
 * @param importTemplate the template data to import
 * @param importOptions  user options about what/how it should be imported
 * @return the registered template that was saved
 */
private RegisteredTemplate registerTemplate(ImportTemplate importTemplate, ImportTemplateOptions importOptions) {
    RegisteredTemplate template = importTemplate.getTemplateToImport();
    ImportComponentOption registeredTemplateOption = importOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA);
    if (registeredTemplateOption.isValidForImport()) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "Registering template " + template.getTemplateName() + " with Kylo metadata.");
        try {
            importTemplate.setNifiTemplateId(template.getNifiTemplateId());
            // register it in the system
            metadataService.registerTemplate(template);
            // get the new template
            template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(template.getId()).templateName(template.getTemplateName()).build());
            importTemplate.setTemplateId(template.getId());
            statusMessage.update("Registered template with Kylo metadata.", true);
        } catch (Exception e) {
            importTemplate.setSuccess(false);
            Throwable root = ExceptionUtils.getRootCause(e);
            String msg = root != null ? root.getMessage() : e.getMessage();
            importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, "Error registering the template " + template.getTemplateName() + " in the Kylo metadata. " + msg, "");
            statusMessage.update("Error registering template with Kylo metadata. " + msg, false);
        }
    }
    return template;
}

Example 17 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ImportTemplateArchive method importFeedTemplate.

@Nullable
private RegisteredTemplate importFeedTemplate(RegisteredTemplate existingTemplate) {
    RegisteredTemplate template = null;
    niFiTemplateImport = importIntoNiFi(this.importTemplate, this.importTemplateOptions);
    ImportComponentOption registeredTemplateImport = this.importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA);
    if (existingTemplate != null) {
        importTemplate.setTemplateId(existingTemplate.getId());
    }
    if (registeredTemplateImport.isShouldImport() && registeredTemplateImport.isValidForImport()) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Importing feed template");
        this.newTemplateInstance = create(niFiTemplateImport, statusMessage);
        if (newTemplateInstance.isSuccess()) {
            importTemplate.setSuccess(true);
            RegisteredTemplate savedTemplate = registerTemplate(importTemplate, importTemplateOptions);
            if (savedTemplate != null) {
                template = savedTemplate;
            }
        } else {
            importTemplate.setSuccess(false);
        }
        statusMessage.complete(importTemplate.isSuccess());
    } else {
        importTemplate.setSuccess(true);
    }
    return template;
}

Example 18 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ValidateImportTemplatesArchive method validateRegisteredTemplate.

/**
 * Validate the Registered Template is valid for importing
 */
private void validateRegisteredTemplate() {
    ImportComponentOption registeredTemplateOption = this.importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA);
    // validate template
    boolean validForImport = true;
    if (!registeredTemplateOption.isUserAcknowledged()) {
        this.importTemplate.setValid(false);
        this.importTemplate.getImportOptions().addErrorMessage(ImportComponent.TEMPLATE_DATA, "A template exists.  Do you want to import it?");
    }
    if (registeredTemplateOption.isUserAcknowledged() && registeredTemplateOption.isShouldImport()) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(this.importTemplateOptions.getUploadKey(), "Validating feed template for import");
        RegisteredTemplate registeredTemplate = this.importTemplate.getTemplateToImport();
        // validate unique
        // 1 find if the template exists in the system running as a service account
        RegisteredTemplate existingTemplate = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateName(registeredTemplate.getTemplateName()).isFeedEdit(true).build());
        if (existingTemplate != null) {
            if (this.importTemplateOptions.stopProcessingAlreadyExists(ImportComponent.TEMPLATE_DATA)) {
                this.importTemplate.setValid(false);
                String msg = "Unable to import the template " + registeredTemplate.getTemplateName() + " It is already registered.";
                this.importTemplate.getImportOptions().addErrorMessage(ImportComponent.TEMPLATE_DATA, msg);
                statusMessage.update("Validation error: The template " + registeredTemplate.getTemplateName() + " is already registered", false);
            } else {
                // skip importing if user doesnt want it.
                if (!registeredTemplateOption.isOverwrite()) {
                    validForImport = false;
                }
                statusMessage.complete(true);
            }
            registeredTemplate.setId(existingTemplate.getId());
            // validate entity access
            if (accessController.isEntityAccessControlled() && registeredTemplateOption.isOverwrite()) {
                // requery as the currently logged in user
                statusMessage = uploadProgressService.addUploadStatus(this.importTemplateOptions.getUploadKey(), "Validating template entity access");
                existingTemplate = registeredTemplateService.findRegisteredTemplate(RegisteredTemplateRequest.requestByTemplateName(registeredTemplate.getTemplateName()));
                // ensure the user can Edit this template
                boolean valid = existingTemplate != null && existingTemplate.getAllowedActions().hasAction(TemplateAccessControl.EDIT_TEMPLATE.getSystemName());
                if (!valid) {
                    this.importTemplate.setValid(false);
                    validForImport = false;
                    statusMessage.update("Access Denied: You do not have edit access for the template " + registeredTemplate.getTemplateName(), false);
                    this.importTemplate.getImportOptions().addErrorMessage(ImportComponent.TEMPLATE_DATA, "Access Denied: You do not have edit access for the template ");
                } else {
                    statusMessage.complete(valid);
                }
            }
        } else {
            // template doesnt exist.. it is new ensure the user is allowed to create templates
            boolean editAccess = accessController.hasPermission(AccessController.SERVICES, FeedServicesAccessControl.EDIT_TEMPLATES);
            if (!editAccess) {
                statusMessage.update("Access Denied: You are not allowed to create the template " + registeredTemplate.getTemplateName(), false);
                this.importTemplate.getImportOptions().addErrorMessage(ImportComponent.TEMPLATE_DATA, "Access Denied: You are not allowed to create the template ");
            }
            registeredTemplate.setId(null);
            statusMessage.complete(editAccess);
        }
        validForImport &= !registeredTemplateOption.hasErrorMessages();
        if (validForImport) {
            boolean isValid = validateTemplateProperties();
            if (this.importTemplate.isValid()) {
                this.importTemplate.setValid(isValid);
            }
            validForImport &= isValid;
        }
        registeredTemplateOption.setValidForImport(validForImport);
    }
    this.uploadProgressService.completeSection(this.importTemplateOptions, ImportSection.Section.VALIDATE_REGISTERED_TEMPLATE);
}

Example 19 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ValidateImportTemplatesArchive method validateReusableTemplate.

/**
 * Validate any reusable templates as part of a zip file upload are valid for importing
 */
private void validateReusableTemplate() {
    // validate the reusable template
    if (this.importTemplate.hasConnectingReusableTemplate()) {
        ImportComponentOption reusableTemplateOption = this.importTemplateOptions.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE);
        UploadProgressMessage reusableTemplateStatusMessage = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Validating Reusable Template. ");
        if (reusableTemplateOption.isShouldImport()) {
            boolean validForImport = true;
            // for each of the connecting template
            for (String reusableTemplateXml : this.importTemplate.getNifiConnectingReusableTemplateXmls()) {
                try {
                    String templateName = NifiTemplateParser.getTemplateName(reusableTemplateXml);
                    UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Validating Reusable Template. " + templateName);
                    TemplateDTO dto = nifiRestClient.getTemplateByName(templateName);
                    // if there is a match and it has not been acknowledged by the user to overwrite or not, error out
                    if (dto != null && !reusableTemplateOption.isUserAcknowledged()) {
                        // error out it exists
                        this.importTemplate.getImportOptions().addErrorMessage(ImportComponent.REUSABLE_TEMPLATE, "A reusable template with the same name " + templateName + " exists.");
                        this.importTemplate.setValid(false);
                        statusMessage.update("Reusable template, " + templateName + ", already exists.", false);
                        validForImport = false;
                    } else if (dto != null && reusableTemplateOption.isUserAcknowledged() && !reusableTemplateOption.isOverwrite()) {
                        validForImport = false;
                        // user has asked to not import the template.
                        uploadProgressService.removeMessage(importTemplateOptions.getUploadKey(), statusMessage);
                    } else {
                        uploadProgressService.removeMessage(importTemplateOptions.getUploadKey(), statusMessage);
                        // statusMessage.update("Validated Reusable Template", true);
                        validForImport &= true;
                    }
                } catch (Exception e) {
                    log.error("Error parsing template name from file {} ", fileName, e);
                    validForImport = false;
                }
            }
            reusableTemplateOption.setValidForImport(validForImport);
            reusableTemplateStatusMessage.update("Validated Reusable Templates ", !reusableTemplateOption.hasErrorMessages());
        } else if (!reusableTemplateOption.isUserAcknowledged()) {
            this.importTemplate.getImportOptions().addErrorMessage(ImportComponent.REUSABLE_TEMPLATE, "The file " + this.importTemplate.getFileName() + " has a reusable template to import.");
            this.importTemplate.setValid(false);
            reusableTemplateStatusMessage.update("A reusable template was found. Additional input needed.", false);
        } else {
            reusableTemplateStatusMessage.update("Reusable template found in import, but it is not marked for importing", true);
        }
    }
    this.uploadProgressService.completeSection(this.importTemplateOptions, ImportSection.Section.VALIDATE_REUSABLE_TEMPLATE);
}

Example 20 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class FeedImporter method importFeed.

// Import
/**
 * Import a feed zip file
 */
private ImportFeed importFeed() throws Exception {
    // read the JSON into the Feed object
    FeedMetadata metadata = importFeed.getFeedToImport();
    // query for this feed.
    String feedCategory = StringUtils.isNotBlank(importFeedOptions.getCategorySystemName()) ? importFeedOptions.getCategorySystemName() : metadata.getSystemCategoryName();
    FeedMetadata existingFeed = metadataAccess.read(() -> metadataService.getFeedByName(feedCategory, metadata.getSystemFeedName()));
    metadata.getCategory().setSystemName(feedCategory);
    ImportTemplateOptions importTemplateOptions = new ImportTemplateOptions();
    importTemplateOptions.setImportComponentOptions(importFeedOptions.getImportComponentOptions());
    importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setContinueIfExists(true);
    ImportTemplate importTemplate = importFeed.getTemplate();
    importTemplate.setImportOptions(importTemplateOptions);
    importTemplateOptions.setUploadKey(importFeedOptions.getUploadKey());
    importTemplate.setValid(true);
    importTemplateOptions.setDeferCleanup(true);
    // Import the Template
    ImportTemplateRoutine importTemplateRoutine = importTemplateRoutineFactory.apply(importTemplate, importTemplateOptions, ImportTemplate.TYPE.ARCHIVE);
    importTemplateRoutine.importTemplate();
    if (importTemplate.isSuccess()) {
        // import the feed
        importFeed.setTemplate(importTemplate);
        // now that we have the Feed object we need to create the instance of the feed
        UploadProgressMessage uploadProgressMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Saving  and creating feed instance in NiFi");
        metadata.setIsNew(existingFeed == null ? true : false);
        metadata.setFeedId(existingFeed != null ? existingFeed.getFeedId() : null);
        metadata.setId(existingFeed != null ? existingFeed.getId() : null);
        // reassign the templateId to the newly registered template id
        metadata.setTemplateId(importTemplate.getTemplateId());
        if (metadata.getRegisteredTemplate() != null) {
            metadata.getRegisteredTemplate().setNifiTemplateId(importTemplate.getNifiTemplateId());
            metadata.getRegisteredTemplate().setId(importTemplate.getTemplateId());
        }
        // get/create category
        FeedCategory category = metadataService.getCategoryBySystemName(metadata.getCategory().getSystemName());
        if (category == null) {
            metadata.getCategory().setId(null);
            metadataService.saveCategory(metadata.getCategory());
        } else {
            metadata.setCategory(category);
        }
        if (importFeedOptions.isDisableUponImport()) {
            metadata.setActive(false);
            metadata.setState(FeedMetadata.STATE.DISABLED.name());
        }
        // remap any preconditions to this new feed/category name.
        if (metadata.getSchedule().hasPreconditions()) {
            metadata.getSchedule().getPreconditions().stream().flatMap(preconditionRule -> preconditionRule.getProperties().stream()).filter(fieldRuleProperty -> PolicyPropertyTypes.PROPERTY_TYPE.currentFeed.name().equals(fieldRuleProperty.getType())).forEach(fieldRuleProperty -> fieldRuleProperty.setValue(metadata.getCategoryAndFeedName()));
        }
        // //for all those properties where the template value is != userEditable and the template value has a metadata. property, remove that property from the feed properties so it can be imported and assigned correctly
        RegisteredTemplate template1 = registeredTemplateService.findRegisteredTemplateById(importTemplate.getTemplateId());
        if (template1 != null) {
            // Find all the properties in the template that have ${metadata. and are not userEditable.
            // These are the properties we need to replace on the feed metadata
            List<NifiProperty> metadataProperties = template1.getProperties().stream().filter(nifiProperty -> {
                return nifiProperty != null && StringUtils.isNotBlank(nifiProperty.getValue()) && !nifiProperty.isUserEditable() && nifiProperty.getValue().contains("${" + MetadataFieldAnnotationFieldNameResolver.metadataPropertyPrefix);
            }).collect(Collectors.toList());
            // Replace the Feed Metadata properties with those that match the template ones from above.
            List<NifiProperty> updatedProperties = metadata.getProperties().stream().map(nifiProperty -> {
                NifiProperty p = NifiPropertyUtil.findPropertyByProcessorName(metadataProperties, nifiProperty);
                return p != null ? p : nifiProperty;
            }).collect(Collectors.toList());
            metadata.setProperties(updatedProperties);
        }
        NifiFeed nifiFeed = metadataService.createFeed(metadata);
        if (nifiFeed != null) {
            importFeed.setFeedName(nifiFeed.getFeedMetadata().getCategoryAndFeedName());
            if (nifiFeed.isSuccess()) {
                uploadProgressMessage.update("Successfully saved the feed " + importFeed.getFeedName(), true);
            } else {
                if (nifiFeed.getFeedProcessGroup() != null && nifiFeed.getFeedProcessGroup().isRolledBack()) {
                    if (importTemplateRoutine != null) {
                        importTemplateRoutine.rollback();
                    }
                }
                uploadProgressMessage.update("Errors were found importing the feed " + importFeed.getFeedName(), false);
            }
            importTemplateRoutine.cleanup();
        }
        importFeed.setNifiFeed(nifiFeed);
        importFeed.setSuccess(nifiFeed != null && nifiFeed.isSuccess());
    } else {
        importFeed.setSuccess(false);
        importFeed.setTemplate(importTemplate);
        importFeed.addErrorMessage(existingFeed, "The feed " + FeedNameUtil.fullName(feedCategory, metadata.getSystemFeedName()) + " needs additional properties to be supplied before importing.");
    }
    uploadProgressService.completeSection(importFeedOptions, ImportSection.Section.IMPORT_FEED_DATA);
    return importFeed;
}