Examples with UploadProgressMessage com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage used on opensource projects

Example 6 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class FeedImporter method validate.

public ImportFeed validate() {
    UploadProgressMessage feedImportStatusMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Validating Feed import.");
    try {
        init();
        // read the JSON into the Feed object
        FeedMetadata metadata = importFeed.getFeedToImport();
        // validate the incoming category exists
        validateFeedCategory();
        // verify if we should overwrite the feed if it already exists
        String feedCategory = StringUtils.isNotBlank(importFeedOptions.getCategorySystemName()) ? importFeedOptions.getCategorySystemName() : metadata.getSystemCategoryName();
        // query for this feed.
        // first read in the feed as a service account
        FeedMetadata existingFeed = metadataAccess.read(() -> {
            return metadataService.getFeedByName(feedCategory, metadata.getSystemFeedName());
        }, MetadataAccess.SERVICE);
        if (!validateOverwriteExistingFeed(existingFeed)) {
            // exit
            return importFeed;
        }
        if (accessController.isEntityAccessControlled()) {
            if (!validateEntityAccess(existingFeed, feedCategory)) {
                return importFeed;
            }
        }
        // sensitive properties
        if (!validateSensitiveProperties()) {
            return importFeed;
        }
        // Valid data sources
        if (!validateUserDatasources()) {
            return importFeed;
        }
        // UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(),"Validating the template data");
        TemplateImporter templateImporter = templateImporterFactory.apply(importFeed.getFileName(), file, importFeedOptions);
        ImportTemplate importTemplate = templateImporter.validate();
        // need to set the importOptions back to the feed options
        // find importOptions for the Template and add them back to the set of options
        // importFeed.getImportOptions().updateOptions(importTemplate.getImportOptions().getImportComponentOptions());
        importFeed.setTemplate(importTemplate);
        // statusMessage.update("Validated the template data",importTemplate.isValid());
        if (!importTemplate.isValid()) {
            importFeed.setValid(false);
            List<String> errorMessages = importTemplate.getTemplateResults().getAllErrors().stream().map(nifiError -> nifiError.getMessage()).collect(Collectors.toList());
            if (!errorMessages.isEmpty()) {
                for (String msg : errorMessages) {
                    importFeed.addErrorMessage(metadata, msg);
                }
            }
        }
    // statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(),"Validation complete: the feed is "+(importFeed.isValid() ? "valid" : "invalid"),true,importFeed.isValid());
    } catch (Exception e) {
        feedImportStatusMessage.update("Validation error. Feed import error: " + e.getMessage(), false);
        throw new UnsupportedOperationException("Error importing template  " + fileName + ".  " + e.getMessage());
    }
    feedImportStatusMessage.update("Validated Feed import.", importFeed.isValid());
    return this.importFeed;
}

Example 7 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class FeedImporter method validateOverwriteExistingFeed.

private boolean validateOverwriteExistingFeed(FeedMetadata existingFeed) {
    FeedMetadata importingFeed = importFeed.getFeedToImport();
    if (existingFeed != null && !importFeedOptions.isImportAndOverwrite(ImportComponent.FEED_DATA)) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Validation error. " + importingFeed.getCategoryAndFeedName() + " already exists.", true, false);
        // if we dont have permission to overwrite then return with error that feed already exists
        importFeed.setValid(false);
        ImportTemplate importTemplate = new ImportTemplate(importFeed.getFileName());
        importFeed.setTemplate(importTemplate);
        String msg = "The feed " + existingFeed.getCategoryAndFeedName() + " already exists.";
        importFeed.getImportOptions().addErrorMessage(ImportComponent.FEED_DATA, msg);
        importFeed.addErrorMessage(existingFeed, msg);
        importFeed.setValid(false);
        return false;
    } else {
        String message = "Validated Feed data.  This import will " + (existingFeed != null ? "overwrite" : "create") + " the feed " + importingFeed.getCategoryAndFeedName();
        uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), message, true, true);
    }
    uploadProgressService.completeSection(importFeedOptions, ImportSection.Section.VALIDATE_FEED);
    return true;
}

Example 8 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class FeedImporter method validateFeedCategory.

private boolean validateFeedCategory() {
    FeedMetadata metadata = importFeed.getFeedToImport();
    boolean valid = true;
    if (StringUtils.isNotBlank(importFeedOptions.getCategorySystemName())) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Validating the newly specified category. Ensure " + importFeedOptions.getCategorySystemName() + " exists.");
        FeedCategory optionsCategory = metadataService.getCategoryBySystemName(importFeedOptions.getCategorySystemName());
        if (optionsCategory == null) {
            importFeed.setValid(false);
            statusMessage.update("Validation Error. The category " + importFeedOptions.getCategorySystemName() + " does not exist, or you dont have access to it.", false);
            valid = false;
        } else {
            if (valid) {
                metadata.getCategory().setSystemName(importFeedOptions.getCategorySystemName());
                statusMessage.update("Validated. The category " + importFeedOptions.getCategorySystemName() + " exists.", true);
            }
        }
    }
    uploadProgressService.completeSection(importFeedOptions, ImportSection.Section.VALIDATE_FEED_CATEGORY);
    return valid;
}

Example 9 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class AbstractImportTemplateRoutine method removeTemporaryProcessGroup.

/**
 * Cleanup and remove the temporary process group associated with this import
 *
 * This should be called after the import to cleanup NiFi
 */
public void removeTemporaryProcessGroup() {
    UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importTemplate.getImportOptions().getUploadKey(), "Cleaning up the temporary process group in NiFi");
    String processGroupName = importTemplate.getTemplateResults().getProcessGroupEntity().getName();
    String processGroupId = importTemplate.getTemplateResults().getProcessGroupEntity().getId();
    String parentProcessGroupId = importTemplate.getTemplateResults().getProcessGroupEntity().getParentGroupId();
    log.info("About to cleanup the temporary process group {} ({})", processGroupName, processGroupId);
    try {
        nifiRestClient.removeProcessGroup(processGroupId, parentProcessGroupId);
        log.info("Successfully cleaned up Nifi and deleted the process group {} ", importTemplate.getTemplateResults().getProcessGroupEntity().getName());
        statusMessage.update("Removed the temporary process group " + processGroupName + " in NiFi", true);
    } catch (NifiClientRuntimeException e) {
        log.error("error attempting to cleanup and remove the temporary process group (" + processGroupId + " during the import of template " + importTemplate.getTemplateName());
        importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, "Issues found in cleaning up the template import: " + importTemplate.getTemplateName() + ".  The Process Group : " + processGroupName + " (" + processGroupId + ")" + " may need to be manually cleaned up in NiFi ", "");
        statusMessage.update("Error cleaning up NiFi. The Process Group : " + processGroupName + " (" + processGroupId + ")", false);
    }
}

Example 10 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class AbstractImportTemplateRoutine method importIntoNiFi.

/**
 * Import a template string into NiFi as a template to use
 */
public NiFiTemplateImport importIntoNiFi(ImportTemplate template, ImportTemplateOptions importOptions) throws ImportException {
    TemplateDTO dto = null;
    String templateName = null;
    String oldTemplateXml = null;
    ImportComponentOption nifiTemplateImport = importOptions.findImportComponentOption(ImportComponent.NIFI_TEMPLATE);
    if (nifiTemplateImport.isValidForImport()) {
        try {
            templateName = NifiTemplateParser.getTemplateName(template.getNifiTemplateXml());
            template.setTemplateName(templateName);
            dto = nifiRestClient.getTemplateByName(templateName);
            if (dto != null) {
                oldTemplateXml = nifiRestClient.getTemplateXml(dto.getId());
                template.setNifiTemplateId(dto.getId());
                if (importOptions.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE) && nifiTemplateImport.isValidForImport()) {
                    nifiRestClient.deleteTemplate(dto.getId());
                } else if (!template.isZipFile()) {
                    // if its not a zip file we need to error out if the user has decided not to overwrite when it already exists
                    uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "The template " + templateName + " already exists in NiFi. Please choose the option to replace the template and try again.", true, false);
                    template.setValid(false);
                }
            }
        } catch (ParserConfigurationException | XPathExpressionException | IOException | SAXException e) {
            throw new ImportException("The xml file you are trying to import is not a valid NiFi template.  Please try again. " + e.getMessage());
        }
        boolean register = (dto == null || (importOptions.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE) && nifiTemplateImport.isValidForImport()));
        // attempt to import the xml into NiFi if its new, or if the user said to overwrite
        if (register) {
            UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "Importing " + templateName + " into NiFi");
            log.info("Attempting to import Nifi Template: {} for file {}", templateName, template.getFileName());
            dto = nifiRestClient.importTemplate(template.getTemplateName(), template.getNifiTemplateXml());
            template.setNifiTemplateId(dto.getId());
            statusMessage.update("Imported " + templateName + " into NiFi", true);
        }
    }
    uploadProgressService.completeSection(importOptions, ImportSection.Section.IMPORT_NIFI_TEMPLATE);
    return new NiFiTemplateImport(oldTemplateXml, dto);
}