Examples with ImportTemplate com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate used on opensource projects

Example 1 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class FeedImporter method validateEntityAccess.

private boolean validateEntityAccess(FeedMetadata existingFeed, String feedCategory) {
    FeedMetadata importingFeed = importFeed.getFeedToImport();
    if (existingFeed != null) {
        FeedMetadata userAccessFeed = metadataAccess.read(() -> {
            return metadataService.getFeedByName(feedCategory, importingFeed.getSystemFeedName());
        });
        if (userAccessFeed == null || !userAccessFeed.hasAction(FeedAccessControl.EDIT_DETAILS.getSystemName())) {
            // error
            importFeed.setValid(false);
            if (importFeed.getTemplate() == null) {
                ImportTemplate importTemplate = new ImportTemplate(importFeed.getFileName());
                importFeed.setTemplate(importTemplate);
            }
            String msg = "Access Denied.  You do not have access to edit this feed.";
            importFeed.getImportOptions().addErrorMessage(ImportComponent.FEED_DATA, msg);
            importFeed.addErrorMessage(existingFeed, msg);
            importFeed.setValid(false);
            return false;
        } else {
            return true;
        }
    } else {
        // ensure the user can create under the category
        Category category = metadataAccess.read(() -> {
            return categoryProvider.findBySystemName(feedCategory);
        }, MetadataAccess.SERVICE);
        if (category == null) {
            // ensure the user has functional access to create categories
            boolean hasPermission = accessController.hasPermission(AccessController.SERVICES, FeedServicesAccessControl.EDIT_CATEGORIES);
            if (!hasPermission) {
                String msg = "Access Denied. The category for this feed," + feedCategory + ", doesn't exist and you do not have access to create a new category.";
                importFeed.getImportOptions().addErrorMessage(ImportComponent.FEED_DATA, msg);
                importFeed.addErrorMessage(existingFeed, msg);
                importFeed.setValid(false);
                return false;
            }
            return true;
        } else {
            // if the feed is new ensure the user has write access to create feeds
            return metadataAccess.read(() -> {
                // Query for Category and ensure the user has access to create feeds on that category
                Category domainCategory = categoryProvider.findBySystemName(feedCategory);
                if (domainCategory == null || (!domainCategory.getAllowedActions().hasPermission(CategoryAccessControl.CREATE_FEED))) {
                    String msg = "Access Denied. You do not have access to create feeds under the category " + feedCategory + ". Attempt made to create feed " + FeedNameUtil.fullName(feedCategory, importingFeed.getSystemFeedName()) + ".";
                    importFeed.getImportOptions().addErrorMessage(ImportComponent.FEED_DATA, msg);
                    importFeed.addErrorMessage(existingFeed, msg);
                    importFeed.setValid(false);
                    return false;
                }
                /*
                       TemplateAccessControl.CREATE_FEED permission is not being used right now.
                       Uncomment this code once/if we should be checking it

                    // Query for Template and ensure the user has access to create feeds
                    final RegisteredTemplate domainTemplate = registeredTemplateService.findRegisteredTemplate(
                        new RegisteredTemplateRequest.Builder().templateName(importingFeed.getTemplateName()).isFeedEdit(true).build());
                    if (domainTemplate != null && !registeredTemplateService.hasTemplatePermission(domainTemplate.getId(), TemplateAccessControl.CREATE_FEED)) {
                        final String msg = "Access Denied. You do not have access to create feeds using the template " + importingFeed.getTemplateName()
                                           + ". Attempt made to create feed " + FeedNameUtil.fullName(feedCategory, importingFeed.getSystemFeedName()) + ".";
                        feed.getImportOptions().addErrorMessage(ImportComponent.FEED_DATA, msg);
                        feed.addErrorMessage(existingFeed, msg);
                        feed.setValid(false);
                        return false;
                    }
                    */
                return true;
            });
        }
    }
}

Example 2 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class FeedImporter method validate.

public ImportFeed validate() {
    UploadProgressMessage feedImportStatusMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Validating Feed import.");
    try {
        init();
        // read the JSON into the Feed object
        FeedMetadata metadata = importFeed.getFeedToImport();
        // validate the incoming category exists
        validateFeedCategory();
        // verify if we should overwrite the feed if it already exists
        String feedCategory = StringUtils.isNotBlank(importFeedOptions.getCategorySystemName()) ? importFeedOptions.getCategorySystemName() : metadata.getSystemCategoryName();
        // query for this feed.
        // first read in the feed as a service account
        FeedMetadata existingFeed = metadataAccess.read(() -> {
            return metadataService.getFeedByName(feedCategory, metadata.getSystemFeedName());
        }, MetadataAccess.SERVICE);
        if (!validateOverwriteExistingFeed(existingFeed)) {
            // exit
            return importFeed;
        }
        if (accessController.isEntityAccessControlled()) {
            if (!validateEntityAccess(existingFeed, feedCategory)) {
                return importFeed;
            }
        }
        // sensitive properties
        if (!validateSensitiveProperties()) {
            return importFeed;
        }
        // Valid data sources
        if (!validateUserDatasources()) {
            return importFeed;
        }
        // UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(),"Validating the template data");
        TemplateImporter templateImporter = templateImporterFactory.apply(importFeed.getFileName(), file, importFeedOptions);
        ImportTemplate importTemplate = templateImporter.validate();
        // need to set the importOptions back to the feed options
        // find importOptions for the Template and add them back to the set of options
        // importFeed.getImportOptions().updateOptions(importTemplate.getImportOptions().getImportComponentOptions());
        importFeed.setTemplate(importTemplate);
        // statusMessage.update("Validated the template data",importTemplate.isValid());
        if (!importTemplate.isValid()) {
            importFeed.setValid(false);
            List<String> errorMessages = importTemplate.getTemplateResults().getAllErrors().stream().map(nifiError -> nifiError.getMessage()).collect(Collectors.toList());
            if (!errorMessages.isEmpty()) {
                for (String msg : errorMessages) {
                    importFeed.addErrorMessage(metadata, msg);
                }
            }
        }
    // statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(),"Validation complete: the feed is "+(importFeed.isValid() ? "valid" : "invalid"),true,importFeed.isValid());
    } catch (Exception e) {
        feedImportStatusMessage.update("Validation error. Feed import error: " + e.getMessage(), false);
        throw new UnsupportedOperationException("Error importing template  " + fileName + ".  " + e.getMessage());
    }
    feedImportStatusMessage.update("Validated Feed import.", importFeed.isValid());
    return this.importFeed;
}

Example 3 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class FeedImporter method validateOverwriteExistingFeed.

private boolean validateOverwriteExistingFeed(FeedMetadata existingFeed) {
    FeedMetadata importingFeed = importFeed.getFeedToImport();
    if (existingFeed != null && !importFeedOptions.isImportAndOverwrite(ImportComponent.FEED_DATA)) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Validation error. " + importingFeed.getCategoryAndFeedName() + " already exists.", true, false);
        // if we dont have permission to overwrite then return with error that feed already exists
        importFeed.setValid(false);
        ImportTemplate importTemplate = new ImportTemplate(importFeed.getFileName());
        importFeed.setTemplate(importTemplate);
        String msg = "The feed " + existingFeed.getCategoryAndFeedName() + " already exists.";
        importFeed.getImportOptions().addErrorMessage(ImportComponent.FEED_DATA, msg);
        importFeed.addErrorMessage(existingFeed, msg);
        importFeed.setValid(false);
        return false;
    } else {
        String message = "Validated Feed data.  This import will " + (existingFeed != null ? "overwrite" : "create") + " the feed " + importingFeed.getCategoryAndFeedName();
        uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), message, true, true);
    }
    uploadProgressService.completeSection(importFeedOptions, ImportSection.Section.VALIDATE_FEED);
    return true;
}

Example 4 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class AdminController method uploadTemplatex.

/**
 * This is used for quick import via scripts.  The UI uses the AdminControllerV2 class
 */
@POST
@Path(IMPORT_TEMPLATE)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a template xml or zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the template metadata.", response = ImportTemplate.class), @ApiResponse(code = 500, message = "There was a problem importing the template.", response = RestResponseStatus.class) })
public Response uploadTemplatex(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @FormDataParam("overwrite") @DefaultValue("false") boolean overwrite, @FormDataParam("importConnectingReusableFlow") @DefaultValue("NOT_SET") ImportTemplateOptions.IMPORT_CONNECTING_FLOW importConnectingFlow, @FormDataParam("createReusableFlow") @DefaultValue("false") boolean createReusableFlow, @FormDataParam("templateProperties") @DefaultValue("") String templateProperties) throws Exception {
    ImportTemplateOptions options = new ImportTemplateOptions();
    String uploadKey = uploadProgressService.newUpload();
    options.setUploadKey(uploadKey);
    boolean isZip = fileMetaData.getFileName().endsWith("zip");
    if (isZip) {
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setShouldImport(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES));
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setUserAcknowledged(!importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.NOT_SET));
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setOverwrite(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES));
    } else {
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setShouldImport(createReusableFlow);
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setUserAcknowledged(true);
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setOverwrite(createReusableFlow);
    }
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setOverwrite(overwrite);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setShouldImport(true);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setContinueIfExists(!overwrite);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setOverwrite(overwrite);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setShouldImport(true);
    if (StringUtils.isNotBlank(templateProperties)) {
        List<ImportProperty> properties = ObjectMapperSerializer.deserialize(templateProperties, new TypeReference<List<ImportProperty>>() {
        });
        options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setProperties(properties);
    }
    byte[] content = ImportUtil.streamToByteArray(fileInputStream);
    TemplateImporter templateImporter = templateImporterFactory.apply(fileMetaData.getFileName(), content, options);
    ImportTemplate importTemplate = templateImporter.validateAndImport();
    return Response.ok(importTemplate).build();
}

Example 5 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class AdminControllerV2 method uploadTemplate.

@POST
@Path(IMPORT_TEMPLATE)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a template xml or zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the template metadata.", response = ImportTemplate.class), @ApiResponse(code = 500, message = "There was a problem importing the template.", response = RestResponseStatus.class) })
public Response uploadTemplate(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @NotNull @FormDataParam("uploadKey") String uploadKey, @FormDataParam("importComponents") String importComponents) throws Exception {
    ImportTemplateOptions options = new ImportTemplateOptions();
    options.setUploadKey(uploadKey);
    ImportTemplate importTemplate = null;
    byte[] content = ImportUtil.streamToByteArray(fileInputStream);
    uploadProgressService.newUpload(uploadKey);
    TemplateImporter templateImporter = null;
    if (importComponents == null) {
        templateImporter = templateImporterFactory.apply(fileMetaData.getFileName(), content, options);
        importTemplate = templateImporter.validate();
        importTemplate.setSuccess(false);
    } else {
        options.setImportComponentOptions(ObjectMapperSerializer.deserialize(importComponents, new TypeReference<Set<ImportComponentOption>>() {
        }));
        templateImporter = templateImporterFactory.apply(fileMetaData.getFileName(), content, options);
        importTemplate = templateImporter.validateAndImport();
    }
    return Response.ok(importTemplate).build();
}