Examples with ImportTemplateOptions com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions used on opensource projects

Example 1 with ImportTemplateOptions

use of com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions in project kylo by Teradata.

the class AdminController method uploadTemplatex.

/**
 * This is used for quick import via scripts.  The UI uses the AdminControllerV2 class
 */
@POST
@Path(IMPORT_TEMPLATE)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a template xml or zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the template metadata.", response = ImportTemplate.class), @ApiResponse(code = 500, message = "There was a problem importing the template.", response = RestResponseStatus.class) })
public Response uploadTemplatex(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @FormDataParam("overwrite") @DefaultValue("false") boolean overwrite, @FormDataParam("importConnectingReusableFlow") @DefaultValue("NOT_SET") ImportTemplateOptions.IMPORT_CONNECTING_FLOW importConnectingFlow, @FormDataParam("createReusableFlow") @DefaultValue("false") boolean createReusableFlow, @FormDataParam("templateProperties") @DefaultValue("") String templateProperties) throws Exception {
    ImportTemplateOptions options = new ImportTemplateOptions();
    String uploadKey = uploadProgressService.newUpload();
    options.setUploadKey(uploadKey);
    boolean isZip = fileMetaData.getFileName().endsWith("zip");
    if (isZip) {
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setShouldImport(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES));
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setUserAcknowledged(!importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.NOT_SET));
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setOverwrite(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES));
    } else {
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setShouldImport(createReusableFlow);
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setUserAcknowledged(true);
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setOverwrite(createReusableFlow);
    }
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setOverwrite(overwrite);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setShouldImport(true);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setContinueIfExists(!overwrite);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setOverwrite(overwrite);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setShouldImport(true);
    if (StringUtils.isNotBlank(templateProperties)) {
        List<ImportProperty> properties = ObjectMapperSerializer.deserialize(templateProperties, new TypeReference<List<ImportProperty>>() {
        });
        options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setProperties(properties);
    }
    byte[] content = ImportUtil.streamToByteArray(fileInputStream);
    TemplateImporter templateImporter = templateImporterFactory.apply(fileMetaData.getFileName(), content, options);
    ImportTemplate importTemplate = templateImporter.validateAndImport();
    return Response.ok(importTemplate).build();
}

Example 2 with ImportTemplateOptions

use of com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions in project kylo by Teradata.

the class AdminControllerV2 method uploadTemplate.

@POST
@Path(IMPORT_TEMPLATE)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a template xml or zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the template metadata.", response = ImportTemplate.class), @ApiResponse(code = 500, message = "There was a problem importing the template.", response = RestResponseStatus.class) })
public Response uploadTemplate(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @NotNull @FormDataParam("uploadKey") String uploadKey, @FormDataParam("importComponents") String importComponents) throws Exception {
    ImportTemplateOptions options = new ImportTemplateOptions();
    options.setUploadKey(uploadKey);
    ImportTemplate importTemplate = null;
    byte[] content = ImportUtil.streamToByteArray(fileInputStream);
    uploadProgressService.newUpload(uploadKey);
    TemplateImporter templateImporter = null;
    if (importComponents == null) {
        templateImporter = templateImporterFactory.apply(fileMetaData.getFileName(), content, options);
        importTemplate = templateImporter.validate();
        importTemplate.setSuccess(false);
    } else {
        options.setImportComponentOptions(ObjectMapperSerializer.deserialize(importComponents, new TypeReference<Set<ImportComponentOption>>() {
        }));
        templateImporter = templateImporterFactory.apply(fileMetaData.getFileName(), content, options);
        importTemplate = templateImporter.validateAndImport();
    }
    return Response.ok(importTemplate).build();
}

Example 3 with ImportTemplateOptions

use of com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions in project kylo by Teradata.

the class TemplateImporter method init.

private void init() throws ImportException {
    this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.IMPORT_TEMPLATES);
    overallStatusMessage = uploadProgressService.addUploadStatus(this.importOptions.getUploadKey(), "Validating template for import");
    // copy the options to ImportTemplateOptions
    ImportTemplateOptions options = new ImportTemplateOptions();
    options.setUploadKey(this.importOptions.getUploadKey());
    options.setImportComponentOptions(this.importOptions.getImportComponentOptions());
    if (!isValidFileImport(fileName)) {
        this.overallStatusMessage.complete(false);
        throw new ImportException("Unable to import " + fileName + ".  The file must be a zip file or a Nifi Template xml file");
    }
    this.importTemplateOptions = options;
    setImportType();
    if (importType == ImportTemplate.TYPE.ARCHIVE) {
        initializeImportTemplateArchive();
    } else {
        initializeImportTemplateFromXml();
    }
}

Example 4 with ImportTemplateOptions

use of com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions in project kylo by Teradata.

the class FeedImporter method importFeed.

// Import
/**
 * Import a feed zip file
 */
private ImportFeed importFeed() throws Exception {
    // read the JSON into the Feed object
    FeedMetadata metadata = importFeed.getFeedToImport();
    // query for this feed.
    String feedCategory = StringUtils.isNotBlank(importFeedOptions.getCategorySystemName()) ? importFeedOptions.getCategorySystemName() : metadata.getSystemCategoryName();
    FeedMetadata existingFeed = metadataAccess.read(() -> metadataService.getFeedByName(feedCategory, metadata.getSystemFeedName()));
    metadata.getCategory().setSystemName(feedCategory);
    ImportTemplateOptions importTemplateOptions = new ImportTemplateOptions();
    importTemplateOptions.setImportComponentOptions(importFeedOptions.getImportComponentOptions());
    importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setContinueIfExists(true);
    ImportTemplate importTemplate = importFeed.getTemplate();
    importTemplate.setImportOptions(importTemplateOptions);
    importTemplateOptions.setUploadKey(importFeedOptions.getUploadKey());
    importTemplate.setValid(true);
    importTemplateOptions.setDeferCleanup(true);
    // Import the Template
    ImportTemplateRoutine importTemplateRoutine = importTemplateRoutineFactory.apply(importTemplate, importTemplateOptions, ImportTemplate.TYPE.ARCHIVE);
    importTemplateRoutine.importTemplate();
    if (importTemplate.isSuccess()) {
        // import the feed
        importFeed.setTemplate(importTemplate);
        // now that we have the Feed object we need to create the instance of the feed
        UploadProgressMessage uploadProgressMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Saving  and creating feed instance in NiFi");
        metadata.setIsNew(existingFeed == null ? true : false);
        metadata.setFeedId(existingFeed != null ? existingFeed.getFeedId() : null);
        metadata.setId(existingFeed != null ? existingFeed.getId() : null);
        // reassign the templateId to the newly registered template id
        metadata.setTemplateId(importTemplate.getTemplateId());
        if (metadata.getRegisteredTemplate() != null) {
            metadata.getRegisteredTemplate().setNifiTemplateId(importTemplate.getNifiTemplateId());
            metadata.getRegisteredTemplate().setId(importTemplate.getTemplateId());
        }
        // get/create category
        FeedCategory category = metadataService.getCategoryBySystemName(metadata.getCategory().getSystemName());
        if (category == null) {
            metadata.getCategory().setId(null);
            metadataService.saveCategory(metadata.getCategory());
        } else {
            metadata.setCategory(category);
        }
        if (importFeedOptions.isDisableUponImport()) {
            metadata.setActive(false);
            metadata.setState(FeedMetadata.STATE.DISABLED.name());
        }
        // remap any preconditions to this new feed/category name.
        if (metadata.getSchedule().hasPreconditions()) {
            metadata.getSchedule().getPreconditions().stream().flatMap(preconditionRule -> preconditionRule.getProperties().stream()).filter(fieldRuleProperty -> PolicyPropertyTypes.PROPERTY_TYPE.currentFeed.name().equals(fieldRuleProperty.getType())).forEach(fieldRuleProperty -> fieldRuleProperty.setValue(metadata.getCategoryAndFeedName()));
        }
        // //for all those properties where the template value is != userEditable and the template value has a metadata. property, remove that property from the feed properties so it can be imported and assigned correctly
        RegisteredTemplate template1 = registeredTemplateService.findRegisteredTemplateById(importTemplate.getTemplateId());
        if (template1 != null) {
            // Find all the properties in the template that have ${metadata. and are not userEditable.
            // These are the properties we need to replace on the feed metadata
            List<NifiProperty> metadataProperties = template1.getProperties().stream().filter(nifiProperty -> {
                return nifiProperty != null && StringUtils.isNotBlank(nifiProperty.getValue()) && !nifiProperty.isUserEditable() && nifiProperty.getValue().contains("${" + MetadataFieldAnnotationFieldNameResolver.metadataPropertyPrefix);
            }).collect(Collectors.toList());
            // Replace the Feed Metadata properties with those that match the template ones from above.
            List<NifiProperty> updatedProperties = metadata.getProperties().stream().map(nifiProperty -> {
                NifiProperty p = NifiPropertyUtil.findPropertyByProcessorName(metadataProperties, nifiProperty);
                return p != null ? p : nifiProperty;
            }).collect(Collectors.toList());
            metadata.setProperties(updatedProperties);
        }
        NifiFeed nifiFeed = metadataService.createFeed(metadata);
        if (nifiFeed != null) {
            importFeed.setFeedName(nifiFeed.getFeedMetadata().getCategoryAndFeedName());
            if (nifiFeed.isSuccess()) {
                uploadProgressMessage.update("Successfully saved the feed " + importFeed.getFeedName(), true);
            } else {
                if (nifiFeed.getFeedProcessGroup() != null && nifiFeed.getFeedProcessGroup().isRolledBack()) {
                    if (importTemplateRoutine != null) {
                        importTemplateRoutine.rollback();
                    }
                }
                uploadProgressMessage.update("Errors were found importing the feed " + importFeed.getFeedName(), false);
            }
            importTemplateRoutine.cleanup();
        }
        importFeed.setNifiFeed(nifiFeed);
        importFeed.setSuccess(nifiFeed != null && nifiFeed.isSuccess());
    } else {
        importFeed.setSuccess(false);
        importFeed.setTemplate(importTemplate);
        importFeed.addErrorMessage(existingFeed, "The feed " + FeedNameUtil.fullName(feedCategory, metadata.getSystemFeedName()) + " needs additional properties to be supplied before importing.");
    }
    uploadProgressService.completeSection(importFeedOptions, ImportSection.Section.IMPORT_FEED_DATA);
    return importFeed;
}

Example 5 with ImportTemplateOptions

use of com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions in project kylo by Teradata.

the class ImportTemplateArchive method importTemplate.

public boolean importTemplate() {
    UploadProgressMessage statusMessage = null;
    // Get information about the import
    RegisteredTemplate template = importTemplate.getTemplateToImport();
    // validateTemplateProperties(template, importTemplate, importOptions);
    // 1 ensure this template doesnt already exist
    importTemplate.setTemplateName(template.getTemplateName());
    RegisteredTemplate existingTemplate = registeredTemplateService.findRegisteredTemplate(RegisteredTemplateRequest.requestByTemplateName(template.getTemplateName()));
    if (existingTemplate != null) {
        template.setId(existingTemplate.getId());
    } else {
        template.setId(null);
    }
    // first we just import the reusable templates as flows
    this.importedReusableTemplates = importReusableTemplateInArchive(importTemplate, this.importTemplateOptions);
    // after the templates are created we then connect the templates
    if (!this.importTemplateOptions.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).hasErrorMessages()) {
        if (!importedReusableTemplates.isEmpty()) {
            connectReusableTemplates();
        } else {
            importTemplate.setSuccess(true);
        }
        if (importTemplate.isSuccess()) {
            RegisteredTemplate newTemplate = importFeedTemplate(existingTemplate);
            if (importTemplate.isSuccess()) {
                if (newTemplate != null) {
                    validateInstance();
                }
                if (!importedReusableTemplates.isEmpty()) {
                    importedReusableTemplates.stream().filter(importReusableTemplate -> importReusableTemplate.getImportTemplate().isSuccess()).map(t -> t.getImportTemplate()).forEach(connectingTemplate -> {
                        nifiRestClient.markConnectionPortsAsRunning(connectingTemplate.getTemplateResults().getProcessGroupEntity());
                    });
                }
            }
        }
    } else {
        rollback();
        // return if invalid
        return false;
    }
    // cleanup any temp process groups for this template
    if (!importTemplateOptions.isDeferCleanup()) {
        cleanup();
    }
    uploadProgressService.completeSection(importTemplateOptions, ImportSection.Section.IMPORT_REGISTERED_TEMPLATE);
    return importTemplate.isSuccess();
}