Examples with ImportComponentOption com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption used on opensource projects

Example 1 with ImportComponentOption

use of com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption in project kylo by Teradata.

the class FeedImporter method validateUserDatasources.

/**
 * Validates that user data sources can be imported with provided properties.
 *
 * @return {@code true} if the feed can be imported, or {@code false} otherwise
 */
private boolean validateUserDatasources() {
    FeedMetadata metadata = importFeed.getFeedToImport();
    final UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importFeed.getImportOptions().getUploadKey(), "Validating data sources.");
    // Get data sources needing to be created
    final Set<String> availableDatasources = metadataAccess.read(() -> datasourceProvider.getDatasources(datasourceProvider.datasetCriteria().type(UserDatasource.class)).stream().map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId).map(Object::toString).collect(Collectors.toSet()));
    final ImportComponentOption componentOption = importFeedOptions.findImportComponentOption(ImportComponent.USER_DATASOURCES);
    final List<Datasource> providedDatasources = Optional.ofNullable(metadata.getUserDatasources()).orElse(Collections.emptyList());
    if (componentOption.getProperties().isEmpty()) {
        componentOption.setProperties(providedDatasources.stream().filter(datasource -> !availableDatasources.contains(datasource.getId())).map(datasource -> new ImportProperty(datasource.getName(), datasource.getId(), null, null, null)).collect(Collectors.toList()));
    }
    // Update feed with re-mapped data sources
    final boolean valid = componentOption.getProperties().stream().allMatch(property -> {
        if (property.getPropertyValue() != null) {
            ImportUtil.replaceDatasource(metadata, property.getProcessorId(), property.getPropertyValue());
            return true;
        } else {
            return false;
        }
    });
    if (valid) {
        statusMessage.update("Validated data sources.", true);
    } else {
        statusMessage.update("Validation Error. Additional properties are needed before uploading the feed.", false);
        importFeed.setValid(false);
    }
    uploadProgressService.completeSection(importFeed.getImportOptions(), ImportSection.Section.VALIDATE_USER_DATASOURCES);
    return valid;
}

Example 2 with ImportComponentOption

use of com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption in project kylo by Teradata.

the class FeedImporter method init.

private void init() {
    this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.IMPORT_FEEDS);
    UploadProgressMessage feedImportStatusMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Initialize feed import.");
    try {
        boolean isValid = isValidFileImport(fileName) && ZipFileUtil.validateZipEntriesWithRequiredEntries(file, getValidZipFileEntries(), Sets.newHashSet(ImportFeed.FEED_JSON_FILE));
        if (!isValid) {
            feedImportStatusMessage.update("Validation error. Feed import error. The zip file you uploaded is not valid feed export.", false);
            throw new ImportFeedException("The zip file you uploaded is not valid feed export.");
        }
        // get the Feed Data
        importFeed = readFeedJson(fileName, file);
        // initially mark as valid.
        importFeed.setValid(true);
        // merge in the file components to the user options
        Set<ImportComponentOption> componentOptions = ImportUtil.inspectZipComponents(file, ImportType.FEED);
        importFeedOptions.addOptionsIfNotExists(componentOptions);
        // importFeedOptions.findImportComponentOption(ImportComponent.TEMPLATE_CONNECTION_INFORMATION).addConnectionInfo(importFeed.getReusableTemplateConnections());
        importFeed.setImportOptions(importFeedOptions);
        feedImportStatusMessage.complete(true);
    } catch (Exception e) {
        throw new ImportException("Unable to import feed. ", e);
    }
}

Example 3 with ImportComponentOption

use of com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption in project kylo by Teradata.

the class AdminControllerV2 method uploadFeed.

@POST
@Path(IMPORT_FEED)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a feed zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed metadata.", response = ImportFeed.class), @ApiResponse(code = 500, message = "There was a problem importing the feed.", response = RestResponseStatus.class) })
public Response uploadFeed(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @NotNull @FormDataParam("uploadKey") String uploadKey, @FormDataParam("categorySystemName") String categorySystemName, @FormDataParam("disableFeedUponImport") @DefaultValue("false") boolean disableFeedUponImport, @FormDataParam("importComponents") String importComponents) throws Exception {
    ImportFeedOptions options = new ImportFeedOptions();
    options.setUploadKey(uploadKey);
    options.setDisableUponImport(disableFeedUponImport);
    ImportFeed importFeed = null;
    options.setCategorySystemName(categorySystemName);
    boolean overwriteFeed = true;
    boolean overwriteTemplate = true;
    uploadProgressService.newUpload(uploadKey);
    FeedImporter feedImporter = null;
    if (importComponents == null) {
        byte[] content = ImportUtil.streamToByteArray(fileInputStream);
        feedImporter = feedImporterFactory.apply(fileMetaData.getFileName(), content, options);
        importFeed = feedImporter.validate();
        importFeed.setSuccess(false);
    } else {
        options.setImportComponentOptions(ObjectMapperSerializer.deserialize(importComponents, new TypeReference<Set<ImportComponentOption>>() {
        }));
        byte[] content = ImportUtil.streamToByteArray(fileInputStream);
        feedImporter = feedImporterFactory.apply(fileMetaData.getFileName(), content, options);
        importFeed = feedImporter.validateAndImport();
    }
    uploadProgressService.removeUpload(uploadKey);
    return Response.ok(importFeed).build();
}

Example 4 with ImportComponentOption

use of com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption in project kylo by Teradata.

the class AdminControllerV2 method uploadTemplate.

@POST
@Path(IMPORT_TEMPLATE)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a template xml or zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the template metadata.", response = ImportTemplate.class), @ApiResponse(code = 500, message = "There was a problem importing the template.", response = RestResponseStatus.class) })
public Response uploadTemplate(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @NotNull @FormDataParam("uploadKey") String uploadKey, @FormDataParam("importComponents") String importComponents) throws Exception {
    ImportTemplateOptions options = new ImportTemplateOptions();
    options.setUploadKey(uploadKey);
    ImportTemplate importTemplate = null;
    byte[] content = ImportUtil.streamToByteArray(fileInputStream);
    uploadProgressService.newUpload(uploadKey);
    TemplateImporter templateImporter = null;
    if (importComponents == null) {
        templateImporter = templateImporterFactory.apply(fileMetaData.getFileName(), content, options);
        importTemplate = templateImporter.validate();
        importTemplate.setSuccess(false);
    } else {
        options.setImportComponentOptions(ObjectMapperSerializer.deserialize(importComponents, new TypeReference<Set<ImportComponentOption>>() {
        }));
        templateImporter = templateImporterFactory.apply(fileMetaData.getFileName(), content, options);
        importTemplate = templateImporter.validateAndImport();
    }
    return Response.ok(importTemplate).build();
}

Example 5 with ImportComponentOption

use of com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption in project kylo by Teradata.

the class AbstractImportTemplateRoutine method importIntoNiFi.

/**
 * Import a template string into NiFi as a template to use
 */
public NiFiTemplateImport importIntoNiFi(ImportTemplate template, ImportTemplateOptions importOptions) throws ImportException {
    TemplateDTO dto = null;
    String templateName = null;
    String oldTemplateXml = null;
    ImportComponentOption nifiTemplateImport = importOptions.findImportComponentOption(ImportComponent.NIFI_TEMPLATE);
    if (nifiTemplateImport.isValidForImport()) {
        try {
            templateName = NifiTemplateParser.getTemplateName(template.getNifiTemplateXml());
            template.setTemplateName(templateName);
            dto = nifiRestClient.getTemplateByName(templateName);
            if (dto != null) {
                oldTemplateXml = nifiRestClient.getTemplateXml(dto.getId());
                template.setNifiTemplateId(dto.getId());
                if (importOptions.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE) && nifiTemplateImport.isValidForImport()) {
                    nifiRestClient.deleteTemplate(dto.getId());
                } else if (!template.isZipFile()) {
                    // if its not a zip file we need to error out if the user has decided not to overwrite when it already exists
                    uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "The template " + templateName + " already exists in NiFi. Please choose the option to replace the template and try again.", true, false);
                    template.setValid(false);
                }
            }
        } catch (ParserConfigurationException | XPathExpressionException | IOException | SAXException e) {
            throw new ImportException("The xml file you are trying to import is not a valid NiFi template.  Please try again. " + e.getMessage());
        }
        boolean register = (dto == null || (importOptions.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE) && nifiTemplateImport.isValidForImport()));
        // attempt to import the xml into NiFi if its new, or if the user said to overwrite
        if (register) {
            UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "Importing " + templateName + " into NiFi");
            log.info("Attempting to import Nifi Template: {} for file {}", templateName, template.getFileName());
            dto = nifiRestClient.importTemplate(template.getTemplateName(), template.getNifiTemplateXml());
            template.setNifiTemplateId(dto.getId());
            statusMessage.update("Imported " + templateName + " into NiFi", true);
        }
    }
    uploadProgressService.completeSection(importOptions, ImportSection.Section.IMPORT_NIFI_TEMPLATE);
    return new NiFiTemplateImport(oldTemplateXml, dto);
}