Examples with ImportProperty com.thinkbiganalytics.feedmgr.rest.model.ImportProperty used on opensource projects

Example 1 with ImportProperty

use of com.thinkbiganalytics.feedmgr.rest.model.ImportProperty in project kylo by Teradata.

the class FeedImporter method validateUserDatasources.

/**
 * Validates that user data sources can be imported with provided properties.
 *
 * @return {@code true} if the feed can be imported, or {@code false} otherwise
 */
private boolean validateUserDatasources() {
    FeedMetadata metadata = importFeed.getFeedToImport();
    final UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importFeed.getImportOptions().getUploadKey(), "Validating data sources.");
    // Get data sources needing to be created
    final Set<String> availableDatasources = metadataAccess.read(() -> datasourceProvider.getDatasources(datasourceProvider.datasetCriteria().type(UserDatasource.class)).stream().map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId).map(Object::toString).collect(Collectors.toSet()));
    final ImportComponentOption componentOption = importFeedOptions.findImportComponentOption(ImportComponent.USER_DATASOURCES);
    final List<Datasource> providedDatasources = Optional.ofNullable(metadata.getUserDatasources()).orElse(Collections.emptyList());
    if (componentOption.getProperties().isEmpty()) {
        componentOption.setProperties(providedDatasources.stream().filter(datasource -> !availableDatasources.contains(datasource.getId())).map(datasource -> new ImportProperty(datasource.getName(), datasource.getId(), null, null, null)).collect(Collectors.toList()));
    }
    // Update feed with re-mapped data sources
    final boolean valid = componentOption.getProperties().stream().allMatch(property -> {
        if (property.getPropertyValue() != null) {
            ImportUtil.replaceDatasource(metadata, property.getProcessorId(), property.getPropertyValue());
            return true;
        } else {
            return false;
        }
    });
    if (valid) {
        statusMessage.update("Validated data sources.", true);
    } else {
        statusMessage.update("Validation Error. Additional properties are needed before uploading the feed.", false);
        importFeed.setValid(false);
    }
    uploadProgressService.completeSection(importFeed.getImportOptions(), ImportSection.Section.VALIDATE_USER_DATASOURCES);
    return valid;
}

Example 2 with ImportProperty

use of com.thinkbiganalytics.feedmgr.rest.model.ImportProperty in project kylo by Teradata.

the class AdminController method uploadTemplatex.

/**
 * This is used for quick import via scripts.  The UI uses the AdminControllerV2 class
 */
@POST
@Path(IMPORT_TEMPLATE)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a template xml or zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the template metadata.", response = ImportTemplate.class), @ApiResponse(code = 500, message = "There was a problem importing the template.", response = RestResponseStatus.class) })
public Response uploadTemplatex(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @FormDataParam("overwrite") @DefaultValue("false") boolean overwrite, @FormDataParam("importConnectingReusableFlow") @DefaultValue("NOT_SET") ImportTemplateOptions.IMPORT_CONNECTING_FLOW importConnectingFlow, @FormDataParam("createReusableFlow") @DefaultValue("false") boolean createReusableFlow, @FormDataParam("templateProperties") @DefaultValue("") String templateProperties) throws Exception {
    ImportTemplateOptions options = new ImportTemplateOptions();
    String uploadKey = uploadProgressService.newUpload();
    options.setUploadKey(uploadKey);
    boolean isZip = fileMetaData.getFileName().endsWith("zip");
    if (isZip) {
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setShouldImport(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES));
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setUserAcknowledged(!importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.NOT_SET));
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setOverwrite(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES));
    } else {
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setShouldImport(createReusableFlow);
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setUserAcknowledged(true);
        options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setOverwrite(createReusableFlow);
    }
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setOverwrite(overwrite);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setShouldImport(true);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setContinueIfExists(!overwrite);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setOverwrite(overwrite);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setShouldImport(true);
    if (StringUtils.isNotBlank(templateProperties)) {
        List<ImportProperty> properties = ObjectMapperSerializer.deserialize(templateProperties, new TypeReference<List<ImportProperty>>() {
        });
        options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setProperties(properties);
    }
    byte[] content = ImportUtil.streamToByteArray(fileInputStream);
    TemplateImporter templateImporter = templateImporterFactory.apply(fileMetaData.getFileName(), content, options);
    ImportTemplate importTemplate = templateImporter.validateAndImport();
    return Response.ok(importTemplate).build();
}

Example 3 with ImportProperty

use of com.thinkbiganalytics.feedmgr.rest.model.ImportProperty in project kylo by Teradata.

the class AdminController method uploadFeed.

/**
 * This is used for quick import via a script.  The UI uses the AdminControllerV2 class
 */
@POST
@Path(IMPORT_FEED)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a feed zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed metadata.", response = ImportFeed.class), @ApiResponse(code = 500, message = "There was a problem importing the feed.", response = RestResponseStatus.class) })
public Response uploadFeed(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @FormDataParam("overwrite") @DefaultValue("false") boolean overwrite, @FormDataParam("overwriteFeedTemplate") @DefaultValue("false") boolean overwriteFeedTemplate, @FormDataParam("categorySystemName") String categorySystemName, @FormDataParam("importConnectingReusableFlow") @DefaultValue("NOT_SET") ImportTemplateOptions.IMPORT_CONNECTING_FLOW importConnectingFlow, @FormDataParam("templateProperties") @DefaultValue("") String templateProperties, @FormDataParam("feedProperties") @DefaultValue("") String feedProperties) throws Exception {
    ImportFeedOptions options = new ImportFeedOptions();
    String uploadKey = uploadProgressService.newUpload();
    options.setUploadKey(uploadKey);
    options.findImportComponentOption(ImportComponent.FEED_DATA).setOverwrite(overwrite);
    options.findImportComponentOption(ImportComponent.FEED_DATA).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.FEED_DATA).setShouldImport(true);
    options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setShouldImport(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES));
    options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setUserAcknowledged(!importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.NOT_SET));
    options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setOverwrite(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES) && overwriteFeedTemplate);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setOverwrite(overwriteFeedTemplate);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setShouldImport(true);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setContinueIfExists(!overwriteFeedTemplate);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setOverwrite(overwriteFeedTemplate);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setShouldImport(true);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setContinueIfExists(!overwriteFeedTemplate);
    options.setCategorySystemName(categorySystemName);
    if (StringUtils.isNotBlank(templateProperties)) {
        List<ImportProperty> properties = ObjectMapperSerializer.deserialize(templateProperties, new TypeReference<List<ImportProperty>>() {
        });
        options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setProperties(properties);
    }
    if (StringUtils.isNotBlank(feedProperties)) {
        List<ImportProperty> properties = ObjectMapperSerializer.deserialize(feedProperties, new TypeReference<List<ImportProperty>>() {
        });
        options.findImportComponentOption(ImportComponent.FEED_DATA).setProperties(properties);
    }
    byte[] content = ImportUtil.streamToByteArray(fileInputStream);
    FeedImporter feedImporter = feedImporterFactory.apply(fileMetaData.getFileName(), content, options);
    ImportFeed importFeed = feedImporter.validateAndImport();
    return Response.ok(importFeed).build();
}

Example 4 with ImportProperty

use of com.thinkbiganalytics.feedmgr.rest.model.ImportProperty in project kylo by Teradata.

the class ImportUtil method addToImportOptionsSensitiveProperties.

public static void addToImportOptionsSensitiveProperties(ImportOptions importOptions, List<NifiProperty> sensitiveProperties, ImportComponent component) {
    ImportComponentOption option = importOptions.findImportComponentOption(component);
    if (option.getProperties().isEmpty()) {
        option.setProperties(sensitiveProperties.stream().map(p -> new ImportProperty(p.getProcessorName(), p.getProcessorId(), p.getKey(), "", p.getProcessorType())).collect(Collectors.toList()));
    } else {
        // only add in those that are unique
        Map<String, ImportProperty> propertyMap = option.getProperties().stream().collect(Collectors.toMap(p -> p.getProcessorNameTypeKey(), p -> p));
        sensitiveProperties.stream().filter(nifiProperty -> !propertyMap.containsKey(nifiProperty.getProcessorNameTypeKey())).forEach(p -> {
            option.getProperties().add(new ImportProperty(p.getProcessorName(), p.getProcessorId(), p.getKey(), "", p.getProcessorType()));
        });
    }
}

Example 5 with ImportProperty

use of com.thinkbiganalytics.feedmgr.rest.model.ImportProperty in project kylo by Teradata.

the class ImportUtil method applyImportPropertiesToTemplate.

public static boolean applyImportPropertiesToTemplate(RegisteredTemplate template, ImportTemplate importTemplate, ImportComponent component) {
    ImportComponentOption option = importTemplate.getImportOptions().findImportComponentOption(component);
    if (!option.getProperties().isEmpty() && option.getProperties().stream().anyMatch(importProperty -> StringUtils.isBlank(importProperty.getPropertyValue()))) {
        importTemplate.setSuccess(false);
        importTemplate.setTemplateResults(new NifiProcessGroup());
        String msg = "Unable to import Template. Additional properties to be supplied before importing.";
        importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, msg, "");
        option.getErrorMessages().add(msg);
        return false;
    } else {
        template.getSensitiveProperties().forEach(nifiProperty -> {
            ImportProperty userSuppliedValue = option.getProperties().stream().filter(importFeedProperty -> nifiProperty.getProcessorId().equalsIgnoreCase(importFeedProperty.getProcessorId()) && nifiProperty.getKey().equalsIgnoreCase(importFeedProperty.getPropertyKey())).findFirst().orElse(null);
            // deal with nulls?
            if (userSuppliedValue == null) {
                // attempt to find it via the name
                userSuppliedValue = option.getProperties().stream().filter(importFeedProperty -> nifiProperty.getProcessorName().equalsIgnoreCase(importFeedProperty.getProcessorName()) && nifiProperty.getKey().equalsIgnoreCase(importFeedProperty.getPropertyKey())).findFirst().orElse(null);
            }
            if (userSuppliedValue != null) {
                nifiProperty.setValue(userSuppliedValue.getPropertyValue());
            }
        });
        return true;
    }
}