Search in sources :

Example 1 with ImportFeed

use of com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed in project kylo by Teradata.

the class AdminControllerV2 method uploadFeed.

@POST
@Path(IMPORT_FEED)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a feed zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed metadata.", response = ImportFeed.class), @ApiResponse(code = 500, message = "There was a problem importing the feed.", response = RestResponseStatus.class) })
public Response uploadFeed(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @NotNull @FormDataParam("uploadKey") String uploadKey, @FormDataParam("categorySystemName") String categorySystemName, @FormDataParam("disableFeedUponImport") @DefaultValue("false") boolean disableFeedUponImport, @FormDataParam("importComponents") String importComponents) throws Exception {
    ImportFeedOptions options = new ImportFeedOptions();
    options.setUploadKey(uploadKey);
    options.setDisableUponImport(disableFeedUponImport);
    ImportFeed importFeed = null;
    options.setCategorySystemName(categorySystemName);
    boolean overwriteFeed = true;
    boolean overwriteTemplate = true;
    uploadProgressService.newUpload(uploadKey);
    FeedImporter feedImporter = null;
    if (importComponents == null) {
        byte[] content = ImportUtil.streamToByteArray(fileInputStream);
        feedImporter = feedImporterFactory.apply(fileMetaData.getFileName(), content, options);
        importFeed = feedImporter.validate();
        importFeed.setSuccess(false);
    } else {
        options.setImportComponentOptions(ObjectMapperSerializer.deserialize(importComponents, new TypeReference<Set<ImportComponentOption>>() {
        }));
        byte[] content = ImportUtil.streamToByteArray(fileInputStream);
        feedImporter = feedImporterFactory.apply(fileMetaData.getFileName(), content, options);
        importFeed = feedImporter.validateAndImport();
    }
    uploadProgressService.removeUpload(uploadKey);
    return Response.ok(importFeed).build();
}
Also used : FeedImporter(com.thinkbiganalytics.feedmgr.service.feed.importing.FeedImporter) ImportFeedOptions(com.thinkbiganalytics.feedmgr.rest.model.ImportFeedOptions) ImportComponentOption(com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption) ImportFeed(com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed) TypeReference(com.fasterxml.jackson.core.type.TypeReference) Path(javax.ws.rs.Path) POST(javax.ws.rs.POST) Consumes(javax.ws.rs.Consumes) Produces(javax.ws.rs.Produces) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Example 2 with ImportFeed

use of com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed in project kylo by Teradata.

the class AdminController method uploadFeed.

/**
 * This is used for quick import via a script.  The UI uses the AdminControllerV2 class
 */
@POST
@Path(IMPORT_FEED)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a feed zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed metadata.", response = ImportFeed.class), @ApiResponse(code = 500, message = "There was a problem importing the feed.", response = RestResponseStatus.class) })
public Response uploadFeed(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @FormDataParam("overwrite") @DefaultValue("false") boolean overwrite, @FormDataParam("overwriteFeedTemplate") @DefaultValue("false") boolean overwriteFeedTemplate, @FormDataParam("categorySystemName") String categorySystemName, @FormDataParam("importConnectingReusableFlow") @DefaultValue("NOT_SET") ImportTemplateOptions.IMPORT_CONNECTING_FLOW importConnectingFlow, @FormDataParam("templateProperties") @DefaultValue("") String templateProperties, @FormDataParam("feedProperties") @DefaultValue("") String feedProperties) throws Exception {
    ImportFeedOptions options = new ImportFeedOptions();
    String uploadKey = uploadProgressService.newUpload();
    options.setUploadKey(uploadKey);
    options.findImportComponentOption(ImportComponent.FEED_DATA).setOverwrite(overwrite);
    options.findImportComponentOption(ImportComponent.FEED_DATA).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.FEED_DATA).setShouldImport(true);
    options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setShouldImport(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES));
    options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setUserAcknowledged(!importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.NOT_SET));
    options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setOverwrite(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES) && overwriteFeedTemplate);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setOverwrite(overwriteFeedTemplate);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setShouldImport(true);
    options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setContinueIfExists(!overwriteFeedTemplate);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setOverwrite(overwriteFeedTemplate);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setUserAcknowledged(true);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setShouldImport(true);
    options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setContinueIfExists(!overwriteFeedTemplate);
    options.setCategorySystemName(categorySystemName);
    if (StringUtils.isNotBlank(templateProperties)) {
        List<ImportProperty> properties = ObjectMapperSerializer.deserialize(templateProperties, new TypeReference<List<ImportProperty>>() {
        });
        options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setProperties(properties);
    }
    if (StringUtils.isNotBlank(feedProperties)) {
        List<ImportProperty> properties = ObjectMapperSerializer.deserialize(feedProperties, new TypeReference<List<ImportProperty>>() {
        });
        options.findImportComponentOption(ImportComponent.FEED_DATA).setProperties(properties);
    }
    byte[] content = ImportUtil.streamToByteArray(fileInputStream);
    FeedImporter feedImporter = feedImporterFactory.apply(fileMetaData.getFileName(), content, options);
    ImportFeed importFeed = feedImporter.validateAndImport();
    return Response.ok(importFeed).build();
}
Also used : FeedImporter(com.thinkbiganalytics.feedmgr.service.feed.importing.FeedImporter) ImportProperty(com.thinkbiganalytics.feedmgr.rest.model.ImportProperty) ImportFeedOptions(com.thinkbiganalytics.feedmgr.rest.model.ImportFeedOptions) List(java.util.List) ImportFeed(com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed) Path(javax.ws.rs.Path) POST(javax.ws.rs.POST) Consumes(javax.ws.rs.Consumes) Produces(javax.ws.rs.Produces) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Example 3 with ImportFeed

use of com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed in project kylo by Teradata.

the class FeedImporter method validate.

public ImportFeed validate() {
    UploadProgressMessage feedImportStatusMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Validating Feed import.");
    try {
        init();
        // read the JSON into the Feed object
        FeedMetadata metadata = importFeed.getFeedToImport();
        // validate the incoming category exists
        validateFeedCategory();
        // verify if we should overwrite the feed if it already exists
        String feedCategory = StringUtils.isNotBlank(importFeedOptions.getCategorySystemName()) ? importFeedOptions.getCategorySystemName() : metadata.getSystemCategoryName();
        // query for this feed.
        // first read in the feed as a service account
        FeedMetadata existingFeed = metadataAccess.read(() -> {
            return metadataService.getFeedByName(feedCategory, metadata.getSystemFeedName());
        }, MetadataAccess.SERVICE);
        if (!validateOverwriteExistingFeed(existingFeed)) {
            // exit
            return importFeed;
        }
        if (accessController.isEntityAccessControlled()) {
            if (!validateEntityAccess(existingFeed, feedCategory)) {
                return importFeed;
            }
        }
        // sensitive properties
        if (!validateProperties()) {
            return importFeed;
        }
        // Valid data sources
        if (!validateUserDatasources()) {
            return importFeed;
        }
        if (!validateUserDataSets()) {
            return importFeed;
        }
        // final ImportComponentOption templateImportOption = importFeedOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA);
        // if(templateImportOption.isShouldImport() && templateImportOption.isOverwrite()) {
        // UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(),"Validating the template data");
        TemplateImporter templateImporter = templateImporterFactory.apply(importFeed.getFileName(), file, importFeedOptions);
        ImportTemplate importTemplate = templateImporter.validate();
        // need to set the importOptions back to the feed options
        // find importOptions for the Template and add them back to the set of options
        // importFeed.getImportOptions().updateOptions(importTemplate.getImportOptions().getImportComponentOptions());
        importFeed.setTemplate(importTemplate);
        // statusMessage.update("Validated the template data",importTemplate.isValid());
        if (!importTemplate.isValid()) {
            importFeed.setValid(false);
            List<String> errorMessages = importTemplate.getTemplateResults().getAllErrors().stream().map(nifiError -> nifiError.getMessage()).collect(Collectors.toList());
            if (!errorMessages.isEmpty()) {
                for (String msg : errorMessages) {
                    importFeed.addErrorMessage(metadata, msg);
                }
            }
            return importFeed;
        }
    // }
    // statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(),"Validation complete: the feed is "+(importFeed.isValid() ? "valid" : "invalid"),true,importFeed.isValid());
    } catch (Exception e) {
        feedImportStatusMessage.update("Validation error. Feed import error: " + e.getMessage(), false);
        throw new UnsupportedOperationException("Error importing template  " + fileName + ".  " + e.getMessage());
    }
    feedImportStatusMessage.update("Validated Feed import.", importFeed.isValid());
    return this.importFeed;
}
Also used : UploadProgressMessage(com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage) UploadProgressService(com.thinkbiganalytics.feedmgr.service.UploadProgressService) Arrays(java.util.Arrays) RegisteredTemplateService(com.thinkbiganalytics.feedmgr.service.template.RegisteredTemplateService) Category(com.thinkbiganalytics.metadata.api.category.Category) DataSourceProvider(com.thinkbiganalytics.metadata.api.catalog.DataSourceProvider) LoggerFactory(org.slf4j.LoggerFactory) DatasourceModelTransform(com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform) TemplateImporter(com.thinkbiganalytics.feedmgr.service.template.importing.TemplateImporter) StringUtils(org.apache.commons.lang3.StringUtils) DataSetProvider(com.thinkbiganalytics.metadata.api.catalog.DataSetProvider) ByteArrayInputStream(java.io.ByteArrayInputStream) FeedAccessControl(com.thinkbiganalytics.metadata.api.feed.security.FeedAccessControl) FeedManagerCategoryService(com.thinkbiganalytics.feedmgr.service.category.FeedManagerCategoryService) NifiPropertyUtil(com.thinkbiganalytics.nifi.rest.support.NifiPropertyUtil) Map(java.util.Map) ImportUtil(com.thinkbiganalytics.feedmgr.util.ImportUtil) AccessController(com.thinkbiganalytics.security.AccessController) NifiFeed(com.thinkbiganalytics.feedmgr.rest.model.NifiFeed) CategoryAccessControl(com.thinkbiganalytics.metadata.api.category.security.CategoryAccessControl) FeedServicesAccessControl(com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) CatalogModelTransform(com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform) ZipEntry(java.util.zip.ZipEntry) MetadataService(com.thinkbiganalytics.feedmgr.service.MetadataService) PolicyPropertyTypes(com.thinkbiganalytics.policy.PolicyPropertyTypes) MetadataFieldAnnotationFieldNameResolver(com.thinkbiganalytics.feedmgr.MetadataFieldAnnotationFieldNameResolver) UserField(com.thinkbiganalytics.feedmgr.rest.model.UserField) ImportTemplate(com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate) NifiProperty(com.thinkbiganalytics.nifi.rest.model.NifiProperty) ImportPropertyBuilder(com.thinkbiganalytics.feedmgr.rest.model.ImportPropertyBuilder) Set(java.util.Set) UploadProgress(com.thinkbiganalytics.feedmgr.rest.model.UploadProgress) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) UploadProgressMessage(com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage) ZipFileUtil(com.thinkbiganalytics.feedmgr.support.ZipFileUtil) Objects(java.util.Objects) List(java.util.List) CategoryProvider(com.thinkbiganalytics.metadata.api.category.CategoryProvider) DataSource(com.thinkbiganalytics.metadata.api.catalog.DataSource) ImportTemplateOptions(com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions) ImportFeed(com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed) ImportFeedException(com.thinkbiganalytics.feedmgr.service.feed.ImportFeedException) ImportComponent(com.thinkbiganalytics.feedmgr.rest.ImportComponent) ZipInputStream(java.util.zip.ZipInputStream) FeedManagerFeedService(com.thinkbiganalytics.feedmgr.service.feed.FeedManagerFeedService) DataSet(com.thinkbiganalytics.metadata.api.catalog.DataSet) HashMap(java.util.HashMap) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) ImportSection(com.thinkbiganalytics.feedmgr.rest.ImportSection) UserProperty(com.thinkbiganalytics.feedmgr.rest.model.UserProperty) ArrayList(java.util.ArrayList) FeedCategory(com.thinkbiganalytics.feedmgr.rest.model.FeedCategory) HashSet(java.util.HashSet) Inject(javax.inject.Inject) ImportProperty(com.thinkbiganalytics.feedmgr.rest.model.ImportProperty) ImportType(com.thinkbiganalytics.feedmgr.rest.ImportType) ImportFeedOptions(com.thinkbiganalytics.feedmgr.rest.model.ImportFeedOptions) RegisteredTemplate(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate) TemplateImporterFactory(com.thinkbiganalytics.feedmgr.service.template.importing.TemplateImporterFactory) ConnectorProvider(com.thinkbiganalytics.metadata.api.catalog.ConnectorProvider) UserDatasource(com.thinkbiganalytics.metadata.api.datasource.UserDatasource) ImportTemplateRoutineFactory(com.thinkbiganalytics.feedmgr.service.template.importing.importprocess.ImportTemplateRoutineFactory) Logger(org.slf4j.Logger) FeedNameUtil(com.thinkbiganalytics.support.FeedNameUtil) ImportTemplateRoutine(com.thinkbiganalytics.feedmgr.service.template.importing.importprocess.ImportTemplateRoutine) IOException(java.io.IOException) ImportException(com.thinkbiganalytics.feedmgr.service.template.importing.ImportException) UserPropertyTransform(com.thinkbiganalytics.feedmgr.service.UserPropertyTransform) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) Collections(java.util.Collections) LegacyNifiRestClient(com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient) ImportComponentOption(com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption) InputStream(java.io.InputStream) TemplateImporter(com.thinkbiganalytics.feedmgr.service.template.importing.TemplateImporter) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) ImportTemplate(com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate) ImportFeedException(com.thinkbiganalytics.feedmgr.service.feed.ImportFeedException) IOException(java.io.IOException) ImportException(com.thinkbiganalytics.feedmgr.service.template.importing.ImportException)

Example 4 with ImportFeed

use of com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed in project kylo by Teradata.

the class SearchEsIntegrationTestBase method importSampleFeed.

protected ImportFeed importSampleFeed() {
    String pathToKyloFeedZip = Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResource(FEEDS_SEARCH_SAMPLE_FEED_PATH)).getPath();
    ImportFeed sampleImportFeed = importFeed(pathToKyloFeedZip);
    LOG.info("Imported Feed ID: {}", sampleImportFeed.getNifiFeed().getFeedMetadata().getFeedId());
    LOG.info("Imported Category.Feed Display Name: {}", sampleImportFeed.getNifiFeed().getFeedMetadata().getCategoryAndFeedDisplayName());
    return sampleImportFeed;
}
Also used : ImportFeed(com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed)

Example 5 with ImportFeed

use of com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed in project kylo by Teradata.

the class FeedIT method importSystemFeeds.

protected void importSystemFeeds() {
    ImportFeed textIndex = importFeed(sampleFeedsPath + INDEX_TEXT_SERVICE_V2_FEED_ZIP);
    enableFeed(textIndex.getNifiFeed().getFeedMetadata().getFeedId());
}
Also used : ImportFeed(com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed)

Aggregations

ImportFeed (com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed)10 ImportComponentOption (com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption)4 ImportFeedOptions (com.thinkbiganalytics.feedmgr.rest.model.ImportFeedOptions)4 ImportProperty (com.thinkbiganalytics.feedmgr.rest.model.ImportProperty)4 List (java.util.List)4 ImportComponent (com.thinkbiganalytics.feedmgr.rest.ImportComponent)3 ImportType (com.thinkbiganalytics.feedmgr.rest.ImportType)3 FeedMetadata (com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata)3 ImportPropertyBuilder (com.thinkbiganalytics.feedmgr.rest.model.ImportPropertyBuilder)3 RegisteredTemplate (com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate)3 ByteArrayInputStream (java.io.ByteArrayInputStream)3 InputStream (java.io.InputStream)3 ZipEntry (java.util.zip.ZipEntry)3 ZipInputStream (java.util.zip.ZipInputStream)3 TypeReference (com.fasterxml.jackson.core.type.TypeReference)2 Sets (com.google.common.collect.Sets)2 MetadataFieldAnnotationFieldNameResolver (com.thinkbiganalytics.feedmgr.MetadataFieldAnnotationFieldNameResolver)2 ImportSection (com.thinkbiganalytics.feedmgr.rest.ImportSection)2 FeedCategory (com.thinkbiganalytics.feedmgr.rest.model.FeedCategory)2 ImportTemplateOptions (com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions)2