use of com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed in project kylo by Teradata.
the class AdminControllerV2 method uploadFeed.
@POST
@Path(IMPORT_FEED)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a feed zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed metadata.", response = ImportFeed.class), @ApiResponse(code = 500, message = "There was a problem importing the feed.", response = RestResponseStatus.class) })
public Response uploadFeed(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @NotNull @FormDataParam("uploadKey") String uploadKey, @FormDataParam("categorySystemName") String categorySystemName, @FormDataParam("disableFeedUponImport") @DefaultValue("false") boolean disableFeedUponImport, @FormDataParam("importComponents") String importComponents) throws Exception {
ImportFeedOptions options = new ImportFeedOptions();
options.setUploadKey(uploadKey);
options.setDisableUponImport(disableFeedUponImport);
ImportFeed importFeed = null;
options.setCategorySystemName(categorySystemName);
boolean overwriteFeed = true;
boolean overwriteTemplate = true;
uploadProgressService.newUpload(uploadKey);
FeedImporter feedImporter = null;
if (importComponents == null) {
byte[] content = ImportUtil.streamToByteArray(fileInputStream);
feedImporter = feedImporterFactory.apply(fileMetaData.getFileName(), content, options);
importFeed = feedImporter.validate();
importFeed.setSuccess(false);
} else {
options.setImportComponentOptions(ObjectMapperSerializer.deserialize(importComponents, new TypeReference<Set<ImportComponentOption>>() {
}));
byte[] content = ImportUtil.streamToByteArray(fileInputStream);
feedImporter = feedImporterFactory.apply(fileMetaData.getFileName(), content, options);
importFeed = feedImporter.validateAndImport();
}
uploadProgressService.removeUpload(uploadKey);
return Response.ok(importFeed).build();
}
use of com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed in project kylo by Teradata.
the class AdminController method uploadFeed.
/**
* This is used for quick import via a script. The UI uses the AdminControllerV2 class
*/
@POST
@Path(IMPORT_FEED)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Imports a feed zip file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed metadata.", response = ImportFeed.class), @ApiResponse(code = 500, message = "There was a problem importing the feed.", response = RestResponseStatus.class) })
public Response uploadFeed(@NotNull @FormDataParam("file") InputStream fileInputStream, @NotNull @FormDataParam("file") FormDataContentDisposition fileMetaData, @FormDataParam("overwrite") @DefaultValue("false") boolean overwrite, @FormDataParam("overwriteFeedTemplate") @DefaultValue("false") boolean overwriteFeedTemplate, @FormDataParam("categorySystemName") String categorySystemName, @FormDataParam("importConnectingReusableFlow") @DefaultValue("NOT_SET") ImportTemplateOptions.IMPORT_CONNECTING_FLOW importConnectingFlow, @FormDataParam("templateProperties") @DefaultValue("") String templateProperties, @FormDataParam("feedProperties") @DefaultValue("") String feedProperties) throws Exception {
ImportFeedOptions options = new ImportFeedOptions();
String uploadKey = uploadProgressService.newUpload();
options.setUploadKey(uploadKey);
options.findImportComponentOption(ImportComponent.FEED_DATA).setOverwrite(overwrite);
options.findImportComponentOption(ImportComponent.FEED_DATA).setUserAcknowledged(true);
options.findImportComponentOption(ImportComponent.FEED_DATA).setShouldImport(true);
options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setShouldImport(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES));
options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setUserAcknowledged(!importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.NOT_SET));
options.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setOverwrite(importConnectingFlow.equals(ImportTemplateOptions.IMPORT_CONNECTING_FLOW.YES) && overwriteFeedTemplate);
options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setOverwrite(overwriteFeedTemplate);
options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setUserAcknowledged(true);
options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setShouldImport(true);
options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE).setContinueIfExists(!overwriteFeedTemplate);
options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setOverwrite(overwriteFeedTemplate);
options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setUserAcknowledged(true);
options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setShouldImport(true);
options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setContinueIfExists(!overwriteFeedTemplate);
options.setCategorySystemName(categorySystemName);
if (StringUtils.isNotBlank(templateProperties)) {
List<ImportProperty> properties = ObjectMapperSerializer.deserialize(templateProperties, new TypeReference<List<ImportProperty>>() {
});
options.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setProperties(properties);
}
if (StringUtils.isNotBlank(feedProperties)) {
List<ImportProperty> properties = ObjectMapperSerializer.deserialize(feedProperties, new TypeReference<List<ImportProperty>>() {
});
options.findImportComponentOption(ImportComponent.FEED_DATA).setProperties(properties);
}
byte[] content = ImportUtil.streamToByteArray(fileInputStream);
FeedImporter feedImporter = feedImporterFactory.apply(fileMetaData.getFileName(), content, options);
ImportFeed importFeed = feedImporter.validateAndImport();
return Response.ok(importFeed).build();
}
use of com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed in project kylo by Teradata.
the class FeedImporter method validate.
public ImportFeed validate() {
UploadProgressMessage feedImportStatusMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Validating Feed import.");
try {
init();
// read the JSON into the Feed object
FeedMetadata metadata = importFeed.getFeedToImport();
// validate the incoming category exists
validateFeedCategory();
// verify if we should overwrite the feed if it already exists
String feedCategory = StringUtils.isNotBlank(importFeedOptions.getCategorySystemName()) ? importFeedOptions.getCategorySystemName() : metadata.getSystemCategoryName();
// query for this feed.
// first read in the feed as a service account
FeedMetadata existingFeed = metadataAccess.read(() -> {
return metadataService.getFeedByName(feedCategory, metadata.getSystemFeedName());
}, MetadataAccess.SERVICE);
if (!validateOverwriteExistingFeed(existingFeed)) {
// exit
return importFeed;
}
if (accessController.isEntityAccessControlled()) {
if (!validateEntityAccess(existingFeed, feedCategory)) {
return importFeed;
}
}
// sensitive properties
if (!validateProperties()) {
return importFeed;
}
// Valid data sources
if (!validateUserDatasources()) {
return importFeed;
}
if (!validateUserDataSets()) {
return importFeed;
}
// final ImportComponentOption templateImportOption = importFeedOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA);
// if(templateImportOption.isShouldImport() && templateImportOption.isOverwrite()) {
// UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(),"Validating the template data");
TemplateImporter templateImporter = templateImporterFactory.apply(importFeed.getFileName(), file, importFeedOptions);
ImportTemplate importTemplate = templateImporter.validate();
// need to set the importOptions back to the feed options
// find importOptions for the Template and add them back to the set of options
// importFeed.getImportOptions().updateOptions(importTemplate.getImportOptions().getImportComponentOptions());
importFeed.setTemplate(importTemplate);
// statusMessage.update("Validated the template data",importTemplate.isValid());
if (!importTemplate.isValid()) {
importFeed.setValid(false);
List<String> errorMessages = importTemplate.getTemplateResults().getAllErrors().stream().map(nifiError -> nifiError.getMessage()).collect(Collectors.toList());
if (!errorMessages.isEmpty()) {
for (String msg : errorMessages) {
importFeed.addErrorMessage(metadata, msg);
}
}
return importFeed;
}
// }
// statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(),"Validation complete: the feed is "+(importFeed.isValid() ? "valid" : "invalid"),true,importFeed.isValid());
} catch (Exception e) {
feedImportStatusMessage.update("Validation error. Feed import error: " + e.getMessage(), false);
throw new UnsupportedOperationException("Error importing template " + fileName + ". " + e.getMessage());
}
feedImportStatusMessage.update("Validated Feed import.", importFeed.isValid());
return this.importFeed;
}
use of com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed in project kylo by Teradata.
the class SearchEsIntegrationTestBase method importSampleFeed.
protected ImportFeed importSampleFeed() {
String pathToKyloFeedZip = Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResource(FEEDS_SEARCH_SAMPLE_FEED_PATH)).getPath();
ImportFeed sampleImportFeed = importFeed(pathToKyloFeedZip);
LOG.info("Imported Feed ID: {}", sampleImportFeed.getNifiFeed().getFeedMetadata().getFeedId());
LOG.info("Imported Category.Feed Display Name: {}", sampleImportFeed.getNifiFeed().getFeedMetadata().getCategoryAndFeedDisplayName());
return sampleImportFeed;
}
use of com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed in project kylo by Teradata.
the class FeedIT method importSystemFeeds.
protected void importSystemFeeds() {
ImportFeed textIndex = importFeed(sampleFeedsPath + INDEX_TEXT_SERVICE_V2_FEED_ZIP);
enableFeed(textIndex.getNifiFeed().getFeedMetadata().getFeedId());
}
Aggregations