Examples with ImportTemplate com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate used on opensource projects

Example 6 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class ImportUtil method applyImportPropertiesToTemplate.

public static boolean applyImportPropertiesToTemplate(RegisteredTemplate template, ImportTemplate importTemplate, ImportComponent component) {
    ImportComponentOption option = importTemplate.getImportOptions().findImportComponentOption(component);
    if (!option.getProperties().isEmpty() && option.getProperties().stream().anyMatch(importProperty -> StringUtils.isBlank(importProperty.getPropertyValue()))) {
        importTemplate.setSuccess(false);
        importTemplate.setTemplateResults(new NifiProcessGroup());
        String msg = "Unable to import Template. Additional properties to be supplied before importing.";
        importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, msg, "");
        option.getErrorMessages().add(msg);
        return false;
    } else {
        template.getSensitiveProperties().forEach(nifiProperty -> {
            ImportProperty userSuppliedValue = option.getProperties().stream().filter(importFeedProperty -> nifiProperty.getProcessorId().equalsIgnoreCase(importFeedProperty.getProcessorId()) && nifiProperty.getKey().equalsIgnoreCase(importFeedProperty.getPropertyKey())).findFirst().orElse(null);
            // deal with nulls?
            if (userSuppliedValue == null) {
                // attempt to find it via the name
                userSuppliedValue = option.getProperties().stream().filter(importFeedProperty -> nifiProperty.getProcessorName().equalsIgnoreCase(importFeedProperty.getProcessorName()) && nifiProperty.getKey().equalsIgnoreCase(importFeedProperty.getPropertyKey())).findFirst().orElse(null);
            }
            if (userSuppliedValue != null) {
                nifiProperty.setValue(userSuppliedValue.getPropertyValue());
            }
        });
        return true;
    }
}

Example 7 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class IntegrationTestBase method createSimpleFeed.

protected FeedMetadata createSimpleFeed(String feedName, String testFile) {
    FeedCategory category = createCategory(FUNCTIONAL_TESTS);
    ImportTemplate template = importSimpleTemplate();
    FeedMetadata request = makeCreateFeedRequest(category, template, feedName, testFile);
    FeedMetadata response = createFeed(request).getFeedMetadata();
    Assert.assertEquals(request.getFeedName(), response.getFeedName());
    return response;
}

Example 8 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class FeedIT method testDataIngestFeed.

@Test
public void testDataIngestFeed() throws Exception {
    prepare();
    importSystemFeeds();
    copyDataToDropzone();
    // create new category
    FeedCategory category = createCategory(CATEGORY_NAME);
    ImportTemplate ingest = importDataIngestTemplate();
    // create standard ingest feed
    FeedMetadata feed = getCreateFeedRequest(category, ingest, createNewFeedName());
    FeedMetadata response = createFeed(feed).getFeedMetadata();
    Assert.assertEquals(feed.getFeedName(), response.getFeedName());
    waitForFeedToComplete();
    assertExecutedJobs(response.getFeedName(), response.getFeedId());
    failJobs(response.getCategoryAndFeedName());
    abandonAllJobs(response.getCategoryAndFeedName());
}

Example 9 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class ImportConnectedReusableTemplateIT method registerConnectedReusableTemplate.

public ConnectedTemplate registerConnectedReusableTemplate() {
    URL resource = IntegrationTestBase.class.getResource("connecting_reusable_flow.xml");
    ImportTemplate template1 = importReusableFlowXmlTemplate(resource.getPath(), null);
    String template1ProcessGroupId = template1.getTemplateResults().getProcessGroupEntity().getId();
    // Now import a reusable flow that has additional output ports in it.
    // Kylo will prompt to connect this to another reusable flow
    URL resource2 = IntegrationTestBase.class.getResource("reusable-flow1.xml");
    ImportTemplate template2 = importReusableFlowXmlTemplate(resource2.getPath(), null);
    // verify it failed
    Assert.assertFalse(template2.isSuccess());
    Assert.assertTrue(template2.isReusableFlowOutputPortConnectionsNeeded());
    // reassign the connections to connect to template1
    ReusableTemplateConnectionInfo connectionInfo = template2.getReusableTemplateConnections().stream().filter(conn -> "to another flow".equalsIgnoreCase(conn.getFeedOutputPortName())).findFirst().orElse(null);
    // Obtain the reusable connection input ports
    // get v1/feedmgr/nifi/reusable-input-ports
    PortDTO[] reusableInputPorts = getReusableInputPorts();
    // the 'connecting_reusable_flow.xml' has an input port named 'from reusable port'.
    // find it and try to connect it to this one
    PortDTO connectingPort = Arrays.stream(reusableInputPorts).filter(portDTO -> portDTO.getName().equalsIgnoreCase("from reusable port")).findFirst().orElse(null);
    if (connectingPort != null) {
        connectionInfo.setInputPortDisplayName(connectingPort.getName());
        connectionInfo.setReusableTemplateInputPortName(connectingPort.getName());
        connectionInfo.setReusableTemplateProcessGroupName(template1.getTemplateResults().getProcessGroupEntity().getName());
    }
    template2 = importReusableFlowXmlTemplate(resource2.getPath(), connectionInfo);
    Assert.assertTrue(template2.isSuccess());
    Assert.assertFalse(template2.isReusableFlowOutputPortConnectionsNeeded());
    // get the flow for the parent processor and verify it is connected to the other reusable flow
    NifiFlowProcessGroup flow = getFlow(template2.getTemplateResults().getProcessGroupEntity().getId());
    Assert.assertNotNull(flow);
    boolean testUpdate2ProcessorExists = flow.getProcessorMap().values().stream().anyMatch(p -> "test-update2".equalsIgnoreCase(p.getName()));
    Assert.assertTrue(testUpdate2ProcessorExists);
    return new ConnectedTemplate(template1, template2);
}

Example 10 with ImportTemplate

use of com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate in project kylo by Teradata.

the class FeedImporter method importFeed.

// Import
/**
 * Import a feed zip file
 */
private ImportFeed importFeed() throws Exception {
    // read the JSON into the Feed object
    FeedMetadata metadata = importFeed.getFeedToImport();
    // query for this feed.
    String feedCategory = StringUtils.isNotBlank(importFeedOptions.getCategorySystemName()) ? importFeedOptions.getCategorySystemName() : metadata.getSystemCategoryName();
    FeedMetadata existingFeed = metadataAccess.read(() -> metadataService.getFeedByName(feedCategory, metadata.getSystemFeedName()));
    metadata.getCategory().setSystemName(feedCategory);
    ImportTemplateOptions importTemplateOptions = new ImportTemplateOptions();
    importTemplateOptions.setImportComponentOptions(importFeedOptions.getImportComponentOptions());
    importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setContinueIfExists(true);
    ImportTemplate importTemplate = importFeed.getTemplate();
    importTemplate.setImportOptions(importTemplateOptions);
    importTemplateOptions.setUploadKey(importFeedOptions.getUploadKey());
    importTemplate.setValid(true);
    importTemplateOptions.setDeferCleanup(true);
    // Import the Template
    ImportTemplateRoutine importTemplateRoutine = importTemplateRoutineFactory.apply(importTemplate, importTemplateOptions, ImportTemplate.TYPE.ARCHIVE);
    importTemplateRoutine.importTemplate();
    if (importTemplate.isSuccess()) {
        // import the feed
        importFeed.setTemplate(importTemplate);
        // now that we have the Feed object we need to create the instance of the feed
        UploadProgressMessage uploadProgressMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Saving  and creating feed instance in NiFi");
        metadata.setIsNew(existingFeed == null ? true : false);
        metadata.setFeedId(existingFeed != null ? existingFeed.getFeedId() : null);
        metadata.setId(existingFeed != null ? existingFeed.getId() : null);
        // reassign the templateId to the newly registered template id
        metadata.setTemplateId(importTemplate.getTemplateId());
        if (metadata.getRegisteredTemplate() != null) {
            metadata.getRegisteredTemplate().setNifiTemplateId(importTemplate.getNifiTemplateId());
            metadata.getRegisteredTemplate().setId(importTemplate.getTemplateId());
        }
        // get/create category
        FeedCategory category = metadataService.getCategoryBySystemName(metadata.getCategory().getSystemName());
        if (category == null) {
            metadata.getCategory().setId(null);
            metadataService.saveCategory(metadata.getCategory());
        } else {
            metadata.setCategory(category);
        }
        if (importFeedOptions.isDisableUponImport()) {
            metadata.setActive(false);
            metadata.setState(FeedMetadata.STATE.DISABLED.name());
        }
        // remap any preconditions to this new feed/category name.
        if (metadata.getSchedule().hasPreconditions()) {
            metadata.getSchedule().getPreconditions().stream().flatMap(preconditionRule -> preconditionRule.getProperties().stream()).filter(fieldRuleProperty -> PolicyPropertyTypes.PROPERTY_TYPE.currentFeed.name().equals(fieldRuleProperty.getType())).forEach(fieldRuleProperty -> fieldRuleProperty.setValue(metadata.getCategoryAndFeedName()));
        }
        // //for all those properties where the template value is != userEditable and the template value has a metadata. property, remove that property from the feed properties so it can be imported and assigned correctly
        RegisteredTemplate template1 = registeredTemplateService.findRegisteredTemplateById(importTemplate.getTemplateId());
        if (template1 != null) {
            // Find all the properties in the template that have ${metadata. and are not userEditable.
            // These are the properties we need to replace on the feed metadata
            List<NifiProperty> metadataProperties = template1.getProperties().stream().filter(nifiProperty -> {
                return nifiProperty != null && StringUtils.isNotBlank(nifiProperty.getValue()) && !nifiProperty.isUserEditable() && nifiProperty.getValue().contains("${" + MetadataFieldAnnotationFieldNameResolver.metadataPropertyPrefix);
            }).collect(Collectors.toList());
            // Replace the Feed Metadata properties with those that match the template ones from above.
            List<NifiProperty> updatedProperties = metadata.getProperties().stream().map(nifiProperty -> {
                NifiProperty p = NifiPropertyUtil.findPropertyByProcessorName(metadataProperties, nifiProperty);
                return p != null ? p : nifiProperty;
            }).collect(Collectors.toList());
            metadata.setProperties(updatedProperties);
        }
        NifiFeed nifiFeed = metadataService.createFeed(metadata);
        if (nifiFeed != null) {
            importFeed.setFeedName(nifiFeed.getFeedMetadata().getCategoryAndFeedName());
            if (nifiFeed.isSuccess()) {
                uploadProgressMessage.update("Successfully saved the feed " + importFeed.getFeedName(), true);
            } else {
                if (nifiFeed.getFeedProcessGroup() != null && nifiFeed.getFeedProcessGroup().isRolledBack()) {
                    if (importTemplateRoutine != null) {
                        importTemplateRoutine.rollback();
                    }
                }
                uploadProgressMessage.update("Errors were found importing the feed " + importFeed.getFeedName(), false);
            }
            importTemplateRoutine.cleanup();
        }
        importFeed.setNifiFeed(nifiFeed);
        importFeed.setSuccess(nifiFeed != null && nifiFeed.isSuccess());
    } else {
        importFeed.setSuccess(false);
        importFeed.setTemplate(importTemplate);
        importFeed.addErrorMessage(existingFeed, "The feed " + FeedNameUtil.fullName(feedCategory, metadata.getSystemFeedName()) + " needs additional properties to be supplied before importing.");
    }
    uploadProgressService.completeSection(importFeedOptions, ImportSection.Section.IMPORT_FEED_DATA);
    return importFeed;
}