Examples with RegisteredTemplate com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate used on opensource projects

Example 21 with RegisteredTemplate

use of com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate in project kylo by Teradata.

the class FileObjectPersistence method getTemplatesFromFile.

public Collection<RegisteredTemplate> getTemplatesFromFile() {
    ObjectMapper mapper = new ObjectMapper();
    File file = new File(filePath + "/" + TEMPLATE_METADATA_FILENAME);
    Collection<RegisteredTemplate> templates = null;
    if (file.exists()) {
        try {
            templates = mapper.readValue(file, new TypeReference<List<RegisteredTemplate>>() {
            });
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    return templates;
}

Example 22 with RegisteredTemplate

use of com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate in project kylo by Teradata.

the class DefaultFeedManagerFeedService method assignFeedDatasources.

/**
 * Assign the feed sources/destinations
 *
 * @param feed       the feed rest model
 * @param domainFeed the domain feed
 */
private void assignFeedDatasources(FeedMetadata feed, Feed domainFeed) {
    final Feed.ID domainFeedId = domainFeed.getId();
    Set<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> sources = new HashSet<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID>();
    Set<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> destinations = new HashSet<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID>();
    String uniqueName = FeedNameUtil.fullName(feed.getCategory().getSystemName(), feed.getSystemFeedName());
    RegisteredTemplate template = feed.getRegisteredTemplate();
    if (template == null) {
        // fetch it for checks
        template = templateRestProvider.getRegisteredTemplate(feed.getTemplateId());
    }
    // find Definition registration
    derivedDatasourceFactory.populateDatasources(feed, template, sources, destinations);
    if (domainFeed.getSources() != null) {
        Set<Datasource.ID> existingSourceIds = ((List<FeedSource>) domainFeed.getSources()).stream().filter(source -> source.getDatasource() != null).map(source1 -> source1.getDatasource().getId()).collect(Collectors.toSet());
        if (!sources.containsAll(existingSourceIds) || (sources.size() != existingSourceIds.size())) {
            // remove older sources
            // cant do it here for some reason.. need to do it in a separate transaction
            feedProvider.removeFeedSources(domainFeedId);
        }
    }
    sources.stream().forEach(sourceId -> feedProvider.ensureFeedSource(domainFeedId, sourceId));
    destinations.stream().forEach(sourceId -> feedProvider.ensureFeedDestination(domainFeedId, sourceId));
}

Example 23 with RegisteredTemplate

use of com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate in project kylo by Teradata.

the class DefaultFeedManagerFeedService method createAndSaveFeed.

/**
 * Create/Update a Feed in NiFi. Save the metadata to Kylo meta store.
 *
 * @param feedMetadata the feed metadata
 * @return an object indicating if the feed creation was successful or not
 */
private NifiFeed createAndSaveFeed(FeedMetadata feedMetadata) {
    Stopwatch stopwatch = Stopwatch.createStarted();
    NifiFeed feed = null;
    if (StringUtils.isBlank(feedMetadata.getId())) {
        feedMetadata.setIsNew(true);
        // If the feed is New we need to ensure the user has CREATE_FEED entity permission
        if (accessController.isEntityAccessControlled()) {
            metadataAccess.read(() -> {
                // ensure the user has rights to create feeds under the category
                Category domainCategory = categoryProvider.findById(categoryProvider.resolveId(feedMetadata.getCategory().getId()));
                if (domainCategory == null) {
                    // throw exception
                    throw new MetadataRepositoryException("Unable to find the category " + feedMetadata.getCategory().getSystemName());
                }
                domainCategory.getAllowedActions().checkPermission(CategoryAccessControl.CREATE_FEED);
                // ensure the user has rights to create feeds using the template
                FeedManagerTemplate domainTemplate = templateProvider.findById(templateProvider.resolveId(feedMetadata.getTemplateId()));
                if (domainTemplate == null) {
                    throw new MetadataRepositoryException("Unable to find the template " + feedMetadata.getTemplateId());
                }
            // domainTemplate.getAllowedActions().checkPermission(TemplateAccessControl.CREATE_FEED);
            });
        }
    } else if (accessController.isEntityAccessControlled()) {
        metadataAccess.read(() -> {
            // perform explict entity access check here as we dont want to modify the NiFi flow unless user has access to edit the feed
            Feed.ID domainId = feedProvider.resolveId(feedMetadata.getId());
            Feed domainFeed = feedProvider.findById(domainId);
            if (domainFeed != null) {
                domainFeed.getAllowedActions().checkPermission(FeedAccessControl.EDIT_DETAILS);
            } else {
                throw new NotFoundException("Feed not found for id " + feedMetadata.getId());
            }
        });
    }
    // replace expressions with values
    if (feedMetadata.getTable() != null) {
        feedMetadata.getTable().updateMetadataFieldValues();
    }
    if (feedMetadata.getProperties() == null) {
        feedMetadata.setProperties(new ArrayList<NifiProperty>());
    }
    // store ref to the originalFeedProperties before resolving and merging with the template
    List<NifiProperty> originalFeedProperties = feedMetadata.getProperties();
    // get all the properties for the metadata
    RegisteredTemplate registeredTemplate = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(feedMetadata.getTemplateId()).templateName(feedMetadata.getTemplateName()).isFeedEdit(true).includeSensitiveProperties(true).build());
    // copy the registered template properties it a new list so it doest get updated
    List<NifiProperty> templateProperties = registeredTemplate.getProperties().stream().map(nifiProperty -> new NifiProperty(nifiProperty)).collect(Collectors.toList());
    // update the template properties with the feedMetadata properties
    List<NifiProperty> matchedProperties = NifiPropertyUtil.matchAndSetPropertyByProcessorName(templateProperties, feedMetadata.getProperties(), NifiPropertyUtil.PROPERTY_MATCH_AND_UPDATE_MODE.UPDATE_ALL_PROPERTIES);
    registeredTemplate.setProperties(templateProperties);
    feedMetadata.setProperties(registeredTemplate.getProperties());
    feedMetadata.setRegisteredTemplate(registeredTemplate);
    // skip any properties that the user supplied which are not ${ values
    List<NifiProperty> propertiesToSkip = originalFeedProperties.stream().filter(property -> !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
    List<NifiProperty> templatePropertiesToSkip = registeredTemplate.getProperties().stream().filter(property -> property.isSelected() && !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
    if (templatePropertiesToSkip != null && !templatePropertiesToSkip.isEmpty()) {
        propertiesToSkip.addAll(templatePropertiesToSkip);
    }
    // resolve any ${metadata.} properties
    List<NifiProperty> resolvedProperties = propertyExpressionResolver.resolvePropertyExpressions(feedMetadata, propertiesToSkip);
    // decrypt the metadata
    feedModelTransform.decryptSensitivePropertyValues(feedMetadata);
    FeedMetadata.STATE state = FeedMetadata.STATE.NEW;
    try {
        state = FeedMetadata.STATE.valueOf(feedMetadata.getState());
    } catch (Exception e) {
    // if the string isnt valid, disregard as it will end up disabling the feed.
    }
    boolean enabled = (FeedMetadata.STATE.NEW.equals(state) && feedMetadata.isActive()) || FeedMetadata.STATE.ENABLED.equals(state);
    // flag to indicate to enable the feed later
    // if this is the first time for this feed and it is set to be enabled, mark it to be enabled after we commit to the JCR store
    boolean enableLater = false;
    if (enabled && feedMetadata.isNew()) {
        enableLater = true;
        enabled = false;
        feedMetadata.setState(FeedMetadata.STATE.DISABLED.name());
    }
    CreateFeedBuilder feedBuilder = CreateFeedBuilder.newFeed(nifiRestClient, nifiFlowCache, feedMetadata, registeredTemplate.getNifiTemplateId(), propertyExpressionResolver, propertyDescriptorTransform, niFiObjectCache, templateConnectionUtil).enabled(enabled).removeInactiveVersionedProcessGroup(removeInactiveNifiVersionedFeedFlows).autoAlign(nifiAutoFeedsAlignAfterSave).withNiFiTemplateCache(niFiTemplateCache);
    if (registeredTemplate.isReusableTemplate()) {
        feedBuilder.setReusableTemplate(true);
        feedMetadata.setIsReusableFeed(true);
    } else {
        feedBuilder.inputProcessorType(feedMetadata.getInputProcessorType()).feedSchedule(feedMetadata.getSchedule()).properties(feedMetadata.getProperties());
        if (registeredTemplate.usesReusableTemplate()) {
            for (ReusableTemplateConnectionInfo connection : registeredTemplate.getReusableTemplateConnections()) {
                feedBuilder.addInputOutputPort(new InputOutputPort(connection.getReusableTemplateInputPortName(), connection.getFeedOutputPortName()));
            }
        }
    }
    stopwatch.stop();
    log.debug("Time to prepare data for saving feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    stopwatch.start();
    NifiProcessGroup entity = feedBuilder.build();
    stopwatch.stop();
    log.debug("Time to save feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    feed = new NifiFeed(feedMetadata, entity);
    // set the original feedProperties back to the feed
    feedMetadata.setProperties(originalFeedProperties);
    // encrypt the metadata properties
    feedModelTransform.encryptSensitivePropertyValues(feedMetadata);
    if (entity.isSuccess()) {
        feedMetadata.setNifiProcessGroupId(entity.getProcessGroupEntity().getId());
        try {
            stopwatch.start();
            saveFeed(feedMetadata);
            // tell NiFi if this is a streaming feed or not
            if (feedMetadata.getRegisteredTemplate().isStream()) {
                streamingFeedJmsNotificationService.updateNiFiStatusJMSTopic(entity, feedMetadata);
            }
            feed.setEnableAfterSave(enableLater);
            feed.setSuccess(true);
            stopwatch.stop();
            log.debug("Time to saveFeed in Kylo: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            stopwatch.start();
            feedBuilder.checkAndRemoveVersionedProcessGroup();
        } catch (Exception e) {
            feed.setSuccess(false);
            feed.addErrorMessage(e);
        }
    } else {
        feed.setSuccess(false);
    }
    if (!feed.isSuccess()) {
        if (!entity.isRolledBack()) {
            try {
                feedBuilder.rollback();
            } catch (FeedRollbackException rollbackException) {
                log.error("Error rolling back feed {}. {} ", feedMetadata.getCategoryAndFeedName(), rollbackException.getMessage());
                feed.addErrorMessage("Error occurred in rolling back the Feed.");
            }
            entity.setRolledBack(true);
        }
    }
    return feed;
}

Example 24 with RegisteredTemplate

use of com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate in project kylo by Teradata.

the class FeedImporter method importFeed.

// Import
/**
 * Import a feed zip file
 */
private ImportFeed importFeed() throws Exception {
    // read the JSON into the Feed object
    FeedMetadata metadata = importFeed.getFeedToImport();
    // query for this feed.
    String feedCategory = StringUtils.isNotBlank(importFeedOptions.getCategorySystemName()) ? importFeedOptions.getCategorySystemName() : metadata.getSystemCategoryName();
    FeedMetadata existingFeed = metadataAccess.read(() -> metadataService.getFeedByName(feedCategory, metadata.getSystemFeedName()));
    metadata.getCategory().setSystemName(feedCategory);
    ImportTemplateOptions importTemplateOptions = new ImportTemplateOptions();
    importTemplateOptions.setImportComponentOptions(importFeedOptions.getImportComponentOptions());
    importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setContinueIfExists(true);
    ImportTemplate importTemplate = importFeed.getTemplate();
    importTemplate.setImportOptions(importTemplateOptions);
    importTemplateOptions.setUploadKey(importFeedOptions.getUploadKey());
    importTemplate.setValid(true);
    importTemplateOptions.setDeferCleanup(true);
    // Import the Template
    ImportTemplateRoutine importTemplateRoutine = importTemplateRoutineFactory.apply(importTemplate, importTemplateOptions, ImportTemplate.TYPE.ARCHIVE);
    importTemplateRoutine.importTemplate();
    if (importTemplate.isSuccess()) {
        // import the feed
        importFeed.setTemplate(importTemplate);
        // now that we have the Feed object we need to create the instance of the feed
        UploadProgressMessage uploadProgressMessage = uploadProgressService.addUploadStatus(importFeedOptions.getUploadKey(), "Saving  and creating feed instance in NiFi");
        metadata.setIsNew(existingFeed == null ? true : false);
        metadata.setFeedId(existingFeed != null ? existingFeed.getFeedId() : null);
        metadata.setId(existingFeed != null ? existingFeed.getId() : null);
        // reassign the templateId to the newly registered template id
        metadata.setTemplateId(importTemplate.getTemplateId());
        if (metadata.getRegisteredTemplate() != null) {
            metadata.getRegisteredTemplate().setNifiTemplateId(importTemplate.getNifiTemplateId());
            metadata.getRegisteredTemplate().setId(importTemplate.getTemplateId());
        }
        // get/create category
        FeedCategory category = metadataService.getCategoryBySystemName(metadata.getCategory().getSystemName());
        if (category == null) {
            metadata.getCategory().setId(null);
            metadataService.saveCategory(metadata.getCategory());
        } else {
            metadata.setCategory(category);
        }
        if (importFeedOptions.isDisableUponImport()) {
            metadata.setActive(false);
            metadata.setState(FeedMetadata.STATE.DISABLED.name());
        }
        // remap any preconditions to this new feed/category name.
        if (metadata.getSchedule().hasPreconditions()) {
            metadata.getSchedule().getPreconditions().stream().flatMap(preconditionRule -> preconditionRule.getProperties().stream()).filter(fieldRuleProperty -> PolicyPropertyTypes.PROPERTY_TYPE.currentFeed.name().equals(fieldRuleProperty.getType())).forEach(fieldRuleProperty -> fieldRuleProperty.setValue(metadata.getCategoryAndFeedName()));
        }
        // //for all those properties where the template value is != userEditable and the template value has a metadata. property, remove that property from the feed properties so it can be imported and assigned correctly
        RegisteredTemplate template1 = registeredTemplateService.findRegisteredTemplateById(importTemplate.getTemplateId());
        if (template1 != null) {
            // Find all the properties in the template that have ${metadata. and are not userEditable.
            // These are the properties we need to replace on the feed metadata
            List<NifiProperty> metadataProperties = template1.getProperties().stream().filter(nifiProperty -> {
                return nifiProperty != null && StringUtils.isNotBlank(nifiProperty.getValue()) && !nifiProperty.isUserEditable() && nifiProperty.getValue().contains("${" + MetadataFieldAnnotationFieldNameResolver.metadataPropertyPrefix);
            }).collect(Collectors.toList());
            // Replace the Feed Metadata properties with those that match the template ones from above.
            List<NifiProperty> updatedProperties = metadata.getProperties().stream().map(nifiProperty -> {
                NifiProperty p = NifiPropertyUtil.findPropertyByProcessorName(metadataProperties, nifiProperty);
                return p != null ? p : nifiProperty;
            }).collect(Collectors.toList());
            metadata.setProperties(updatedProperties);
        }
        NifiFeed nifiFeed = metadataService.createFeed(metadata);
        if (nifiFeed != null) {
            importFeed.setFeedName(nifiFeed.getFeedMetadata().getCategoryAndFeedName());
            if (nifiFeed.isSuccess()) {
                uploadProgressMessage.update("Successfully saved the feed " + importFeed.getFeedName(), true);
            } else {
                if (nifiFeed.getFeedProcessGroup() != null && nifiFeed.getFeedProcessGroup().isRolledBack()) {
                    if (importTemplateRoutine != null) {
                        importTemplateRoutine.rollback();
                    }
                }
                uploadProgressMessage.update("Errors were found importing the feed " + importFeed.getFeedName(), false);
            }
            importTemplateRoutine.cleanup();
        }
        importFeed.setNifiFeed(nifiFeed);
        importFeed.setSuccess(nifiFeed != null && nifiFeed.isSuccess());
    } else {
        importFeed.setSuccess(false);
        importFeed.setTemplate(importTemplate);
        importFeed.addErrorMessage(existingFeed, "The feed " + FeedNameUtil.fullName(feedCategory, metadata.getSystemFeedName()) + " needs additional properties to be supplied before importing.");
    }
    uploadProgressService.completeSection(importFeedOptions, ImportSection.Section.IMPORT_FEED_DATA);
    return importFeed;
}

Example 25 with RegisteredTemplate

use of com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method registerTemplate.

@Override
public RegisteredTemplate registerTemplate(RegisteredTemplate registeredTemplate) {
    boolean isNew = StringUtils.isBlank(registeredTemplate.getId());
    // detect if we changed the stream setting
    RegisteredTemplate previousTemplate = null;
    if (!isNew) {
        // run as service account
        previousTemplate = registeredTemplateService.findRegisteredTemplate(RegisteredTemplateRequest.requestAccessAsServiceAccountByTemplateName(registeredTemplate.getTemplateName()));
    }
    RegisteredTemplate template = saveRegisteredTemplate(registeredTemplate);
    if (template.isUpdated()) {
        nifiFlowCache.updateRegisteredTemplate(template, true);
        // update ops manager for feed.isStream
        notifyOperationsManagerOfStreamingUpdate(template.getTemplateName(), previousTemplate);
        // only allow update to the Access control if the user has the CHANGE_PERMS permission on this template entity
        if (accessController.isEntityAccessControlled() && template.hasAction(TemplateAccessControl.CHANGE_PERMS.getSystemName())) {
            // update the access control
            registeredTemplate.toRoleMembershipChangeList().stream().forEach(roleMembershipChange -> securityService.changeTemplateRoleMemberships(template.getId(), roleMembershipChange));
        }
        // notify audit of the change
        FeedManagerTemplate.State state = FeedManagerTemplate.State.valueOf(template.getState());
        FeedManagerTemplate.ID id = templateProvider.resolveId(registeredTemplate.getId());
        MetadataChange.ChangeType changeType = isNew ? MetadataChange.ChangeType.CREATE : MetadataChange.ChangeType.UPDATE;
        notifyTemplateStateChange(registeredTemplate, id, state, changeType);
    }
    return template;
}