Examples with FeedManagerTemplate com.thinkbiganalytics.metadata.api.template.FeedManagerTemplate used on opensource projects

Example 6 with FeedManagerTemplate

use of com.thinkbiganalytics.metadata.api.template.FeedManagerTemplate in project kylo by Teradata.

the class RegisteredTemplateService method hasTemplatePermission.

/**
 * Checks the current security context has been granted permission to perform the specified action(s)
 * on the template with the specified ID.  If the template does not exist then no check is made.
 *
 * @param id     the template ID
 * @param action an action to check
 * @param more   any additional actions to check
 * @return true if the template existed and the check passed, otherwise false
 */
public boolean hasTemplatePermission(final String id, final Action action, final Action... more) {
    if (accessController.isEntityAccessControlled()) {
        return metadataAccess.read(() -> {
            final FeedManagerTemplate.ID domainId = templateProvider.resolveId(id);
            final FeedManagerTemplate domainTemplate = templateProvider.findById(domainId);
            return domainTemplate != null && domainTemplate.getAllowedActions().hasPermission(action, more);
        });
    } else {
        return true;
    }
}

Example 7 with FeedManagerTemplate

use of com.thinkbiganalytics.metadata.api.template.FeedManagerTemplate in project kylo by Teradata.

the class RegisteredTemplateService method checkTemplatePermission.

/**
 * Checks the current security context has been granted permission to perform the specified action(s)
 * on the template with the specified ID.  If the template does not exist then no check is made.
 *
 * @param id     the template ID
 * @param action an action to check
 * @param more   any additional actions to check
 * @return true if the template existed, otherwise false
 * @throws AccessControlException thrown if the template exists and the action(s) checked are not permitted
 */
public boolean checkTemplatePermission(final String id, final Action action, final Action... more) {
    if (accessController.isEntityAccessControlled()) {
        return metadataAccess.read(() -> {
            final FeedManagerTemplate.ID domainId = templateProvider.resolveId(id);
            final FeedManagerTemplate domainTemplate = templateProvider.findById(domainId);
            if (domainTemplate != null) {
                domainTemplate.getAllowedActions().checkPermission(action, more);
                return true;
            } else {
                return false;
            }
        });
    } else {
        return true;
    }
}

Example 8 with FeedManagerTemplate

use of com.thinkbiganalytics.metadata.api.template.FeedManagerTemplate in project kylo by Teradata.

the class RegisteredTemplateService method saveTemplate.

/**
 * This needs to be wrapped in a MetadataAccess transaction
 *
 * @param registeredTemplate the template to save
 * @return the saved template
 */
private RegisteredTemplate saveTemplate(RegisteredTemplate registeredTemplate) {
    // ensure that the incoming template name doesnt already exist.
    // if so remove and replace with this one
    RegisteredTemplate template = findRegisteredTemplate(RegisteredTemplateRequest.requestByTemplateName(registeredTemplate.getTemplateName()));
    if (registeredTemplate.getId() == null && template != null) {
        registeredTemplate.setId(template.getId());
    }
    if (template != null && !template.getId().equalsIgnoreCase(registeredTemplate.getId())) {
        // Warning cant save.. duplicate Name
        log.error("Unable to save template {}.  There is already a template with this name registered in the system", registeredTemplate.getTemplateName());
        return null;
    } else {
        if (StringUtils.isNotBlank(registeredTemplate.getId())) {
            checkTemplatePermission(registeredTemplate.getId(), TemplateAccessControl.EDIT_TEMPLATE);
        }
        log.info("About to save Registered Template {} ({}), nifi template Id of {} ", registeredTemplate.getTemplateName(), registeredTemplate.getId(), registeredTemplate.getNifiTemplateId());
        ensureRegisteredTemplateInputProcessors(registeredTemplate);
        FeedManagerTemplate domain = templateModelTransform.REGISTERED_TEMPLATE_TO_DOMAIN.apply(registeredTemplate);
        ensureNifiTemplateId(domain);
        if (domain != null) {
            log.info("Domain Object is {} ({}), nifi template Id of {}", domain.getName(), domain.getId(), domain.getNifiTemplateId());
        }
        domain = templateProvider.update(domain);
        // query it back to display to the ui
        domain = templateProvider.findById(domain.getId());
        RegisteredTemplate updatedTemplate = templateModelTransform.DOMAIN_TO_REGISTERED_TEMPLATE.apply(domain);
        updatedTemplate.setUpdated(true);
        registeredTemplateCache.invalidateProcessors(updatedTemplate.getId());
        niFiTemplateCache.updateSelectedProperties(registeredTemplate);
        return updatedTemplate;
    }
}

Example 9 with FeedManagerTemplate

use of com.thinkbiganalytics.metadata.api.template.FeedManagerTemplate in project kylo by Teradata.

the class TemplateModelTransform method DOMAIN_TO_REGISTERED_TEMPLATE.

public final Function<FeedManagerTemplate, RegisteredTemplate> DOMAIN_TO_REGISTERED_TEMPLATE(boolean includeFeedNames, boolean includeEncryptedProperties) {
    return new Function<FeedManagerTemplate, RegisteredTemplate>() {

        @Override
        public RegisteredTemplate apply(FeedManagerTemplate domain) {
            String json = domain.getJson();
            RegisteredTemplate template = deserialize(json, includeEncryptedProperties);
            template.setId(domain.getId().toString());
            template.setState(domain.getState().name());
            template.setNifiTemplateId(domain.getNifiTemplateId());
            template.setAllowPreconditions(domain.isAllowPreconditions());
            List<Feed> feeds = domain.getFeeds();
            template.setFeedsCount(feeds == null ? 0 : feeds.size());
            template.setStream(domain.isStream());
            if (includeFeedNames && feeds != null) {
                template.setFeedNames(feeds.stream().map(feedManagerFeed -> FeedNameUtil.fullName(feedManagerFeed.getCategory().getSystemName(), feedManagerFeed.getName())).collect(Collectors.toSet()));
            }
            if (domain.getCreatedTime() != null) {
                template.setCreateDate(domain.getCreatedTime().toDate());
            }
            if (domain.getModifiedTime() != null) {
                template.setUpdateDate(domain.getModifiedTime().toDate());
            }
            template.setOrder(domain.getOrder());
            template.setTemplateTableOption(domain.getTemplateTableOption());
            securityTransform.applyAccessControl(domain, template);
            return template;
        }
    };
}

Example 10 with FeedManagerTemplate

use of com.thinkbiganalytics.metadata.api.template.FeedManagerTemplate in project kylo by Teradata.

the class DefaultFeedManagerFeedService method createAndSaveFeed.

/**
 * Create/Update a Feed in NiFi. Save the metadata to Kylo meta store.
 *
 * @param feedMetadata the feed metadata
 * @return an object indicating if the feed creation was successful or not
 */
private NifiFeed createAndSaveFeed(FeedMetadata feedMetadata) {
    Stopwatch stopwatch = Stopwatch.createStarted();
    NifiFeed feed = null;
    if (StringUtils.isBlank(feedMetadata.getId())) {
        feedMetadata.setIsNew(true);
        // If the feed is New we need to ensure the user has CREATE_FEED entity permission
        if (accessController.isEntityAccessControlled()) {
            metadataAccess.read(() -> {
                // ensure the user has rights to create feeds under the category
                Category domainCategory = categoryProvider.findById(categoryProvider.resolveId(feedMetadata.getCategory().getId()));
                if (domainCategory == null) {
                    // throw exception
                    throw new MetadataRepositoryException("Unable to find the category " + feedMetadata.getCategory().getSystemName());
                }
                domainCategory.getAllowedActions().checkPermission(CategoryAccessControl.CREATE_FEED);
                // ensure the user has rights to create feeds using the template
                FeedManagerTemplate domainTemplate = templateProvider.findById(templateProvider.resolveId(feedMetadata.getTemplateId()));
                if (domainTemplate == null) {
                    throw new MetadataRepositoryException("Unable to find the template " + feedMetadata.getTemplateId());
                }
            // domainTemplate.getAllowedActions().checkPermission(TemplateAccessControl.CREATE_FEED);
            });
        }
    } else if (accessController.isEntityAccessControlled()) {
        metadataAccess.read(() -> {
            // perform explict entity access check here as we dont want to modify the NiFi flow unless user has access to edit the feed
            Feed.ID domainId = feedProvider.resolveId(feedMetadata.getId());
            Feed domainFeed = feedProvider.findById(domainId);
            if (domainFeed != null) {
                domainFeed.getAllowedActions().checkPermission(FeedAccessControl.EDIT_DETAILS);
            } else {
                throw new NotFoundException("Feed not found for id " + feedMetadata.getId());
            }
        });
    }
    // replace expressions with values
    if (feedMetadata.getTable() != null) {
        feedMetadata.getTable().updateMetadataFieldValues();
    }
    if (feedMetadata.getProperties() == null) {
        feedMetadata.setProperties(new ArrayList<NifiProperty>());
    }
    // store ref to the originalFeedProperties before resolving and merging with the template
    List<NifiProperty> originalFeedProperties = feedMetadata.getProperties();
    // get all the properties for the metadata
    RegisteredTemplate registeredTemplate = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(feedMetadata.getTemplateId()).templateName(feedMetadata.getTemplateName()).isFeedEdit(true).includeSensitiveProperties(true).build());
    // copy the registered template properties it a new list so it doest get updated
    List<NifiProperty> templateProperties = registeredTemplate.getProperties().stream().map(nifiProperty -> new NifiProperty(nifiProperty)).collect(Collectors.toList());
    // update the template properties with the feedMetadata properties
    List<NifiProperty> matchedProperties = NifiPropertyUtil.matchAndSetPropertyByProcessorName(templateProperties, feedMetadata.getProperties(), NifiPropertyUtil.PROPERTY_MATCH_AND_UPDATE_MODE.UPDATE_ALL_PROPERTIES);
    registeredTemplate.setProperties(templateProperties);
    feedMetadata.setProperties(registeredTemplate.getProperties());
    feedMetadata.setRegisteredTemplate(registeredTemplate);
    // skip any properties that the user supplied which are not ${ values
    List<NifiProperty> propertiesToSkip = originalFeedProperties.stream().filter(property -> !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
    List<NifiProperty> templatePropertiesToSkip = registeredTemplate.getProperties().stream().filter(property -> property.isSelected() && !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
    if (templatePropertiesToSkip != null && !templatePropertiesToSkip.isEmpty()) {
        propertiesToSkip.addAll(templatePropertiesToSkip);
    }
    // resolve any ${metadata.} properties
    List<NifiProperty> resolvedProperties = propertyExpressionResolver.resolvePropertyExpressions(feedMetadata, propertiesToSkip);
    // decrypt the metadata
    feedModelTransform.decryptSensitivePropertyValues(feedMetadata);
    FeedMetadata.STATE state = FeedMetadata.STATE.NEW;
    try {
        state = FeedMetadata.STATE.valueOf(feedMetadata.getState());
    } catch (Exception e) {
    // if the string isnt valid, disregard as it will end up disabling the feed.
    }
    boolean enabled = (FeedMetadata.STATE.NEW.equals(state) && feedMetadata.isActive()) || FeedMetadata.STATE.ENABLED.equals(state);
    // flag to indicate to enable the feed later
    // if this is the first time for this feed and it is set to be enabled, mark it to be enabled after we commit to the JCR store
    boolean enableLater = false;
    if (enabled && feedMetadata.isNew()) {
        enableLater = true;
        enabled = false;
        feedMetadata.setState(FeedMetadata.STATE.DISABLED.name());
    }
    CreateFeedBuilder feedBuilder = CreateFeedBuilder.newFeed(nifiRestClient, nifiFlowCache, feedMetadata, registeredTemplate.getNifiTemplateId(), propertyExpressionResolver, propertyDescriptorTransform, niFiObjectCache, templateConnectionUtil).enabled(enabled).removeInactiveVersionedProcessGroup(removeInactiveNifiVersionedFeedFlows).autoAlign(nifiAutoFeedsAlignAfterSave).withNiFiTemplateCache(niFiTemplateCache);
    if (registeredTemplate.isReusableTemplate()) {
        feedBuilder.setReusableTemplate(true);
        feedMetadata.setIsReusableFeed(true);
    } else {
        feedBuilder.inputProcessorType(feedMetadata.getInputProcessorType()).feedSchedule(feedMetadata.getSchedule()).properties(feedMetadata.getProperties());
        if (registeredTemplate.usesReusableTemplate()) {
            for (ReusableTemplateConnectionInfo connection : registeredTemplate.getReusableTemplateConnections()) {
                feedBuilder.addInputOutputPort(new InputOutputPort(connection.getReusableTemplateInputPortName(), connection.getFeedOutputPortName()));
            }
        }
    }
    stopwatch.stop();
    log.debug("Time to prepare data for saving feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    stopwatch.start();
    NifiProcessGroup entity = feedBuilder.build();
    stopwatch.stop();
    log.debug("Time to save feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    feed = new NifiFeed(feedMetadata, entity);
    // set the original feedProperties back to the feed
    feedMetadata.setProperties(originalFeedProperties);
    // encrypt the metadata properties
    feedModelTransform.encryptSensitivePropertyValues(feedMetadata);
    if (entity.isSuccess()) {
        feedMetadata.setNifiProcessGroupId(entity.getProcessGroupEntity().getId());
        try {
            stopwatch.start();
            saveFeed(feedMetadata);
            // tell NiFi if this is a streaming feed or not
            if (feedMetadata.getRegisteredTemplate().isStream()) {
                streamingFeedJmsNotificationService.updateNiFiStatusJMSTopic(entity, feedMetadata);
            }
            feed.setEnableAfterSave(enableLater);
            feed.setSuccess(true);
            stopwatch.stop();
            log.debug("Time to saveFeed in Kylo: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            stopwatch.start();
            feedBuilder.checkAndRemoveVersionedProcessGroup();
        } catch (Exception e) {
            feed.setSuccess(false);
            feed.addErrorMessage(e);
        }
    } else {
        feed.setSuccess(false);
    }
    if (!feed.isSuccess()) {
        if (!entity.isRolledBack()) {
            try {
                feedBuilder.rollback();
            } catch (FeedRollbackException rollbackException) {
                log.error("Error rolling back feed {}. {} ", feedMetadata.getCategoryAndFeedName(), rollbackException.getMessage());
                feed.addErrorMessage("Error occurred in rolling back the Feed.");
            }
            entity.setRolledBack(true);
        }
    }
    return feed;
}