Search in sources :

Example 11 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class DefaultFeedManagerFeedService method createAndSaveFeed.

/**
 * Create/Update a Feed in NiFi. Save the metadata to Kylo meta store.
 *
 * @param feedMetadata the feed metadata
 * @return an object indicating if the feed creation was successful or not
 * @deprecated
 */
@Deprecated
private NifiFeed createAndSaveFeed(final FeedMetadata feedMetadata) {
    Stopwatch stopwatch = Stopwatch.createStarted();
    NifiFeed feed = null;
    feedMetadata.setIsNew(StringUtils.isBlank(feedMetadata.getId()));
    metadataAccess.read(() -> {
        if (feedMetadata.isNew()) {
            // ensure the user has rights to create feeds under the category by checking if it is visible (has read access)
            Category domainCategory = categoryProvider.findById(categoryProvider.resolveId(feedMetadata.getCategory().getId()));
            if (domainCategory == null) {
                // throw exception
                throw new MetadataRepositoryException("Unable to find the category " + feedMetadata.getCategory().getSystemName());
            }
            accessController.checkPermission(domainCategory, CategoryAccessControl.CREATE_FEED);
            // ensure the user has rights to create feeds using the template by checking if it is visible.
            FeedManagerTemplate domainTemplate = templateProvider.findById(templateProvider.resolveId(feedMetadata.getTemplateId()));
            if (domainTemplate == null) {
                throw new MetadataRepositoryException("Unable to find the template " + feedMetadata.getTemplateId());
            }
        // accessController.checkPermission(domainTemplate, TemplateAccessControl.CREATE_FEED);
        } else {
            // perform explict entity access check here as we dont want to modify the NiFi flow unless user has access to edit the feed
            Feed.ID domainId = feedProvider.resolveId(feedMetadata.getId());
            Feed domainFeed = feedProvider.findById(domainId);
            if (domainFeed != null) {
                accessController.checkPermission(domainFeed, FeedAccessControl.EDIT_DETAILS);
            } else {
                throw new NotFoundException("Feed not found for id " + feedMetadata.getId());
            }
        }
    });
    FeedMetadata feedToSave = feedMetadata;
    if (!feedToSave.isNew()) {
        // create a new version if this is not a new feed
        feedToSave = saveDraftFeed(feedToSave);
    }
    // replace expressions with values
    if (feedToSave.getTable() != null) {
        feedToSave.getTable().updateMetadataFieldValues();
    }
    if (feedToSave.getProperties() == null) {
        feedToSave.setProperties(new ArrayList<NifiProperty>());
    }
    FeedMetadata.STATE state = FeedMetadata.STATE.NEW;
    try {
        state = FeedMetadata.STATE.valueOf(feedToSave.getState());
    } catch (Exception e) {
    // if the string isnt valid, disregard as it will end up disabling the feed.
    }
    boolean enabled = (FeedMetadata.STATE.NEW.equals(state) && feedToSave.isActive()) || FeedMetadata.STATE.ENABLED.equals(state);
    // store ref to the originalFeedProperties before resolving and merging with the template
    List<NifiProperty> originalFeedProperties = feedToSave.getProperties();
    // get all the properties for the metadata
    RegisteredTemplate registeredTemplate = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(feedToSave.getTemplateId()).templateName(feedToSave.getTemplateName()).isFeedEdit(true).includeSensitiveProperties(true).build());
    // copy the registered template properties it a new list so it doest get updated
    List<NifiProperty> templateProperties = registeredTemplate.getProperties().stream().map(nifiProperty -> new NifiProperty(nifiProperty)).collect(Collectors.toList());
    // update the template properties with the feedToSave properties
    NifiPropertyUtil.matchAndSetPropertyByProcessorName(templateProperties, feedToSave.getProperties(), NifiPropertyUtil.PropertyUpdateMode.UPDATE_ALL_PROPERTIES);
    registeredTemplate.setProperties(templateProperties);
    feedToSave.setProperties(registeredTemplate.getProperties());
    feedToSave.setRegisteredTemplate(registeredTemplate);
    // skip any properties that the user supplied which are not ${ values
    List<NifiProperty> propertiesToSkip = originalFeedProperties.stream().filter(property -> !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
    List<NifiProperty> templatePropertiesToSkip = registeredTemplate.getProperties().stream().filter(property -> property.isSelected() && !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
    if (templatePropertiesToSkip != null && !templatePropertiesToSkip.isEmpty()) {
        propertiesToSkip.addAll(templatePropertiesToSkip);
    }
    // resolve any ${metadata.} properties
    List<NifiProperty> resolvedProperties = propertyExpressionResolver.resolvePropertyExpressions(feedToSave, propertiesToSkip);
    // decrypt the metadata
    feedModelTransform.decryptSensitivePropertyValues(feedToSave);
    // flag to indicate to enable the feed later
    // if this is the first time for this feed and it is set to be enabled, mark it to be enabled after we commit to the JCR store
    boolean enableLater = false;
    if (enabled && feedToSave.isNew()) {
        enableLater = true;
        enabled = false;
        feedToSave.setState(FeedMetadata.STATE.DISABLED.name());
    }
    CreateFeedBuilder feedBuilder = CreateFeedBuilder.newFeed(nifiRestClient, nifiFlowCache, feedToSave, registeredTemplate.getNifiTemplateId(), propertyExpressionResolver, propertyDescriptorTransform, niFiObjectCache, templateConnectionUtil).enabled(enabled).setOriginalFeedProperties(originalFeedProperties).removeInactiveVersionedProcessGroup(removeInactiveNifiVersionedFeedFlows).autoAlign(nifiAutoFeedsAlignAfterSave).withNiFiTemplateCache(niFiTemplateCache);
    if (registeredTemplate.isReusableTemplate()) {
        feedBuilder.setReusableTemplate(true);
        feedToSave.setIsReusableFeed(true);
    } else {
        feedBuilder.inputProcessorType(feedToSave.getInputProcessorType()).feedSchedule(feedToSave.getSchedule()).properties(feedToSave.getProperties());
        if (registeredTemplate.usesReusableTemplate()) {
            for (ReusableTemplateConnectionInfo connection : registeredTemplate.getReusableTemplateConnections()) {
                feedBuilder.addInputOutputPort(new InputOutputPort(connection.getReusableTemplateInputPortName(), connection.getFeedOutputPortName()));
            }
        }
    }
    stopwatch.stop();
    log.debug("Time to prepare data for saving feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    stopwatch.start();
    NifiProcessGroup entity = feedBuilder.build();
    stopwatch.stop();
    log.debug("Time to save feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    feed = new NifiFeed(feedToSave, entity);
    // set the original feedProperties back to the feed
    feedToSave.setProperties(originalFeedProperties);
    // encrypt the metadata properties
    feedModelTransform.encryptSensitivePropertyValues(feedToSave);
    if (entity.isSuccess()) {
        feedToSave.setNifiProcessGroupId(entity.getProcessGroupEntity().getId());
        try {
            stopwatch.start();
            saveFeed(feedToSave);
            // tell NiFi if this is a streaming feed or not
            if (feedToSave.getRegisteredTemplate().isStream()) {
                streamingFeedJmsNotificationService.updateNiFiStatusJMSTopic(entity, feedToSave);
            }
            feed.setEnableAfterSave(enableLater);
            feed.setSuccess(true);
            stopwatch.stop();
            log.debug("Time to saveFeed in Kylo: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            stopwatch.start();
            feedBuilder.checkAndRemoveVersionedProcessGroup();
        } catch (Exception e) {
            feed.setSuccess(false);
            feed.addErrorMessage(e);
        }
    } else {
        feed.setSuccess(false);
    }
    if (!feed.isSuccess()) {
        if (!entity.isRolledBack()) {
            try {
                feedBuilder.rollback();
            } catch (FeedRollbackException rollbackException) {
                log.error("Error rolling back feed {}. {} ", feedToSave.getCategoryAndFeedName(), rollbackException.getMessage());
                feed.addErrorMessage("Error occurred in rolling back the Feed.");
            }
            entity.setRolledBack(true);
        }
    }
    return feed;
}
Also used : MetadataRepositoryException(com.thinkbiganalytics.metadata.modeshape.MetadataRepositoryException) Action(com.thinkbiganalytics.security.action.Action) RegisteredTemplateService(com.thinkbiganalytics.feedmgr.service.template.RegisteredTemplateService) Category(com.thinkbiganalytics.metadata.api.category.Category) DataSourceProvider(com.thinkbiganalytics.metadata.api.catalog.DataSourceProvider) ReusableTemplateConnectionInfo(com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo) Autowired(org.springframework.beans.factory.annotation.Autowired) StringUtils(org.apache.commons.lang3.StringUtils) FeedProvider(com.thinkbiganalytics.metadata.api.feed.FeedProvider) FeedAccessControl(com.thinkbiganalytics.metadata.api.feed.security.FeedAccessControl) Map(java.util.Map) FeedPropertyChangeEvent(com.thinkbiganalytics.metadata.api.event.feed.FeedPropertyChangeEvent) AccessController(com.thinkbiganalytics.security.AccessController) NifiFeed(com.thinkbiganalytics.feedmgr.rest.model.NifiFeed) CategoryAccessControl(com.thinkbiganalytics.metadata.api.category.security.CategoryAccessControl) FeedServicesAccessControl(com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl) SecurityContextHolder(org.springframework.security.core.context.SecurityContextHolder) FeedManagerTemplateProvider(com.thinkbiganalytics.metadata.api.template.FeedManagerTemplateProvider) FeedManagerTemplateService(com.thinkbiganalytics.feedmgr.service.template.FeedManagerTemplateService) MetadataEventListener(com.thinkbiganalytics.metadata.api.event.MetadataEventListener) Obligation(com.thinkbiganalytics.metadata.rest.model.sla.Obligation) NifiProperty(com.thinkbiganalytics.nifi.rest.model.NifiProperty) LabelValue(com.thinkbiganalytics.rest.model.LabelValue) ActionGroup(com.thinkbiganalytics.security.rest.model.ActionGroup) PageRequest(org.springframework.data.domain.PageRequest) Set(java.util.Set) Page(org.springframework.data.domain.Page) EntityVersionDifference(com.thinkbiganalytics.feedmgr.rest.model.EntityVersionDifference) MetadataEventService(com.thinkbiganalytics.metadata.api.event.MetadataEventService) Serializable(java.io.Serializable) DraftEntityVersion(com.thinkbiganalytics.feedmgr.rest.model.DraftEntityVersion) CategoryProvider(com.thinkbiganalytics.metadata.api.category.CategoryProvider) SecurityModelTransform(com.thinkbiganalytics.security.rest.controller.SecurityModelTransform) DataSourceNotFoundException(com.thinkbiganalytics.metadata.api.catalog.DataSourceNotFoundException) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) MetadataRepositoryException(com.thinkbiganalytics.metadata.modeshape.MetadataRepositoryException) ServiceLevelAgreementBuilder(com.thinkbiganalytics.metadata.sla.spi.ServiceLevelAgreementBuilder) RegisteredTemplateRequest(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplateRequest) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) NiFiObjectCache(com.thinkbiganalytics.nifi.rest.NiFiObjectCache) DerivedDatasourceFactory(com.thinkbiganalytics.feedmgr.service.feed.datasource.DerivedDatasourceFactory) MetadataChange(com.thinkbiganalytics.metadata.api.event.MetadataChange) RegisteredTemplate(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate) ObligationGroup(com.thinkbiganalytics.metadata.sla.api.ObligationGroup) Nullable(javax.annotation.Nullable) Datasource(com.thinkbiganalytics.metadata.api.datasource.Datasource) Properties(java.util.Properties) SecurityService(com.thinkbiganalytics.feedmgr.service.security.SecurityService) FeedProperties(com.thinkbiganalytics.metadata.api.feed.FeedProperties) FeedManagerTemplate(com.thinkbiganalytics.metadata.api.template.FeedManagerTemplate) HadoopSecurityGroup(com.thinkbiganalytics.metadata.api.security.HadoopSecurityGroup) ImmutableAllowableAction(com.thinkbiganalytics.security.role.ImmutableAllowableAction) TemplateConnectionUtil(com.thinkbiganalytics.feedmgr.nifi.TemplateConnectionUtil) HadoopAuthorizationService(com.thinkbiganalytics.datalake.authorization.service.HadoopAuthorizationService) ListUtils(org.apache.commons.collections.ListUtils) LoggerFactory(org.slf4j.LoggerFactory) FeedChange(com.thinkbiganalytics.metadata.api.event.feed.FeedChange) NiFiPropertyDescriptorTransform(com.thinkbiganalytics.nifi.rest.model.NiFiPropertyDescriptorTransform) Precondition(com.thinkbiganalytics.policy.precondition.Precondition) PreDestroy(javax.annotation.PreDestroy) DataSetProvider(com.thinkbiganalytics.metadata.api.catalog.DataSetProvider) NifiPropertyUtil(com.thinkbiganalytics.nifi.rest.support.NifiPropertyUtil) Pageable(org.springframework.data.domain.Pageable) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) CatalogModelTransform(com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform) FeedVersions(com.thinkbiganalytics.feedmgr.rest.model.FeedVersions) DeployResponseEntityVersion(com.thinkbiganalytics.feedmgr.rest.model.DeployResponseEntityVersion) FeedDestination(com.thinkbiganalytics.metadata.api.feed.FeedDestination) OpsManagerFeedProvider(com.thinkbiganalytics.metadata.api.feed.OpsManagerFeedProvider) ImmutableAllowedActions(com.thinkbiganalytics.security.role.ImmutableAllowedActions) UserField(com.thinkbiganalytics.feedmgr.rest.model.UserField) ImmutableMap(com.google.common.collect.ImmutableMap) DataSetNotFoundException(com.thinkbiganalytics.metadata.api.catalog.DataSetNotFoundException) Collection(java.util.Collection) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) NotFoundException(javax.ws.rs.NotFoundException) EntityVersion(com.thinkbiganalytics.feedmgr.rest.model.EntityVersion) FeedSummary(com.thinkbiganalytics.feedmgr.rest.model.FeedSummary) InputOutputPort(com.thinkbiganalytics.nifi.feedmgr.InputOutputPort) List(java.util.List) Principal(java.security.Principal) DataSource(com.thinkbiganalytics.metadata.api.catalog.DataSource) PostConstruct(javax.annotation.PostConstruct) Optional(java.util.Optional) NifiProcessGroup(com.thinkbiganalytics.nifi.rest.model.NifiProcessGroup) DerivedDatasource(com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource) PreconditionRule(com.thinkbiganalytics.policy.rest.model.PreconditionRule) DataAccessException(org.springframework.dao.DataAccessException) Stopwatch(com.google.common.base.Stopwatch) Feed(com.thinkbiganalytics.metadata.api.feed.Feed) HashMap(java.util.HashMap) AllowedActions(com.thinkbiganalytics.security.action.AllowedActions) UserProperty(com.thinkbiganalytics.feedmgr.rest.model.UserProperty) HashSet(java.util.HashSet) Inject(javax.inject.Inject) UIFeed(com.thinkbiganalytics.feedmgr.rest.model.UIFeed) PropertyExpressionResolver(com.thinkbiganalytics.feedmgr.nifi.PropertyExpressionResolver) ServiceLevelAgreementService(com.thinkbiganalytics.feedmgr.sla.ServiceLevelAgreementService) FeedChangeEvent(com.thinkbiganalytics.metadata.api.event.feed.FeedChangeEvent) Qualifier(org.springframework.beans.factory.annotation.Qualifier) FeedRollbackException(com.thinkbiganalytics.nifi.feedmgr.FeedRollbackException) FeedSource(com.thinkbiganalytics.metadata.api.feed.FeedSource) Nonnull(javax.annotation.Nonnull) FeedNotFoundException(com.thinkbiganalytics.metadata.api.feed.FeedNotFoundException) Logger(org.slf4j.Logger) FeedNameUtil(com.thinkbiganalytics.support.FeedNameUtil) CreateFeedBuilder(com.thinkbiganalytics.feedmgr.nifi.CreateFeedBuilder) FeedHistoryDataReindexingService(com.thinkbiganalytics.feedmgr.service.feed.reindexing.FeedHistoryDataReindexingService) DateTime(org.joda.time.DateTime) UserFieldDescriptor(com.thinkbiganalytics.metadata.api.extension.UserFieldDescriptor) FieldRuleProperty(com.thinkbiganalytics.policy.rest.model.FieldRuleProperty) TimeUnit(java.util.concurrent.TimeUnit) NifiFlowCache(com.thinkbiganalytics.feedmgr.nifi.cache.NifiFlowCache) AbstractMap(java.util.AbstractMap) UserPropertyTransform(com.thinkbiganalytics.feedmgr.service.UserPropertyTransform) NiFiTemplateCache(com.thinkbiganalytics.feedmgr.service.template.NiFiTemplateCache) DependentFeedPrecondition(com.thinkbiganalytics.policy.precondition.DependentFeedPrecondition) PreconditionPolicyTransformer(com.thinkbiganalytics.policy.precondition.transform.PreconditionPolicyTransformer) Comparator(java.util.Comparator) Collections(java.util.Collections) LegacyNifiRestClient(com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient) Category(com.thinkbiganalytics.metadata.api.category.Category) FeedRollbackException(com.thinkbiganalytics.nifi.feedmgr.FeedRollbackException) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) ServiceLevelAgreementBuilder(com.thinkbiganalytics.metadata.sla.spi.ServiceLevelAgreementBuilder) CreateFeedBuilder(com.thinkbiganalytics.feedmgr.nifi.CreateFeedBuilder) Stopwatch(com.google.common.base.Stopwatch) InputOutputPort(com.thinkbiganalytics.nifi.feedmgr.InputOutputPort) DataSourceNotFoundException(com.thinkbiganalytics.metadata.api.catalog.DataSourceNotFoundException) DataSetNotFoundException(com.thinkbiganalytics.metadata.api.catalog.DataSetNotFoundException) NotFoundException(javax.ws.rs.NotFoundException) FeedNotFoundException(com.thinkbiganalytics.metadata.api.feed.FeedNotFoundException) CreateFeedBuilder(com.thinkbiganalytics.feedmgr.nifi.CreateFeedBuilder) DataSourceNotFoundException(com.thinkbiganalytics.metadata.api.catalog.DataSourceNotFoundException) MetadataRepositoryException(com.thinkbiganalytics.metadata.modeshape.MetadataRepositoryException) DataSetNotFoundException(com.thinkbiganalytics.metadata.api.catalog.DataSetNotFoundException) NotFoundException(javax.ws.rs.NotFoundException) DataAccessException(org.springframework.dao.DataAccessException) FeedRollbackException(com.thinkbiganalytics.nifi.feedmgr.FeedRollbackException) FeedNotFoundException(com.thinkbiganalytics.metadata.api.feed.FeedNotFoundException) NifiProperty(com.thinkbiganalytics.nifi.rest.model.NifiProperty) RegisteredTemplate(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate) NifiProcessGroup(com.thinkbiganalytics.nifi.rest.model.NifiProcessGroup) FeedManagerTemplate(com.thinkbiganalytics.metadata.api.template.FeedManagerTemplate) ReusableTemplateConnectionInfo(com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo) NifiFeed(com.thinkbiganalytics.feedmgr.rest.model.NifiFeed) NifiFeed(com.thinkbiganalytics.feedmgr.rest.model.NifiFeed) Feed(com.thinkbiganalytics.metadata.api.feed.Feed) UIFeed(com.thinkbiganalytics.feedmgr.rest.model.UIFeed)

Aggregations

ReusableTemplateConnectionInfo (com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo)11 List (java.util.List)10 LegacyNifiRestClient (com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient)9 AccessController (com.thinkbiganalytics.security.AccessController)9 ArrayList (java.util.ArrayList)9 HashSet (java.util.HashSet)9 Map (java.util.Map)9 Set (java.util.Set)9 Collectors (java.util.stream.Collectors)9 Inject (javax.inject.Inject)9 StringUtils (org.apache.commons.lang3.StringUtils)9 TemplateConnectionUtil (com.thinkbiganalytics.feedmgr.nifi.TemplateConnectionUtil)8 Optional (java.util.Optional)8 RegisteredTemplate (com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate)7 RegisteredTemplateRequest (com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplateRequest)7 FeedServicesAccessControl (com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl)7 NifiProperty (com.thinkbiganalytics.nifi.rest.model.NifiProperty)7 Collections (java.util.Collections)7 HashMap (java.util.HashMap)7 TemplateDTO (org.apache.nifi.web.api.dto.TemplateDTO)7