Examples with FeedMetadata com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata used on opensource projects

Example 51 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class DefaultFeedManagerFeedService method assignFeedDependencies.

/**
 * Looks for the Feed Preconditions and assigns the Feed Dependencies
 */
private void assignFeedDependencies(FeedMetadata feed, Feed domainFeed) {
    final Feed.ID domainFeedId = domainFeed.getId();
    List<PreconditionRule> preconditions = feed.getSchedule().getPreconditions();
    if (preconditions != null) {
        PreconditionPolicyTransformer transformer = new PreconditionPolicyTransformer(preconditions);
        transformer.applyFeedNameToCurrentFeedProperties(feed.getCategory().getSystemName(), feed.getSystemFeedName());
        List<com.thinkbiganalytics.metadata.rest.model.sla.ObligationGroup> transformedPreconditions = transformer.getPreconditionObligationGroups();
        ServiceLevelAgreementBuilder preconditionBuilder = feedProvider.buildPrecondition(domainFeed.getId()).name("Precondition for feed " + feed.getCategoryAndFeedName() + "  (" + domainFeed.getId() + ")");
        for (com.thinkbiganalytics.metadata.rest.model.sla.ObligationGroup precondition : transformedPreconditions) {
            for (Obligation group : precondition.getObligations()) {
                preconditionBuilder.obligationGroupBuilder(ObligationGroup.Condition.valueOf(precondition.getCondition())).obligationBuilder().metric(group.getMetrics()).build();
            }
        }
        preconditionBuilder.build();
        // add in the lineage dependency relationships
        // will the feed exist in the jcr store here if it is new??
        // store the existing list of dependent feeds to track and delete those that dont match
        Set<Feed.ID> oldDependentFeedIds = new HashSet<Feed.ID>();
        Set<Feed.ID> newDependentFeedIds = new HashSet<Feed.ID>();
        List<Feed> dependentFeeds = domainFeed.getDependentFeeds();
        if (dependentFeeds != null && !dependentFeeds.isEmpty()) {
            dependentFeeds.stream().forEach(dependentFeed -> {
                oldDependentFeedIds.add(dependentFeed.getId());
            });
        }
        // find those preconditions that are marked as dependent feed types
        List<Precondition> preconditionPolicies = transformer.getPreconditionPolicies();
        preconditionPolicies.stream().filter(precondition -> precondition instanceof DependentFeedPrecondition).forEach(dependentFeedPrecondition -> {
            DependentFeedPrecondition feedPrecondition = (DependentFeedPrecondition) dependentFeedPrecondition;
            List<String> dependentFeedNames = feedPrecondition.getDependentFeedNames();
            if (dependentFeedNames != null && !dependentFeedNames.isEmpty()) {
                // find the feed
                for (String dependentFeedName : dependentFeedNames) {
                    Feed dependentFeed = feedProvider.findBySystemName(dependentFeedName);
                    if (dependentFeed != null) {
                        Feed.ID newDependentFeedId = dependentFeed.getId();
                        newDependentFeedIds.add(newDependentFeedId);
                        // add and persist it if it doesnt already exist
                        if (!oldDependentFeedIds.contains(newDependentFeedId)) {
                            feedProvider.addDependent(domainFeedId, dependentFeed.getId());
                        }
                    }
                }
            }
        });
        // delete any of those dependent feed ids from the oldDependentFeeds that are not part of the newDependentFeedIds
        oldDependentFeedIds.stream().filter(oldFeedId -> !newDependentFeedIds.contains(oldFeedId)).forEach(dependentFeedToDelete -> feedProvider.removeDependent(domainFeedId, dependentFeedToDelete));
    }
}

Example 52 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedModelTransform method deserializeFeedMetadata.

public FeedMetadata deserializeFeedMetadata(Feed domain, boolean clearSensitiveProperties) {
    String json = domain.getJson();
    FeedMetadata feedMetadata = ObjectMapperSerializer.deserialize(json, FeedMetadata.class);
    populate(feedMetadata, domain);
    if (clearSensitiveProperties) {
        clearSensitivePropertyValues(feedMetadata);
    }
    return feedMetadata;
}

Example 53 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class FeedModelTransform method stripMetadata.

/**
 * Clean out any excess or redundant data that should not be serialized and stored
 * with the feed domain entity as JSON.
 * @param source the source metadata
 * @return a new metadata instance without the excess data
 */
private FeedMetadata stripMetadata(FeedMetadata source) {
    FeedMetadata result = new FeedMetadata();
    result.setDataOwner(source.getDataOwner());
    result.setDataTransformation(source.getDataTransformation());
    result.setHadoopAuthorizationType(source.getHadoopAuthorizationType());
    result.setInputProcessorType(source.getInputProcessorType());
    result.setInputProcessorName(source.getInputProcessorName());
    result.setIsReusableFeed(source.isReusableFeed());
    result.setNifiProcessGroupId(source.getNifiProcessGroupId());
    result.setOptions(source.getOptions());
    result.setProperties(source.getProperties());
    result.setSchedule(source.getSchedule());
    result.setTable(source.getTable());
    result.setTableOption(source.getTableOption());
    return result;
}

Example 54 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class DerivedDatasourceFactory method ensureDatasource.

public Datasource.ID ensureDatasource(TemplateProcessorDatasourceDefinition definition, FeedMetadata feedMetadata, List<NifiProperty> allProperties) {
    return metadataAccess.commit(() -> {
        List<NifiProperty> propertiesToEvalulate = new ArrayList<NifiProperty>();
        // fetch the def
        DatasourceDefinition datasourceDefinition = datasourceDefinitionProvider.findByProcessorType(definition.getProcessorType());
        if (datasourceDefinition != null) {
            // find out if there are any saved properties on the Feed that match the datasourceDef
            List<NifiProperty> feedProperties = feedMetadata.getProperties().stream().filter(property -> matchesDefinition(definition, property) && datasourceDefinition.getDatasourcePropertyKeys().contains(property.getKey())).collect(Collectors.toList());
            // resolve any ${metadata.} properties
            List<NifiProperty> resolvedFeedProperties = propertyExpressionResolver.resolvePropertyExpressions(feedProperties, feedMetadata);
            List<NifiProperty> resolvedAllProperties = propertyExpressionResolver.resolvePropertyExpressions(allProperties, feedMetadata);
            // propetyHash
            propertiesToEvalulate.addAll(feedProperties);
            propertiesToEvalulate.addAll(allProperties);
            propertyExpressionResolver.resolveStaticProperties(propertiesToEvalulate);
            String identityString = datasourceDefinition.getIdentityString();
            String desc = datasourceDefinition.getDescription();
            String title = datasourceDefinition.getTitle();
            PropertyExpressionResolver.ResolvedVariables identityStringPropertyResolution = propertyExpressionResolver.resolveVariables(identityString, propertiesToEvalulate);
            identityString = identityStringPropertyResolution.getResolvedString();
            PropertyExpressionResolver.ResolvedVariables titlePropertyResolution = propertyExpressionResolver.resolveVariables(title, propertiesToEvalulate);
            title = titlePropertyResolution.getResolvedString();
            if (desc != null) {
                PropertyExpressionResolver.ResolvedVariables descriptionPropertyResolution = propertyExpressionResolver.resolveVariables(desc, propertiesToEvalulate);
                desc = descriptionPropertyResolution.getResolvedString();
            }
            // if the identityString still contains unresolved variables then make the title readable and replace the idstring with the feed.id
            if (propertyExpressionResolver.containsVariablesPatterns(identityString)) {
                title = propertyExpressionResolver.replaceAll(title, " {runtime variable} ");
                identityString = propertyExpressionResolver.replaceAll(identityString, feedMetadata.getId());
            }
            // if it is the Source ensure the feed matches this ds
            if (isCreateDatasource(datasourceDefinition, feedMetadata)) {
                Map<String, String> controllerServiceProperties = parseControllerServiceProperties(datasourceDefinition, feedProperties);
                Map<String, Object> properties = new HashMap<String, Object>(identityStringPropertyResolution.getResolvedVariables());
                properties.putAll(controllerServiceProperties);
                DerivedDatasource derivedDatasource = datasourceProvider.ensureDerivedDatasource(datasourceDefinition.getDatasourceType(), identityString, title, desc, properties);
                if (derivedDatasource != null) {
                    if ("HiveDatasource".equals(derivedDatasource.getDatasourceType()) && Optional.ofNullable(feedMetadata.getTable()).map(TableSetup::getTableSchema).map(TableSchema::getFields).isPresent()) {
                        derivedDatasource.setGenericProperties(Collections.singletonMap("columns", (Serializable) feedMetadata.getTable().getTableSchema().getFields()));
                    }
                    return derivedDatasource.getId();
                }
            }
            return null;
        } else {
            return null;
        }
    }, MetadataAccess.SERVICE);
}

Example 55 with FeedMetadata

use of com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata in project kylo by Teradata.

the class DefaultFeedExporter method exportFeed.

/**
 * Export a feed as a zip file
 *
 * @param feedId the id {@link Feed#getId()} of the feed to export
 * @return object containing the zip file with data about the feed.
 */
@Override
public ExportFeed exportFeed(String feedId) throws IOException {
    this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.EXPORT_FEEDS);
    this.metadataService.checkFeedPermission(feedId, FeedAccessControl.EXPORT);
    // Prepare feed metadata
    final FeedMetadata feed = metadataService.getFeedById(feedId);
    if (feed == null) {
        // feed will not be found when user is allowed to export feeds but has no entity access to feed with feed id
        throw new NotFoundException("Feed not found for id " + feedId);
    }
    final List<Datasource> userDatasources = Optional.ofNullable(feed.getDataTransformation()).map(FeedDataTransformation::getDatasourceIds).map(datasourceIds -> metadataAccess.read(() -> datasourceIds.stream().map(datasourceProvider::resolve).map(datasourceProvider::getDatasource).map(domain -> datasourceTransform.toDatasource(domain, DatasourceModelTransform.Level.FULL)).map(datasource -> {
        // Clear sensitive fields
        datasource.getDestinationForFeeds().clear();
        datasource.getSourceForFeeds().clear();
        return datasource;
    }).collect(Collectors.toList()))).orElse(null);
    if (userDatasources != null && !userDatasources.isEmpty()) {
        this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
        feed.setUserDatasources(userDatasources);
    }
    // Add feed json to template zip file
    final ExportTemplate exportTemplate = templateExporter.exportTemplateForFeedExport(feed.getTemplateId());
    final String feedJson = ObjectMapperSerializer.serialize(feed);
    final byte[] zipFile = ZipFileUtil.addToZip(exportTemplate.getFile(), feedJson, ImportFeed.FEED_JSON_FILE);
    return new ExportFeed(feed.getSystemFeedName() + ".feed.zip", zipFile);
}