Examples with Feed com.thinkbiganalytics.metadata.api.feed.Feed used on opensource projects

Example 51 with Feed

use of com.thinkbiganalytics.metadata.api.feed.Feed in project kylo by Teradata.

the class DefaultFeedExporter method exportFeed.

/**
 * Export a feed as a zip file
 *
 * @param feedId the id {@link Feed#getId()} of the feed to export
 * @return object containing the zip file with data about the feed.
 */
@Override
public ExportFeed exportFeed(String feedId) throws IOException {
    this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.EXPORT_FEEDS);
    this.metadataService.checkFeedPermission(feedId, FeedAccessControl.EXPORT);
    // Prepare feed metadata
    final FeedMetadata feed = metadataService.getFeedById(feedId);
    if (feed == null) {
        // feed will not be found when user is allowed to export feeds but has no entity access to feed with feed id
        throw new NotFoundException("Feed not found for id " + feedId);
    }
    final List<Datasource> userDatasources = Optional.ofNullable(feed.getDataTransformation()).map(FeedDataTransformation::getDatasourceIds).map(datasourceIds -> metadataAccess.read(() -> datasourceIds.stream().map(datasourceProvider::resolve).map(datasourceProvider::getDatasource).map(domain -> datasourceTransform.toDatasource(domain, DatasourceModelTransform.Level.FULL)).map(datasource -> {
        // Clear sensitive fields
        datasource.getDestinationForFeeds().clear();
        datasource.getSourceForFeeds().clear();
        return datasource;
    }).collect(Collectors.toList()))).orElse(null);
    if (userDatasources != null && !userDatasources.isEmpty()) {
        this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
        feed.setUserDatasources(userDatasources);
    }
    // Add feed json to template zip file
    final ExportTemplate exportTemplate = templateExporter.exportTemplateForFeedExport(feed.getTemplateId());
    final String feedJson = ObjectMapperSerializer.serialize(feed);
    final byte[] zipFile = ZipFileUtil.addToZip(exportTemplate.getFile(), feedJson, ImportFeed.FEED_JSON_FILE);
    return new ExportFeed(feed.getSystemFeedName() + ".feed.zip", zipFile);
}

Example 52 with Feed

use of com.thinkbiganalytics.metadata.api.feed.Feed in project kylo by Teradata.

the class DefaultSecurityService method accessFeed.

private Optional<Feed> accessFeed(String id) {
    this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_FEEDS);
    Feed.ID feedId = feedProvider.resolveFeed(id);
    Feed feed = feedProvider.getFeed(feedId);
    return Optional.ofNullable(feed);
}

Example 53 with Feed

use of com.thinkbiganalytics.metadata.api.feed.Feed in project kylo by Teradata.

the class FeedLineageBuilder method build.

private com.thinkbiganalytics.metadata.rest.model.feed.Feed build(Feed domainFeed) {
    com.thinkbiganalytics.metadata.rest.model.feed.Feed feed = restFeeds.containsKey(domainFeed.getId().toString()) ? restFeeds.get(domainFeed.getId().toString()) : model.domainToFeed(domainFeed);
    restFeeds.put(feed.getId(), feed);
    @SuppressWarnings("unchecked") List<? extends com.thinkbiganalytics.metadata.api.feed.FeedSource> sources = domainFeed.getSources();
    Set<FeedSource> feedSources = new HashSet<FeedSource>();
    if (sources != null) {
        sources.stream().forEach(feedSource -> {
            FeedSource src = new FeedSource();
            Datasource ds = buildDatasource(feedSource.getDatasource());
            src.setDatasource(ds);
            feedSources.add(src);
        });
    }
    feed.setSources(feedSources);
    Set<FeedDestination> feedDestinations = new HashSet<FeedDestination>();
    List<? extends com.thinkbiganalytics.metadata.api.feed.FeedDestination> destinations = domainFeed.getDestinations();
    if (destinations != null) {
        destinations.stream().forEach(feedDestination -> {
            FeedDestination dest = new FeedDestination();
            Datasource ds = buildDatasource(feedDestination.getDatasource());
            dest.setDatasource(ds);
            feedDestinations.add(dest);
        });
    }
    feed.setDestinations(feedDestinations);
    if (domainFeed.getDependentFeeds() != null) {
        List<Feed> depFeeds = domainFeed.getDependentFeeds();
        depFeeds.stream().forEach(depFeed -> {
            com.thinkbiganalytics.metadata.rest.model.feed.Feed restFeed = restFeeds.get(depFeed.getId().toString());
            if (restFeed == null) {
                com.thinkbiganalytics.metadata.rest.model.feed.Feed depRestFeed = model.domainToFeed(depFeed);
                restFeeds.put(depRestFeed.getId(), depRestFeed);
                feed.getDependentFeeds().add(depRestFeed);
                build(depFeed);
            } else {
                feed.getDependentFeeds().add(restFeed);
            }
        });
    }
    if (domainFeed.getUsedByFeeds() != null) {
        List<Feed> usedByFeeds = domainFeed.getUsedByFeeds();
        usedByFeeds.stream().forEach(usedByFeed -> {
            com.thinkbiganalytics.metadata.rest.model.feed.Feed restFeed = restFeeds.get(usedByFeed.getId().toString());
            if (restFeed == null) {
                com.thinkbiganalytics.metadata.rest.model.feed.Feed usedByRestFeed = model.domainToFeed(usedByFeed);
                restFeeds.put(usedByRestFeed.getId(), usedByRestFeed);
                feed.getUsedByFeeds().add(usedByRestFeed);
                build(usedByFeed);
            } else {
                feed.getUsedByFeeds().add(restFeed);
            }
        });
    }
    return feed;
}

Example 54 with Feed

use of com.thinkbiganalytics.metadata.api.feed.Feed in project kylo by Teradata.

the class FeedLineageBuilder method buildDatasource.

private Datasource buildDatasource(com.thinkbiganalytics.metadata.api.datasource.Datasource domainDatasource) {
    Datasource ds = restDatasources.get(domainDatasource.getId().toString());
    if (ds == null) {
        // build the data source
        ds = datasourceTransform.toDatasource(domainDatasource, DatasourceModelTransform.Level.BASIC);
        restDatasources.put(ds.getId(), ds);
        // populate the Feed relationships
        if (domainDatasource.getFeedSources() != null) {
            List<com.thinkbiganalytics.metadata.rest.model.feed.Feed> feedList = new ArrayList<>();
            for (com.thinkbiganalytics.metadata.api.feed.FeedSource domainSrc : domainDatasource.getFeedSources()) {
                com.thinkbiganalytics.metadata.rest.model.feed.Feed feed = build(domainSrc.getFeed());
                feedList.add(feed);
            }
            ds.getSourceForFeeds().addAll(feedList);
        }
        if (domainDatasource.getFeedDestinations() != null) {
            List<com.thinkbiganalytics.metadata.rest.model.feed.Feed> feedList = new ArrayList<>();
            for (com.thinkbiganalytics.metadata.api.feed.FeedDestination domainDest : domainDatasource.getFeedDestinations()) {
                com.thinkbiganalytics.metadata.rest.model.feed.Feed feed = build(domainDest.getFeed());
                feedList.add(feed);
            }
            ds.getDestinationForFeeds().addAll(feedList);
        }
    }
    return ds;
}

Example 55 with Feed

use of com.thinkbiganalytics.metadata.api.feed.Feed in project kylo by Teradata.

the class FeedWaterMarkService method getWaterMark.

public Optional<String> getWaterMark(String feedId, String waterMarkName) {
    return this.metadata.read(() -> {
        this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_FEEDS);
        com.thinkbiganalytics.metadata.api.feed.Feed.ID id = feedProvider.resolveFeed(feedId);
        com.thinkbiganalytics.metadata.api.feed.Feed feed = feedProvider.getFeed(id);
        if (feed != null) {
            return feed.getWaterMarkValue(waterMarkName);
        } else {
            throw new FeedNotFoundException(id);
        }
    });
}