Examples with FeedDestination com.thinkbiganalytics.metadata.api.feed.FeedDestination used on opensource projects

Example 6 with FeedDestination

use of com.thinkbiganalytics.metadata.api.feed.FeedDestination in project kylo by Teradata.

the class BaseFeed method addDestination.

public FeedDestination addDestination(Datasource ds) {
    FeedDestination dest = new Destination(ds);
    this.destinations.add(dest);
    return dest;
}

Example 7 with FeedDestination

use of com.thinkbiganalytics.metadata.api.feed.FeedDestination in project kylo by Teradata.

the class DefaultFeedManagerFeedService method saveDeployedFeed.

private void saveDeployedFeed(final FeedMetadata feed, com.thinkbiganalytics.metadata.api.versioning.EntityVersion<Feed.ID, Feed> version) {
    Stopwatch stopwatch = Stopwatch.createStarted();
    List<? extends HadoopSecurityGroup> previousSavedSecurityGroups = null;
    // Store the old security groups before saving because we need to compare afterward
    Feed previousStateBeforeSaving = feedProvider.findById(feedProvider.resolveId(feed.getId()));
    Map<String, String> userProperties = previousStateBeforeSaving.getUserProperties();
    previousSavedSecurityGroups = previousStateBeforeSaving.getSecurityGroups();
    // if this is the first time saving this feed create a new one
    Feed domainFeed = version.getEntity().get();
    Feed.ID domainId = domainFeed.getId();
    if (domainFeed.getState() == null) {
        domainFeed.setState(Feed.State.ENABLED);
    } else if (Feed.State.ENABLED.name().contentEquals(feed.getState())) {
        domainFeed.setState(Feed.State.ENABLED);
    } else {
        domainFeed.setState(Feed.State.DISABLED);
    }
    stopwatch.stop();
    log.debug("Time to transform the feed to a domain object for saving: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    final String feedName = FeedNameUtil.fullName(domainFeed.getCategory().getSystemName(), domainFeed.getName());
    stopwatch.start();
    boolean isStream = feed.getRegisteredTemplate() != null ? feed.getRegisteredTemplate().isStream() : false;
    Long timeBetweenBatchJobs = feed.getRegisteredTemplate() != null ? feed.getRegisteredTemplate().getTimeBetweenStartingBatchJobs() : 0L;
    // sync the feed information to ops manager
    metadataAccess.commit(() -> opsManagerFeedProvider.save(opsManagerFeedProvider.resolveId(domainId.toString()), feedName, isStream, timeBetweenBatchJobs));
    stopwatch.stop();
    log.debug("Time to sync feed data with Operations Manager: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    // Update hadoop security group polices if the groups changed
    if (!ListUtils.isEqualList(previousSavedSecurityGroups, domainFeed.getSecurityGroups())) {
        stopwatch.start();
        List<? extends HadoopSecurityGroup> securityGroups = domainFeed.getSecurityGroups();
        List<String> groupsAsCommaList = securityGroups.stream().map(group -> group.getName()).collect(Collectors.toList());
        hadoopAuthorizationService.updateSecurityGroupsForAllPolicies(feed.getSystemCategoryName(), feed.getSystemFeedName(), groupsAsCommaList, domainFeed.getProperties());
        stopwatch.stop();
        log.debug("Time to update hadoop security groups: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
    }
    // Update Hive metastore
    if (hiveTargetSyncColumnDescriptions) {
        stopwatch.start();
        final boolean hasHiveDestination = domainFeed.getDestinations().stream().map(FeedDestination::getDatasource).filter(DerivedDatasource.class::isInstance).map(DerivedDatasource.class::cast).anyMatch(datasource -> "HiveDatasource".equals(datasource.getDatasourceType()));
        if (hasHiveDestination) {
            try {
                feedHiveTableService.updateColumnDescriptions(feed);
            } catch (final DataAccessException e) {
                log.warn("Failed to update column descriptions for feed: {}", feed.getCategoryAndFeedDisplayName(), e);
            }
        }
        stopwatch.stop();
        log.debug("Time to update hive metastore: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
    }
    // Update Kylo metastore
    stopwatch.start();
    feedProvider.setDeployed(domainId, version.getId());
    stopwatch.stop();
    log.debug("Time to call feedProvider.update: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
}