use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.
the class AccessControlConfigurator method ensureCatalogAccessControl.
/**
* Ensures that the entity-level access control is setup up for the entities introduced by the connector architecture.
*/
public void ensureCatalogAccessControl() {
List<Connector> connectors = connectorProvider.findAll(true);
List<SecurityRole> conntorRoles = this.roleProvider.getEntityRoles(SecurityRole.CONNECTOR);
Optional<AllowedActions> connectorActions = this.actionsProvider.getAvailableActions(AllowedActions.CONNECTOR);
connectors.stream().forEach(conn -> {
Principal owner = conn.getOwner() != null ? conn.getOwner() : JcrMetadataAccess.getActiveUser();
connectorActions.ifPresent(actions -> ((JcrConnector) conn).enableAccessControl((JcrAllowedActions) actions, owner, conntorRoles));
});
List<DataSource> dataSources = dataSourceProvider.findAll();
List<SecurityRole> dataSourceRoles = this.roleProvider.getEntityRoles(SecurityRole.DATASOURCE);
Optional<AllowedActions> dataSourceActions = this.actionsProvider.getAvailableActions(AllowedActions.DATASOURCE);
dataSources.stream().map(JcrDataSource.class::cast).forEach(dataSource -> {
Principal owner = dataSource.getOwner() != null ? dataSource.getOwner() : JcrMetadataAccess.getActiveUser();
dataSourceActions.ifPresent(actions -> dataSource.enableAccessControl((JcrAllowedActions) actions, owner, dataSourceRoles));
});
}
use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.
the class DefaultFeedManagerFeedService method assignFeedDatasources.
/**
* Assign the feed sources/destinations
*
* @param feed the feed rest model
* @param domainFeed the domain feed
*/
private void assignFeedDatasources(FeedMetadata feed, Feed domainFeed) {
final Feed.ID domainFeedId = domainFeed.getId();
Set<com.thinkbiganalytics.metadata.api.catalog.DataSet.ID> sourceDataSets = new HashSet<>();
Set<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> sourceDatasources = new HashSet<>();
Set<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> destinationDatasources = new HashSet<>();
String uniqueName = FeedNameUtil.fullName(feed.getCategory().getSystemName(), feed.getSystemFeedName());
RegisteredTemplate template = feed.getRegisteredTemplate();
if (template == null) {
// fetch it for checks
template = templateRestProvider.getRegisteredTemplate(feed.getTemplateId());
}
// Collect the IDs of the legacy datasources the feed had referenced
Set<Datasource.ID> previousSourceIds = domainFeed.getSources().stream().filter(fs -> fs.getDatasource().isPresent()).map(fs -> fs.getDatasource().get().getId()).collect(Collectors.toSet());
Set<Datasource.ID> previousDestIds = domainFeed.getDestinations().stream().filter(// Currently will always be true as there are no destination data sets yet.
fs -> fs.getDatasource().isPresent()).map(fs -> fs.getDatasource().get().getId()).collect(Collectors.toSet());
boolean isSampleDataSet = isTreatSourceDataSetsAsSample(feed, template);
// find Definition registration
derivedDatasourceFactory.populateDatasources(feed, template, sourceDatasources, destinationDatasources);
// Replace the older legacy datasource references with the new ones.
previousSourceIds.stream().filter(id -> !sourceDatasources.contains(id)).forEach(id -> feedProvider.removeFeedSource(domainFeedId, id));
sourceDatasources.stream().forEach(sourceId -> feedProvider.ensureFeedSource(domainFeedId, sourceId));
previousDestIds.stream().filter(id -> !destinationDatasources.contains(id)).forEach(id -> feedProvider.removeFeedDestination(domainFeedId, id));
destinationDatasources.stream().forEach(sourceId -> feedProvider.ensureFeedDestination(domainFeedId, sourceId));
// Update data sets
if (feed.getSourceDataSets() != null) {
// Collect the IDs of source data sets the feed had referenced
Set<com.thinkbiganalytics.metadata.api.catalog.DataSet.ID> currentDataSetIds = domainFeed.getSources().stream().map(FeedSource::getDataSet).filter(Optional::isPresent).map(Optional::get).map(com.thinkbiganalytics.metadata.api.catalog.DataSet::getId).collect(Collectors.toSet());
Set<com.thinkbiganalytics.metadata.api.catalog.DataSet.ID> newDataSetIds = new HashSet<>();
feed.getSourceDataSets().forEach(dataSet -> {
com.thinkbiganalytics.metadata.api.catalog.DataSet addedDataSet;
if (dataSet.getId() == null) {
DataSource.ID dataSourceId = dataSourceProvider.resolveId(dataSet.getDataSource().getId());
dataSourceProvider.find(dataSourceId).orElseThrow(() -> new DataSourceNotFoundException(dataSourceId));
addedDataSet = catalogModelTransform.buildDataSet(dataSet, dataSetProvider.build(dataSourceId));
} else {
com.thinkbiganalytics.metadata.api.catalog.DataSet.ID dataSetId = dataSetProvider.resolveId(dataSet.getId());
addedDataSet = dataSetProvider.find(dataSetId).orElseThrow(() -> new DataSetNotFoundException(dataSetId));
}
newDataSetIds.add(addedDataSet.getId());
catalogModelTransform.updateDataSet(dataSet, addedDataSet);
feedProvider.ensureFeedSource(domainFeedId, addedDataSet.getId(), isSampleDataSet);
});
// Remove any data set sources no longer referenced in the updated feed.
currentDataSetIds.stream().filter(id -> !newDataSetIds.contains(id)).forEach(id -> feedProvider.removeFeedSource(domainFeedId, id));
}
}
use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.
the class DefaultFeedManagerFeedService method saveDeployedFeed.
private void saveDeployedFeed(final FeedMetadata feed, com.thinkbiganalytics.metadata.api.versioning.EntityVersion<Feed.ID, Feed> version) {
Stopwatch stopwatch = Stopwatch.createStarted();
List<? extends HadoopSecurityGroup> previousSavedSecurityGroups = null;
// Store the old security groups before saving because we need to compare afterward
Feed previousStateBeforeSaving = feedProvider.findById(feedProvider.resolveId(feed.getId()));
Map<String, String> userProperties = previousStateBeforeSaving.getUserProperties();
previousSavedSecurityGroups = previousStateBeforeSaving.getSecurityGroups();
// if this is the first time saving this feed create a new one
Feed domainFeed = version.getEntity().get();
Feed.ID domainId = domainFeed.getId();
if (domainFeed.getState() == null) {
domainFeed.setState(Feed.State.ENABLED);
} else if (Feed.State.ENABLED.name().contentEquals(feed.getState())) {
domainFeed.setState(Feed.State.ENABLED);
} else {
domainFeed.setState(Feed.State.DISABLED);
}
stopwatch.stop();
log.debug("Time to transform the feed to a domain object for saving: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
final String feedName = FeedNameUtil.fullName(domainFeed.getCategory().getSystemName(), domainFeed.getName());
stopwatch.start();
boolean isStream = feed.getRegisteredTemplate() != null ? feed.getRegisteredTemplate().isStream() : false;
Long timeBetweenBatchJobs = feed.getRegisteredTemplate() != null ? feed.getRegisteredTemplate().getTimeBetweenStartingBatchJobs() : 0L;
// sync the feed information to ops manager
metadataAccess.commit(() -> opsManagerFeedProvider.save(opsManagerFeedProvider.resolveId(domainId.toString()), feedName, isStream, timeBetweenBatchJobs));
stopwatch.stop();
log.debug("Time to sync feed data with Operations Manager: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
// Update hadoop security group polices if the groups changed
if (!ListUtils.isEqualList(previousSavedSecurityGroups, domainFeed.getSecurityGroups())) {
stopwatch.start();
List<? extends HadoopSecurityGroup> securityGroups = domainFeed.getSecurityGroups();
List<String> groupsAsCommaList = securityGroups.stream().map(group -> group.getName()).collect(Collectors.toList());
hadoopAuthorizationService.updateSecurityGroupsForAllPolicies(feed.getSystemCategoryName(), feed.getSystemFeedName(), groupsAsCommaList, domainFeed.getProperties());
stopwatch.stop();
log.debug("Time to update hadoop security groups: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
}
// Update Hive metastore
if (hiveTargetSyncColumnDescriptions) {
stopwatch.start();
final boolean hasHiveDestination = domainFeed.getDestinations().stream().map(FeedDestination::getDatasource).filter(DerivedDatasource.class::isInstance).map(DerivedDatasource.class::cast).anyMatch(datasource -> "HiveDatasource".equals(datasource.getDatasourceType()));
if (hasHiveDestination) {
try {
feedHiveTableService.updateColumnDescriptions(feed);
} catch (final DataAccessException e) {
log.warn("Failed to update column descriptions for feed: {}", feed.getCategoryAndFeedDisplayName(), e);
}
}
stopwatch.stop();
log.debug("Time to update hive metastore: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
}
// Update Kylo metastore
stopwatch.start();
feedProvider.setDeployed(domainId, version.getId());
stopwatch.stop();
log.debug("Time to call feedProvider.update: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
}
Aggregations