use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.
the class DefaultDataSourcesUpgradeAction method upgradeTo.
@Override
public void upgradeTo(final KyloVersion targetVersion) {
log.info("Creating default catalog data sources: {}", targetVersion);
Optional<ConnectorPlugin> plugin = pluginManager.getPlugin("hive");
if (plugin.isPresent()) {
Optional<Connector> connector = connectorProvider.findByPlugin(plugin.get().getId());
if (connector.isPresent()) {
List<DataSource> hiveSources = dataSourceProvider.findByConnector(connector.get().getId());
// If at least one Hive data source exists then do nothing.
if (hiveSources.size() == 0) {
log.info("Creating default Hive data source");
DataSource ds = dataSourceProvider.create(connector.get().getId(), "Hive");
ds.setDescription("The default Hive data source");
DataSetSparkParameters params = ds.getSparkParameters();
params.setFormat("hive");
params.addOption("driver", this.driver);
params.addOption("url", this.url);
params.addOption("user", this.user);
params.addOption("password", this.password);
} else {
log.info("One or more Hive data sources already found: {}", hiveSources.stream().map(DataSource::toString).collect(Collectors.toList()));
}
} else {
log.warn("No Hive connector found - cannot create a default Hive data source");
}
} else {
log.warn("No Hive connector plugin found - cannot create a default Hive data source");
}
}
use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.
the class DefaultFeedManagerFeedService method saveFeed.
@Deprecated
private void saveFeed(final FeedMetadata feed) {
Feed.ID feedId = metadataAccess.commit(() -> {
Stopwatch stopwatch = Stopwatch.createStarted();
List<? extends HadoopSecurityGroup> previousSavedSecurityGroups = null;
// Store the old security groups before saving because we need to compare afterward
if (!feed.isNew()) {
Feed previousStateBeforeSaving = feedProvider.findById(feedProvider.resolveId(feed.getId()));
Map<String, String> userProperties = previousStateBeforeSaving.getUserProperties();
previousSavedSecurityGroups = previousStateBeforeSaving.getSecurityGroups();
}
// if this is the first time saving this feed create a new one
Feed domainFeed = feedModelTransform.feedToDomain(feed);
Feed.ID domainId = domainFeed.getId();
if (domainFeed.getState() == null) {
domainFeed.setState(Feed.State.ENABLED);
}
stopwatch.stop();
log.debug("Time to transform the feed to a domain object for saving: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
// initially save the feed
if (feed.isNew()) {
stopwatch.start();
domainFeed = feedProvider.update(domainFeed);
stopwatch.stop();
log.debug("Time to save the New feed: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
}
final String feedName = FeedNameUtil.fullName(domainFeed.getCategory().getSystemName(), domainFeed.getName());
// Build preconditions
stopwatch.start();
assignFeedDependencies(feed, domainFeed);
stopwatch.stop();
log.debug("Time to assignFeedDependencies: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
// Assign the datasources
stopwatch.start();
assignFeedDatasources(feed, domainFeed);
stopwatch.stop();
log.debug("Time to assignFeedDatasources: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
stopwatch.start();
boolean isStream = feed.getRegisteredTemplate() != null ? feed.getRegisteredTemplate().isStream() : false;
Long timeBetweenBatchJobs = feed.getRegisteredTemplate() != null ? feed.getRegisteredTemplate().getTimeBetweenStartingBatchJobs() : 0L;
// sync the feed information to ops manager
metadataAccess.commit(() -> opsManagerFeedProvider.save(opsManagerFeedProvider.resolveId(domainId.toString()), feedName, isStream, timeBetweenBatchJobs));
stopwatch.stop();
log.debug("Time to sync feed data with Operations Manager: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
// Update hadoop security group polices if the groups changed
if (!feed.isNew() && !ListUtils.isEqualList(previousSavedSecurityGroups, domainFeed.getSecurityGroups())) {
stopwatch.start();
List<? extends HadoopSecurityGroup> securityGroups = domainFeed.getSecurityGroups();
List<String> groupsAsCommaList = securityGroups.stream().map(group -> group.getName()).collect(Collectors.toList());
hadoopAuthorizationService.updateSecurityGroupsForAllPolicies(feed.getSystemCategoryName(), feed.getSystemFeedName(), groupsAsCommaList, domainFeed.getProperties());
stopwatch.stop();
log.debug("Time to update hadoop security groups: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
}
// Update Hive metastore
if (hiveTargetSyncColumnDescriptions) {
stopwatch.start();
final boolean hasHiveDestination = domainFeed.getDestinations().stream().map(FeedDestination::getDatasource).filter(DerivedDatasource.class::isInstance).map(DerivedDatasource.class::cast).anyMatch(datasource -> "HiveDatasource".equals(datasource.getDatasourceType()));
if (hasHiveDestination) {
try {
feedHiveTableService.updateColumnDescriptions(feed);
} catch (final DataAccessException e) {
log.warn("Failed to update column descriptions for feed: {}", feed.getCategoryAndFeedDisplayName(), e);
}
}
stopwatch.stop();
log.debug("Time to update hive metastore: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
}
// Update Kylo metastore
stopwatch.start();
domainFeed = feedProvider.update(domainFeed);
stopwatch.stop();
log.debug("Time to call feedProvider.update: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
stopwatch.reset();
return domainFeed.getId();
}, (e) -> {
if (feed.isNew() && StringUtils.isNotBlank(feed.getId())) {
// Rollback ops Manager insert if it is newly created
metadataAccess.commit(() -> {
opsManagerFeedProvider.delete(opsManagerFeedProvider.resolveId(feed.getId()));
});
}
});
if (feedId != null) {
// set deployed as a service account since it needs access to the versionable node
metadataAccess.commit(() -> {
// TODO TEMPORARY
com.thinkbiganalytics.metadata.api.versioning.EntityVersion<Feed.ID, Feed> version = feedProvider.createVersion(feedId, null, false);
feedProvider.setDeployed(feedId, version.getId());
// TODO TEMPORARY
}, MetadataAccess.SERVICE);
}
}
use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.
the class FeedImporter method validateUserDatasources.
/**
* Validates that user data sources can be imported with provided properties.
* Legacy UserDatasources will be remapped toe Catalog DataSources or DataSets
*
* @return {@code true} if the feed can be imported, or {@code false} otherwise
*/
private boolean validateUserDatasources() {
FeedMetadata metadata = importFeed.getFeedToImport();
final UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importFeed.getImportOptions().getUploadKey(), "Validating data sources.");
// Get data sources needing to be created
final Set<String> availableDatasources = metadataAccess.read(() -> datasourceProvider.getDatasources(datasourceProvider.datasetCriteria().type(UserDatasource.class)).stream().map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId).map(Object::toString).collect(Collectors.toSet()));
final ImportComponentOption componentOption = importFeedOptions.findImportComponentOption(ImportComponent.USER_DATASOURCES);
if (componentOption.getProperties().isEmpty()) {
componentOption.setProperties(FeedImportDatasourceUtil.buildDatasourceAssignmentProperties(metadata, availableDatasources));
// add in any catalogDataSourceIds
if (metadata.getDataTransformation().getCatalogDataSourceIds() != null && !metadata.getDataTransformation().getCatalogDataSourceIds().isEmpty()) {
final Set<String> catalogDataSources = metadataAccess.read(() -> catalogDataSourceProvider.findAll().stream().map(com.thinkbiganalytics.metadata.api.catalog.DataSource::getId).map(Object::toString).collect(Collectors.toSet()));
componentOption.getProperties().addAll(FeedImportDatasourceUtil.buildCatalogDatasourceAssignmentProperties(metadata, catalogDataSources));
}
}
// Update feed with re-mapped data sources
Map<String, String> chartModelReplacements = new HashMap<>();
String sourceDataSetIds = metadata.getSourceDataSetIds();
Set<String> datasetIds = StringUtils.isNotBlank(sourceDataSetIds) ? new HashSet<String>(Arrays.asList(sourceDataSetIds)) : new HashSet<String>();
final boolean valid = componentOption.getProperties().stream().allMatch(property -> {
if (property.getPropertyValue() != null) {
if (property.getAdditionalPropertyValue(FeedImportDatasourceUtil.LEGACY_TABLE_DATA_SOURCE_KEY) != null && "true".equalsIgnoreCase(property.getAdditionalPropertyValue(FeedImportDatasourceUtil.LEGACY_TABLE_DATA_SOURCE_KEY))) {
// remap
String table = property.getAdditionalPropertyValue("table");
String datasourceId = property.getAdditionalPropertyValue("datasourceId");
String datasetId = property.getPropertyValue();
com.thinkbiganalytics.kylo.catalog.rest.model.DataSet dataSet = metadataAccess.read(() -> {
return catalogDataSetProvider.find(catalogDataSetProvider.resolveId(datasetId)).map(catalogDataSet -> catalogModelTransform.dataSetToRestModel().apply(catalogDataSet)).orElse(null);
});
if (dataSet != null) {
FeedImportDatasourceUtil.replaceLegacyDataSourceScript(metadata, table, datasourceId, dataSet);
datasetIds.add(dataSet.getId());
// TODO is this needed?
chartModelReplacements.put(datasourceId, dataSet.getDataSource().getId());
return true;
} else {
return false;
}
}
if (property.getAdditionalPropertyValue(FeedImportDatasourceUtil.LEGACY_QUERY_DATA_SOURCE_KEY) != null && "true".equalsIgnoreCase(property.getAdditionalPropertyValue(FeedImportDatasourceUtil.LEGACY_QUERY_DATA_SOURCE_KEY))) {
// remap
// thre is only 1 datasource throughout, replace the method call and the datasource id with the new one
String datasourceId = property.getAdditionalPropertyValue("datasourceId");
String catalogDataSourceId = property.getPropertyValue();
com.thinkbiganalytics.kylo.catalog.rest.model.DataSource dataSource = metadataAccess.read(() -> {
return catalogDataSourceProvider.find(catalogDataSourceProvider.resolveId(catalogDataSourceId)).map(catalogDataSource -> catalogModelTransform.dataSourceToRestModel().apply(catalogDataSource)).orElse(null);
});
if (dataSource != null) {
FeedImportDatasourceUtil.replaceLegacyQueryDataSourceScript(metadata, datasourceId, dataSource);
// TODO is this needed?
chartModelReplacements.put(datasourceId, dataSource.getId());
return true;
} else {
return false;
}
}
if (property.getAdditionalPropertyValue(FeedImportDatasourceUtil.CATALOG_DATASOURCE_KEY) != null && "true".equalsIgnoreCase(property.getAdditionalPropertyValue(FeedImportDatasourceUtil.CATALOG_DATASOURCE_KEY))) {
String datasourceId = property.getAdditionalPropertyValue("datasourceId");
String catalogDataSourceId = property.getPropertyValue();
com.thinkbiganalytics.kylo.catalog.rest.model.DataSource dataSource = metadataAccess.read(() -> {
return catalogDataSourceProvider.find(catalogDataSourceProvider.resolveId(catalogDataSourceId)).map(catalogDataSource -> catalogModelTransform.dataSourceToRestModel().apply(catalogDataSource)).orElse(null);
});
if (dataSource != null) {
List<String> newIds = metadata.getDataTransformation().getCatalogDataSourceIds().stream().map(id -> id.equalsIgnoreCase(datasourceId) ? catalogDataSourceId : id).collect(Collectors.toList());
metadata.getDataTransformation().setCatalogDataSourceIds(newIds);
String script = metadata.getDataTransformation().getDataTransformScript();
script = script.replaceAll(datasourceId, catalogDataSourceId);
metadata.getDataTransformation().setDataTransformScript(script);
chartModelReplacements.put(datasourceId, catalogDataSourceId);
return true;
} else {
return false;
}
} else {
// ImportUtil.replaceDatasource(metadata, property.getProcessorId(), property.getPropertyValue());
return false;
}
} else {
return false;
}
});
if (valid) {
// make the final replacements and add in the sources
if (datasetIds != null) {
metadata.setSourceDataSetIds(datasetIds.stream().collect(Collectors.joining(",")));
}
FeedImportDatasourceUtil.ensureConnectionKeysMatch(metadata);
FeedImportDatasourceUtil.replaceChartModelReferences(metadata, chartModelReplacements);
FeedImportDatasourceUtil.populateFeedDatasourceIdsProperty(metadata);
statusMessage.update("Validated data sources.", true);
} else {
statusMessage.update("Validation Error. Additional properties are needed before uploading the feed.", false);
importFeed.setValid(false);
}
uploadProgressService.completeSection(importFeed.getImportOptions(), ImportSection.Section.VALIDATE_USER_DATASOURCES);
return valid;
}
use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.
the class JcrDataSourceProviderTest method createDataSource.
protected DataSource createDataSource(int tag) {
DataSource ds = this.dsProvider.create(this.connId, "Test " + tag + " Data Source");
ds.setDescription("Test description " + tag);
return ds;
}
use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.
the class JcrDataSourceProviderTest method testFindPage.
@Test(dependsOnMethods = "testDelete")
public void testFindPage() {
metadata.read(() -> {
Page<DataSource> conns = this.dsProvider.findPage(new PageRequest(0, 5), null);
assertThat(conns).isNotNull();
assertThat(conns).hasSize(5);
assertThat(conns).extracting("systemName", "title", "description").contains(dsTuple(1), dsTuple(2), dsTuple(3), dsTuple(4), dsTuple(5));
}, MetadataAccess.SERVICE);
}
Aggregations