Search in sources :

Example 1 with DataSource

use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.

the class DefaultDataSourcesUpgradeAction method upgradeTo.

@Override
public void upgradeTo(final KyloVersion targetVersion) {
    log.info("Creating default catalog data sources: {}", targetVersion);
    Optional<ConnectorPlugin> plugin = pluginManager.getPlugin("hive");
    if (plugin.isPresent()) {
        Optional<Connector> connector = connectorProvider.findByPlugin(plugin.get().getId());
        if (connector.isPresent()) {
            List<DataSource> hiveSources = dataSourceProvider.findByConnector(connector.get().getId());
            // If at least one Hive data source exists then do nothing.
            if (hiveSources.size() == 0) {
                log.info("Creating default Hive data source");
                DataSource ds = dataSourceProvider.create(connector.get().getId(), "Hive");
                ds.setDescription("The default Hive data source");
                DataSetSparkParameters params = ds.getSparkParameters();
                params.setFormat("hive");
                params.addOption("driver", this.driver);
                params.addOption("url", this.url);
                params.addOption("user", this.user);
                params.addOption("password", this.password);
            } else {
                log.info("One or more Hive data sources already found: {}", hiveSources.stream().map(DataSource::toString).collect(Collectors.toList()));
            }
        } else {
            log.warn("No Hive connector found - cannot create a default Hive data source");
        }
    } else {
        log.warn("No Hive connector plugin found - cannot create a default Hive data source");
    }
}
Also used : ConnectorPlugin(com.thinkbiganalytics.kylo.catalog.spi.ConnectorPlugin) Connector(com.thinkbiganalytics.metadata.api.catalog.Connector) DataSetSparkParameters(com.thinkbiganalytics.metadata.api.catalog.DataSetSparkParameters) DataSource(com.thinkbiganalytics.metadata.api.catalog.DataSource)

Example 2 with DataSource

use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.

the class DefaultFeedManagerFeedService method saveFeed.

@Deprecated
private void saveFeed(final FeedMetadata feed) {
    Feed.ID feedId = metadataAccess.commit(() -> {
        Stopwatch stopwatch = Stopwatch.createStarted();
        List<? extends HadoopSecurityGroup> previousSavedSecurityGroups = null;
        // Store the old security groups before saving because we need to compare afterward
        if (!feed.isNew()) {
            Feed previousStateBeforeSaving = feedProvider.findById(feedProvider.resolveId(feed.getId()));
            Map<String, String> userProperties = previousStateBeforeSaving.getUserProperties();
            previousSavedSecurityGroups = previousStateBeforeSaving.getSecurityGroups();
        }
        // if this is the first time saving this feed create a new one
        Feed domainFeed = feedModelTransform.feedToDomain(feed);
        Feed.ID domainId = domainFeed.getId();
        if (domainFeed.getState() == null) {
            domainFeed.setState(Feed.State.ENABLED);
        }
        stopwatch.stop();
        log.debug("Time to transform the feed to a domain object for saving: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        // initially save the feed
        if (feed.isNew()) {
            stopwatch.start();
            domainFeed = feedProvider.update(domainFeed);
            stopwatch.stop();
            log.debug("Time to save the New feed: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
        }
        final String feedName = FeedNameUtil.fullName(domainFeed.getCategory().getSystemName(), domainFeed.getName());
        // Build preconditions
        stopwatch.start();
        assignFeedDependencies(feed, domainFeed);
        stopwatch.stop();
        log.debug("Time to assignFeedDependencies: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        // Assign the datasources
        stopwatch.start();
        assignFeedDatasources(feed, domainFeed);
        stopwatch.stop();
        log.debug("Time to assignFeedDatasources: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        stopwatch.start();
        boolean isStream = feed.getRegisteredTemplate() != null ? feed.getRegisteredTemplate().isStream() : false;
        Long timeBetweenBatchJobs = feed.getRegisteredTemplate() != null ? feed.getRegisteredTemplate().getTimeBetweenStartingBatchJobs() : 0L;
        // sync the feed information to ops manager
        metadataAccess.commit(() -> opsManagerFeedProvider.save(opsManagerFeedProvider.resolveId(domainId.toString()), feedName, isStream, timeBetweenBatchJobs));
        stopwatch.stop();
        log.debug("Time to sync feed data with Operations Manager: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        // Update hadoop security group polices if the groups changed
        if (!feed.isNew() && !ListUtils.isEqualList(previousSavedSecurityGroups, domainFeed.getSecurityGroups())) {
            stopwatch.start();
            List<? extends HadoopSecurityGroup> securityGroups = domainFeed.getSecurityGroups();
            List<String> groupsAsCommaList = securityGroups.stream().map(group -> group.getName()).collect(Collectors.toList());
            hadoopAuthorizationService.updateSecurityGroupsForAllPolicies(feed.getSystemCategoryName(), feed.getSystemFeedName(), groupsAsCommaList, domainFeed.getProperties());
            stopwatch.stop();
            log.debug("Time to update hadoop security groups: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
        }
        // Update Hive metastore
        if (hiveTargetSyncColumnDescriptions) {
            stopwatch.start();
            final boolean hasHiveDestination = domainFeed.getDestinations().stream().map(FeedDestination::getDatasource).filter(DerivedDatasource.class::isInstance).map(DerivedDatasource.class::cast).anyMatch(datasource -> "HiveDatasource".equals(datasource.getDatasourceType()));
            if (hasHiveDestination) {
                try {
                    feedHiveTableService.updateColumnDescriptions(feed);
                } catch (final DataAccessException e) {
                    log.warn("Failed to update column descriptions for feed: {}", feed.getCategoryAndFeedDisplayName(), e);
                }
            }
            stopwatch.stop();
            log.debug("Time to update hive metastore: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
        }
        // Update Kylo metastore
        stopwatch.start();
        domainFeed = feedProvider.update(domainFeed);
        stopwatch.stop();
        log.debug("Time to call feedProvider.update: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        return domainFeed.getId();
    }, (e) -> {
        if (feed.isNew() && StringUtils.isNotBlank(feed.getId())) {
            // Rollback ops Manager insert if it is newly created
            metadataAccess.commit(() -> {
                opsManagerFeedProvider.delete(opsManagerFeedProvider.resolveId(feed.getId()));
            });
        }
    });
    if (feedId != null) {
        // set deployed as  a service account since it needs access to the versionable node
        metadataAccess.commit(() -> {
            // TODO TEMPORARY
            com.thinkbiganalytics.metadata.api.versioning.EntityVersion<Feed.ID, Feed> version = feedProvider.createVersion(feedId, null, false);
            feedProvider.setDeployed(feedId, version.getId());
        // TODO TEMPORARY
        }, MetadataAccess.SERVICE);
    }
}
Also used : Action(com.thinkbiganalytics.security.action.Action) RegisteredTemplateService(com.thinkbiganalytics.feedmgr.service.template.RegisteredTemplateService) Category(com.thinkbiganalytics.metadata.api.category.Category) DataSourceProvider(com.thinkbiganalytics.metadata.api.catalog.DataSourceProvider) ReusableTemplateConnectionInfo(com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo) Autowired(org.springframework.beans.factory.annotation.Autowired) StringUtils(org.apache.commons.lang3.StringUtils) FeedProvider(com.thinkbiganalytics.metadata.api.feed.FeedProvider) FeedAccessControl(com.thinkbiganalytics.metadata.api.feed.security.FeedAccessControl) Map(java.util.Map) FeedPropertyChangeEvent(com.thinkbiganalytics.metadata.api.event.feed.FeedPropertyChangeEvent) AccessController(com.thinkbiganalytics.security.AccessController) NifiFeed(com.thinkbiganalytics.feedmgr.rest.model.NifiFeed) CategoryAccessControl(com.thinkbiganalytics.metadata.api.category.security.CategoryAccessControl) FeedServicesAccessControl(com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl) SecurityContextHolder(org.springframework.security.core.context.SecurityContextHolder) FeedManagerTemplateProvider(com.thinkbiganalytics.metadata.api.template.FeedManagerTemplateProvider) FeedManagerTemplateService(com.thinkbiganalytics.feedmgr.service.template.FeedManagerTemplateService) MetadataEventListener(com.thinkbiganalytics.metadata.api.event.MetadataEventListener) Obligation(com.thinkbiganalytics.metadata.rest.model.sla.Obligation) NifiProperty(com.thinkbiganalytics.nifi.rest.model.NifiProperty) LabelValue(com.thinkbiganalytics.rest.model.LabelValue) ActionGroup(com.thinkbiganalytics.security.rest.model.ActionGroup) PageRequest(org.springframework.data.domain.PageRequest) Set(java.util.Set) Page(org.springframework.data.domain.Page) EntityVersionDifference(com.thinkbiganalytics.feedmgr.rest.model.EntityVersionDifference) MetadataEventService(com.thinkbiganalytics.metadata.api.event.MetadataEventService) Serializable(java.io.Serializable) DraftEntityVersion(com.thinkbiganalytics.feedmgr.rest.model.DraftEntityVersion) CategoryProvider(com.thinkbiganalytics.metadata.api.category.CategoryProvider) SecurityModelTransform(com.thinkbiganalytics.security.rest.controller.SecurityModelTransform) DataSourceNotFoundException(com.thinkbiganalytics.metadata.api.catalog.DataSourceNotFoundException) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) MetadataRepositoryException(com.thinkbiganalytics.metadata.modeshape.MetadataRepositoryException) ServiceLevelAgreementBuilder(com.thinkbiganalytics.metadata.sla.spi.ServiceLevelAgreementBuilder) RegisteredTemplateRequest(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplateRequest) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) NiFiObjectCache(com.thinkbiganalytics.nifi.rest.NiFiObjectCache) DerivedDatasourceFactory(com.thinkbiganalytics.feedmgr.service.feed.datasource.DerivedDatasourceFactory) MetadataChange(com.thinkbiganalytics.metadata.api.event.MetadataChange) RegisteredTemplate(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate) ObligationGroup(com.thinkbiganalytics.metadata.sla.api.ObligationGroup) Nullable(javax.annotation.Nullable) Datasource(com.thinkbiganalytics.metadata.api.datasource.Datasource) Properties(java.util.Properties) SecurityService(com.thinkbiganalytics.feedmgr.service.security.SecurityService) FeedProperties(com.thinkbiganalytics.metadata.api.feed.FeedProperties) FeedManagerTemplate(com.thinkbiganalytics.metadata.api.template.FeedManagerTemplate) HadoopSecurityGroup(com.thinkbiganalytics.metadata.api.security.HadoopSecurityGroup) ImmutableAllowableAction(com.thinkbiganalytics.security.role.ImmutableAllowableAction) TemplateConnectionUtil(com.thinkbiganalytics.feedmgr.nifi.TemplateConnectionUtil) HadoopAuthorizationService(com.thinkbiganalytics.datalake.authorization.service.HadoopAuthorizationService) ListUtils(org.apache.commons.collections.ListUtils) LoggerFactory(org.slf4j.LoggerFactory) FeedChange(com.thinkbiganalytics.metadata.api.event.feed.FeedChange) NiFiPropertyDescriptorTransform(com.thinkbiganalytics.nifi.rest.model.NiFiPropertyDescriptorTransform) Precondition(com.thinkbiganalytics.policy.precondition.Precondition) PreDestroy(javax.annotation.PreDestroy) DataSetProvider(com.thinkbiganalytics.metadata.api.catalog.DataSetProvider) NifiPropertyUtil(com.thinkbiganalytics.nifi.rest.support.NifiPropertyUtil) Pageable(org.springframework.data.domain.Pageable) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) CatalogModelTransform(com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform) FeedVersions(com.thinkbiganalytics.feedmgr.rest.model.FeedVersions) DeployResponseEntityVersion(com.thinkbiganalytics.feedmgr.rest.model.DeployResponseEntityVersion) FeedDestination(com.thinkbiganalytics.metadata.api.feed.FeedDestination) OpsManagerFeedProvider(com.thinkbiganalytics.metadata.api.feed.OpsManagerFeedProvider) ImmutableAllowedActions(com.thinkbiganalytics.security.role.ImmutableAllowedActions) UserField(com.thinkbiganalytics.feedmgr.rest.model.UserField) ImmutableMap(com.google.common.collect.ImmutableMap) DataSetNotFoundException(com.thinkbiganalytics.metadata.api.catalog.DataSetNotFoundException) Collection(java.util.Collection) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) NotFoundException(javax.ws.rs.NotFoundException) EntityVersion(com.thinkbiganalytics.feedmgr.rest.model.EntityVersion) FeedSummary(com.thinkbiganalytics.feedmgr.rest.model.FeedSummary) InputOutputPort(com.thinkbiganalytics.nifi.feedmgr.InputOutputPort) List(java.util.List) Principal(java.security.Principal) DataSource(com.thinkbiganalytics.metadata.api.catalog.DataSource) PostConstruct(javax.annotation.PostConstruct) Optional(java.util.Optional) NifiProcessGroup(com.thinkbiganalytics.nifi.rest.model.NifiProcessGroup) DerivedDatasource(com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource) PreconditionRule(com.thinkbiganalytics.policy.rest.model.PreconditionRule) DataAccessException(org.springframework.dao.DataAccessException) Stopwatch(com.google.common.base.Stopwatch) Feed(com.thinkbiganalytics.metadata.api.feed.Feed) HashMap(java.util.HashMap) AllowedActions(com.thinkbiganalytics.security.action.AllowedActions) UserProperty(com.thinkbiganalytics.feedmgr.rest.model.UserProperty) HashSet(java.util.HashSet) Inject(javax.inject.Inject) UIFeed(com.thinkbiganalytics.feedmgr.rest.model.UIFeed) PropertyExpressionResolver(com.thinkbiganalytics.feedmgr.nifi.PropertyExpressionResolver) ServiceLevelAgreementService(com.thinkbiganalytics.feedmgr.sla.ServiceLevelAgreementService) FeedChangeEvent(com.thinkbiganalytics.metadata.api.event.feed.FeedChangeEvent) Qualifier(org.springframework.beans.factory.annotation.Qualifier) FeedRollbackException(com.thinkbiganalytics.nifi.feedmgr.FeedRollbackException) FeedSource(com.thinkbiganalytics.metadata.api.feed.FeedSource) Nonnull(javax.annotation.Nonnull) FeedNotFoundException(com.thinkbiganalytics.metadata.api.feed.FeedNotFoundException) Logger(org.slf4j.Logger) FeedNameUtil(com.thinkbiganalytics.support.FeedNameUtil) CreateFeedBuilder(com.thinkbiganalytics.feedmgr.nifi.CreateFeedBuilder) FeedHistoryDataReindexingService(com.thinkbiganalytics.feedmgr.service.feed.reindexing.FeedHistoryDataReindexingService) DateTime(org.joda.time.DateTime) UserFieldDescriptor(com.thinkbiganalytics.metadata.api.extension.UserFieldDescriptor) FieldRuleProperty(com.thinkbiganalytics.policy.rest.model.FieldRuleProperty) TimeUnit(java.util.concurrent.TimeUnit) NifiFlowCache(com.thinkbiganalytics.feedmgr.nifi.cache.NifiFlowCache) AbstractMap(java.util.AbstractMap) UserPropertyTransform(com.thinkbiganalytics.feedmgr.service.UserPropertyTransform) NiFiTemplateCache(com.thinkbiganalytics.feedmgr.service.template.NiFiTemplateCache) DependentFeedPrecondition(com.thinkbiganalytics.policy.precondition.DependentFeedPrecondition) PreconditionPolicyTransformer(com.thinkbiganalytics.policy.precondition.transform.PreconditionPolicyTransformer) Comparator(java.util.Comparator) Collections(java.util.Collections) LegacyNifiRestClient(com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient) Stopwatch(com.google.common.base.Stopwatch) DerivedDatasource(com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource) DataAccessException(org.springframework.dao.DataAccessException) NifiFeed(com.thinkbiganalytics.feedmgr.rest.model.NifiFeed) Feed(com.thinkbiganalytics.metadata.api.feed.Feed) UIFeed(com.thinkbiganalytics.feedmgr.rest.model.UIFeed)

Example 3 with DataSource

use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.

the class FeedImporter method validateUserDatasources.

/**
 * Validates that user data sources can be imported with provided properties.
 * Legacy UserDatasources will be remapped toe Catalog DataSources or DataSets
 *
 * @return {@code true} if the feed can be imported, or {@code false} otherwise
 */
private boolean validateUserDatasources() {
    FeedMetadata metadata = importFeed.getFeedToImport();
    final UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importFeed.getImportOptions().getUploadKey(), "Validating data sources.");
    // Get data sources needing to be created
    final Set<String> availableDatasources = metadataAccess.read(() -> datasourceProvider.getDatasources(datasourceProvider.datasetCriteria().type(UserDatasource.class)).stream().map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId).map(Object::toString).collect(Collectors.toSet()));
    final ImportComponentOption componentOption = importFeedOptions.findImportComponentOption(ImportComponent.USER_DATASOURCES);
    if (componentOption.getProperties().isEmpty()) {
        componentOption.setProperties(FeedImportDatasourceUtil.buildDatasourceAssignmentProperties(metadata, availableDatasources));
        // add in any catalogDataSourceIds
        if (metadata.getDataTransformation().getCatalogDataSourceIds() != null && !metadata.getDataTransformation().getCatalogDataSourceIds().isEmpty()) {
            final Set<String> catalogDataSources = metadataAccess.read(() -> catalogDataSourceProvider.findAll().stream().map(com.thinkbiganalytics.metadata.api.catalog.DataSource::getId).map(Object::toString).collect(Collectors.toSet()));
            componentOption.getProperties().addAll(FeedImportDatasourceUtil.buildCatalogDatasourceAssignmentProperties(metadata, catalogDataSources));
        }
    }
    // Update feed with re-mapped data sources
    Map<String, String> chartModelReplacements = new HashMap<>();
    String sourceDataSetIds = metadata.getSourceDataSetIds();
    Set<String> datasetIds = StringUtils.isNotBlank(sourceDataSetIds) ? new HashSet<String>(Arrays.asList(sourceDataSetIds)) : new HashSet<String>();
    final boolean valid = componentOption.getProperties().stream().allMatch(property -> {
        if (property.getPropertyValue() != null) {
            if (property.getAdditionalPropertyValue(FeedImportDatasourceUtil.LEGACY_TABLE_DATA_SOURCE_KEY) != null && "true".equalsIgnoreCase(property.getAdditionalPropertyValue(FeedImportDatasourceUtil.LEGACY_TABLE_DATA_SOURCE_KEY))) {
                // remap
                String table = property.getAdditionalPropertyValue("table");
                String datasourceId = property.getAdditionalPropertyValue("datasourceId");
                String datasetId = property.getPropertyValue();
                com.thinkbiganalytics.kylo.catalog.rest.model.DataSet dataSet = metadataAccess.read(() -> {
                    return catalogDataSetProvider.find(catalogDataSetProvider.resolveId(datasetId)).map(catalogDataSet -> catalogModelTransform.dataSetToRestModel().apply(catalogDataSet)).orElse(null);
                });
                if (dataSet != null) {
                    FeedImportDatasourceUtil.replaceLegacyDataSourceScript(metadata, table, datasourceId, dataSet);
                    datasetIds.add(dataSet.getId());
                    // TODO is this needed?
                    chartModelReplacements.put(datasourceId, dataSet.getDataSource().getId());
                    return true;
                } else {
                    return false;
                }
            }
            if (property.getAdditionalPropertyValue(FeedImportDatasourceUtil.LEGACY_QUERY_DATA_SOURCE_KEY) != null && "true".equalsIgnoreCase(property.getAdditionalPropertyValue(FeedImportDatasourceUtil.LEGACY_QUERY_DATA_SOURCE_KEY))) {
                // remap
                // thre is only 1 datasource throughout, replace the method call and the datasource id with the new one
                String datasourceId = property.getAdditionalPropertyValue("datasourceId");
                String catalogDataSourceId = property.getPropertyValue();
                com.thinkbiganalytics.kylo.catalog.rest.model.DataSource dataSource = metadataAccess.read(() -> {
                    return catalogDataSourceProvider.find(catalogDataSourceProvider.resolveId(catalogDataSourceId)).map(catalogDataSource -> catalogModelTransform.dataSourceToRestModel().apply(catalogDataSource)).orElse(null);
                });
                if (dataSource != null) {
                    FeedImportDatasourceUtil.replaceLegacyQueryDataSourceScript(metadata, datasourceId, dataSource);
                    // TODO is this needed?
                    chartModelReplacements.put(datasourceId, dataSource.getId());
                    return true;
                } else {
                    return false;
                }
            }
            if (property.getAdditionalPropertyValue(FeedImportDatasourceUtil.CATALOG_DATASOURCE_KEY) != null && "true".equalsIgnoreCase(property.getAdditionalPropertyValue(FeedImportDatasourceUtil.CATALOG_DATASOURCE_KEY))) {
                String datasourceId = property.getAdditionalPropertyValue("datasourceId");
                String catalogDataSourceId = property.getPropertyValue();
                com.thinkbiganalytics.kylo.catalog.rest.model.DataSource dataSource = metadataAccess.read(() -> {
                    return catalogDataSourceProvider.find(catalogDataSourceProvider.resolveId(catalogDataSourceId)).map(catalogDataSource -> catalogModelTransform.dataSourceToRestModel().apply(catalogDataSource)).orElse(null);
                });
                if (dataSource != null) {
                    List<String> newIds = metadata.getDataTransformation().getCatalogDataSourceIds().stream().map(id -> id.equalsIgnoreCase(datasourceId) ? catalogDataSourceId : id).collect(Collectors.toList());
                    metadata.getDataTransformation().setCatalogDataSourceIds(newIds);
                    String script = metadata.getDataTransformation().getDataTransformScript();
                    script = script.replaceAll(datasourceId, catalogDataSourceId);
                    metadata.getDataTransformation().setDataTransformScript(script);
                    chartModelReplacements.put(datasourceId, catalogDataSourceId);
                    return true;
                } else {
                    return false;
                }
            } else {
                // ImportUtil.replaceDatasource(metadata, property.getProcessorId(), property.getPropertyValue());
                return false;
            }
        } else {
            return false;
        }
    });
    if (valid) {
        // make the final replacements and add in the sources
        if (datasetIds != null) {
            metadata.setSourceDataSetIds(datasetIds.stream().collect(Collectors.joining(",")));
        }
        FeedImportDatasourceUtil.ensureConnectionKeysMatch(metadata);
        FeedImportDatasourceUtil.replaceChartModelReferences(metadata, chartModelReplacements);
        FeedImportDatasourceUtil.populateFeedDatasourceIdsProperty(metadata);
        statusMessage.update("Validated data sources.", true);
    } else {
        statusMessage.update("Validation Error. Additional properties are needed before uploading the feed.", false);
        importFeed.setValid(false);
    }
    uploadProgressService.completeSection(importFeed.getImportOptions(), ImportSection.Section.VALIDATE_USER_DATASOURCES);
    return valid;
}
Also used : UploadProgressService(com.thinkbiganalytics.feedmgr.service.UploadProgressService) Arrays(java.util.Arrays) RegisteredTemplateService(com.thinkbiganalytics.feedmgr.service.template.RegisteredTemplateService) Category(com.thinkbiganalytics.metadata.api.category.Category) DataSourceProvider(com.thinkbiganalytics.metadata.api.catalog.DataSourceProvider) LoggerFactory(org.slf4j.LoggerFactory) DatasourceModelTransform(com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform) TemplateImporter(com.thinkbiganalytics.feedmgr.service.template.importing.TemplateImporter) StringUtils(org.apache.commons.lang3.StringUtils) DataSetProvider(com.thinkbiganalytics.metadata.api.catalog.DataSetProvider) ByteArrayInputStream(java.io.ByteArrayInputStream) FeedAccessControl(com.thinkbiganalytics.metadata.api.feed.security.FeedAccessControl) FeedManagerCategoryService(com.thinkbiganalytics.feedmgr.service.category.FeedManagerCategoryService) NifiPropertyUtil(com.thinkbiganalytics.nifi.rest.support.NifiPropertyUtil) Map(java.util.Map) ImportUtil(com.thinkbiganalytics.feedmgr.util.ImportUtil) AccessController(com.thinkbiganalytics.security.AccessController) NifiFeed(com.thinkbiganalytics.feedmgr.rest.model.NifiFeed) CategoryAccessControl(com.thinkbiganalytics.metadata.api.category.security.CategoryAccessControl) FeedServicesAccessControl(com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) CatalogModelTransform(com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform) ZipEntry(java.util.zip.ZipEntry) MetadataService(com.thinkbiganalytics.feedmgr.service.MetadataService) PolicyPropertyTypes(com.thinkbiganalytics.policy.PolicyPropertyTypes) MetadataFieldAnnotationFieldNameResolver(com.thinkbiganalytics.feedmgr.MetadataFieldAnnotationFieldNameResolver) UserField(com.thinkbiganalytics.feedmgr.rest.model.UserField) ImportTemplate(com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate) NifiProperty(com.thinkbiganalytics.nifi.rest.model.NifiProperty) ImportPropertyBuilder(com.thinkbiganalytics.feedmgr.rest.model.ImportPropertyBuilder) Set(java.util.Set) UploadProgress(com.thinkbiganalytics.feedmgr.rest.model.UploadProgress) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) UploadProgressMessage(com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage) ZipFileUtil(com.thinkbiganalytics.feedmgr.support.ZipFileUtil) Objects(java.util.Objects) List(java.util.List) CategoryProvider(com.thinkbiganalytics.metadata.api.category.CategoryProvider) DataSource(com.thinkbiganalytics.metadata.api.catalog.DataSource) ImportTemplateOptions(com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions) ImportFeed(com.thinkbiganalytics.feedmgr.service.feed.importing.model.ImportFeed) ImportFeedException(com.thinkbiganalytics.feedmgr.service.feed.ImportFeedException) ImportComponent(com.thinkbiganalytics.feedmgr.rest.ImportComponent) ZipInputStream(java.util.zip.ZipInputStream) FeedManagerFeedService(com.thinkbiganalytics.feedmgr.service.feed.FeedManagerFeedService) DataSet(com.thinkbiganalytics.metadata.api.catalog.DataSet) HashMap(java.util.HashMap) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) ImportSection(com.thinkbiganalytics.feedmgr.rest.ImportSection) UserProperty(com.thinkbiganalytics.feedmgr.rest.model.UserProperty) ArrayList(java.util.ArrayList) FeedCategory(com.thinkbiganalytics.feedmgr.rest.model.FeedCategory) HashSet(java.util.HashSet) Inject(javax.inject.Inject) ImportProperty(com.thinkbiganalytics.feedmgr.rest.model.ImportProperty) ImportType(com.thinkbiganalytics.feedmgr.rest.ImportType) ImportFeedOptions(com.thinkbiganalytics.feedmgr.rest.model.ImportFeedOptions) RegisteredTemplate(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate) TemplateImporterFactory(com.thinkbiganalytics.feedmgr.service.template.importing.TemplateImporterFactory) ConnectorProvider(com.thinkbiganalytics.metadata.api.catalog.ConnectorProvider) UserDatasource(com.thinkbiganalytics.metadata.api.datasource.UserDatasource) ImportTemplateRoutineFactory(com.thinkbiganalytics.feedmgr.service.template.importing.importprocess.ImportTemplateRoutineFactory) Logger(org.slf4j.Logger) FeedNameUtil(com.thinkbiganalytics.support.FeedNameUtil) ImportTemplateRoutine(com.thinkbiganalytics.feedmgr.service.template.importing.importprocess.ImportTemplateRoutine) IOException(java.io.IOException) ImportException(com.thinkbiganalytics.feedmgr.service.template.importing.ImportException) UserPropertyTransform(com.thinkbiganalytics.feedmgr.service.UserPropertyTransform) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) Collections(java.util.Collections) LegacyNifiRestClient(com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient) ImportComponentOption(com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption) InputStream(java.io.InputStream) HashMap(java.util.HashMap) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) ImportComponentOption(com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption) UploadProgressMessage(com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage)

Example 4 with DataSource

use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.

the class JcrDataSourceProviderTest method createDataSource.

protected DataSource createDataSource(int tag) {
    DataSource ds = this.dsProvider.create(this.connId, "Test " + tag + " Data Source");
    ds.setDescription("Test description " + tag);
    return ds;
}
Also used : DataSource(com.thinkbiganalytics.metadata.api.catalog.DataSource)

Example 5 with DataSource

use of com.thinkbiganalytics.metadata.api.catalog.DataSource in project kylo by Teradata.

the class JcrDataSourceProviderTest method testFindPage.

@Test(dependsOnMethods = "testDelete")
public void testFindPage() {
    metadata.read(() -> {
        Page<DataSource> conns = this.dsProvider.findPage(new PageRequest(0, 5), null);
        assertThat(conns).isNotNull();
        assertThat(conns).hasSize(5);
        assertThat(conns).extracting("systemName", "title", "description").contains(dsTuple(1), dsTuple(2), dsTuple(3), dsTuple(4), dsTuple(5));
    }, MetadataAccess.SERVICE);
}
Also used : PageRequest(org.springframework.data.domain.PageRequest) DataSource(com.thinkbiganalytics.metadata.api.catalog.DataSource) Test(org.testng.annotations.Test) SpringBootTest(org.springframework.boot.test.context.SpringBootTest)

Aggregations

DataSource (com.thinkbiganalytics.metadata.api.catalog.DataSource)8 Sets (com.google.common.collect.Sets)4 FeedMetadata (com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata)4 NifiFeed (com.thinkbiganalytics.feedmgr.rest.model.NifiFeed)4 RegisteredTemplate (com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate)4 UserField (com.thinkbiganalytics.feedmgr.rest.model.UserField)4 UserProperty (com.thinkbiganalytics.feedmgr.rest.model.UserProperty)4 FeedServicesAccessControl (com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl)4 UserPropertyTransform (com.thinkbiganalytics.feedmgr.service.UserPropertyTransform)4 RegisteredTemplateService (com.thinkbiganalytics.feedmgr.service.template.RegisteredTemplateService)4 CatalogModelTransform (com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform)4 MetadataAccess (com.thinkbiganalytics.metadata.api.MetadataAccess)4 DataSetProvider (com.thinkbiganalytics.metadata.api.catalog.DataSetProvider)4 DataSourceProvider (com.thinkbiganalytics.metadata.api.catalog.DataSourceProvider)4 Category (com.thinkbiganalytics.metadata.api.category.Category)4 CategoryProvider (com.thinkbiganalytics.metadata.api.category.CategoryProvider)4 Stopwatch (com.google.common.base.Stopwatch)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 HadoopAuthorizationService (com.thinkbiganalytics.datalake.authorization.service.HadoopAuthorizationService)2 CreateFeedBuilder (com.thinkbiganalytics.feedmgr.nifi.CreateFeedBuilder)2