Examples with DataSource com.thinkbiganalytics.kylo.catalog.rest.model.DataSource used on opensource projects

Example 11 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class DataSetProvider method createDataSet.

/**
 * Creates a new data set using the specified template.
 * @param encryptedCredentials
 *
 * @throws CatalogException if the data set is not valid
 */
@Nonnull
public DataSet createDataSet(@Nonnull final DataSet source, boolean encryptedCredentials) {
    // Find data source
    final DataSource dataSource = Optional.of(source).map(DataSet::getDataSource).map(DataSource::getId).flatMap(dataSourceProvider::findDataSource).orElseThrow(() -> new CatalogException("catalog.dataset.datasource.invalid"));
    // Validate data set
    final DataSet dataSet = new DataSet(source);
    dataSet.setDataSource(dataSource);
    validateDataSet(dataSet);
    return metadataService.commit(() -> {
        // Require admin permission if the results should include unencrypted credentials.
        accessController.checkPermission(AccessController.SERVICES, encryptedCredentials ? FeedServicesAccessControl.ACCESS_DATASOURCES : FeedServicesAccessControl.ADMIN_DATASOURCES);
        // Resolve the real data set if possible, otherwise create
        com.thinkbiganalytics.metadata.api.catalog.DataSource.ID dataSourceId = dataSourceMetadataProvider.resolveId(dataSource.getId());
        com.thinkbiganalytics.metadata.api.catalog.DataSet ds = modelTransform.buildDataSet(source, metadataProvider.build(dataSourceId));
        return modelTransform.dataSetToRestModel(encryptedCredentials).apply(ds);
    });
}

Example 12 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class DataSourceProvider method findAllDataSources.

/**
 * Gets the list of available connectors.
 */
@Nonnull
public Page<DataSource> findAllDataSources(@Nonnull final Pageable pageable, @Nullable final String filter) {
    // Filter catalog data sources
    final Supplier<Stream<DataSource>> catalogDataSources = () -> metadataService.read(() -> metadataProvider.findAll().stream().map(modelTransform.dataSourceToRestModel()));
    // Filter feed data sources
    final DatasourceCriteria feedDataSourcesCriteria = feedDataSourceProvider.datasetCriteria().type(UserDatasource.class);
    final Supplier<Stream<DataSource>> feedDataSources = () -> feedDataSourceProvider.getDatasources(feedDataSourcesCriteria).stream().filter(containsIgnoreCase(Datasource::getName, filter)).map(datasource -> toDataSource(datasource, DatasourceModelTransform.Level.BASIC)).filter(Objects::nonNull);
    // Sort and paginate
    final List<DataSource> filteredDataSources = Stream.concat(catalogDataSources.get(), feedDataSources.get()).sorted(Comparator.comparing(DataSource::getTitle)).skip(pageable.getOffset()).limit(pageable.getPageSize()).peek(findConnector()).collect(Collectors.toList());
    return new PageImpl<>(filteredDataSources, pageable, catalogDataSources.get().count() + feedDataSources.get().count());
}

Example 13 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class DataSourceProvider method updateDataSource.

/**
 * Updates a data source using the specified template.
 *
 * @throws CatalogException if the data source is not valid
 */
@Nonnull
public DataSource updateDataSource(@Nonnull final DataSource source) {
    return metadataService.commit(() -> {
        // Find data source
        final Optional<com.thinkbiganalytics.metadata.api.catalog.DataSource.ID> domainId = Optional.ofNullable(source.getId()).map(metadataProvider::resolveId);
        final com.thinkbiganalytics.metadata.api.catalog.DataSource domain = domainId.flatMap(metadataProvider::find).orElseThrow(() -> new CatalogException("catalog.datasource.notFound.id", source.getId()));
        // Create or update controller service
        final ConnectorPluginDescriptor plugin = pluginManager.getPlugin(domain.getConnector().getPluginId()).map(ConnectorPlugin::getDescriptor).orElse(null);
        if (plugin != null && plugin.getNifiControllerService() != null) {
            createOrUpdateNiFiControllerService(source, plugin, false);
        }
        // Update catalog
        final DataSource updatedDataSource = this.credentialManager.applyPlaceholders(source, SecurityContextUtil.getCurrentPrincipals());
        modelTransform.updateDataSource(updatedDataSource, domain);
        // Return a copy with the connector
        return modelTransform.dataSourceToRestModel().apply(domain);
    });
}

Example 14 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class DataSetUtilTest method getPaths.

/**
 * Verify merging paths for a data set.
 */
@Test
public void getPaths() {
    // Create mock connector
    final DefaultDataSetTemplate connectorTemplate = new DefaultDataSetTemplate();
    connectorTemplate.setOptions(Collections.singletonMap("path", "connector1.txt"));
    connectorTemplate.setPaths(Collections.singletonList("connector2.txt"));
    final Connector connector = new Connector();
    connector.setTemplate(connectorTemplate);
    // Create mock data source
    final DefaultDataSetTemplate dataSourceTemplate = new DefaultDataSetTemplate();
    dataSourceTemplate.setOptions(Collections.singletonMap("path", "datasource1.txt"));
    dataSourceTemplate.setPaths(Collections.singletonList("datasource2.txt"));
    final DataSource dataSource = new DataSource();
    dataSource.setConnector(connector);
    dataSource.setTemplate(dataSourceTemplate);
    // Create mock data set
    final DataSet dataSet = new DataSet();
    dataSet.setDataSource(dataSource);
    dataSet.setOptions(Collections.singletonMap("path", "dataset1.txt"));
    dataSet.setPaths(Collections.singletonList("dataset2.txt"));
    // Test retrieving data set paths
    Assert.assertEquals(Arrays.asList("dataset1.txt", "dataset2.txt"), DataSetUtil.getPaths(dataSet).orElse(null));
    // Test retrieving data source paths
    dataSet.setOptions(null);
    Assert.assertEquals(Arrays.asList("datasource1.txt", "dataset2.txt"), DataSetUtil.getPaths(dataSet).orElse(null));
    dataSet.setPaths(null);
    Assert.assertEquals(Arrays.asList("datasource1.txt", "datasource2.txt"), DataSetUtil.getPaths(dataSet).orElse(null));
    // Test retrieving connector paths
    dataSourceTemplate.setOptions(null);
    Assert.assertEquals(Arrays.asList("connector1.txt", "datasource2.txt"), DataSetUtil.getPaths(dataSet).orElse(null));
    dataSourceTemplate.setPaths(null);
    Assert.assertEquals(Arrays.asList("connector1.txt", "connector2.txt"), DataSetUtil.getPaths(dataSet).orElse(null));
    // Test retrieving empty paths
    connectorTemplate.setOptions(null);
    connectorTemplate.setPaths(null);
    Assert.assertEquals(Optional.empty(), DataSetUtil.getPaths(dataSet));
}

Example 15 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class CatalogFileManagerTest method createDataSet.

/**
 * Creates a new data set.
 */
@Nonnull
private DataSet createDataSet() {
    final DefaultDataSetTemplate dataSourceTemplate = new DefaultDataSetTemplate();
    dataSourceTemplate.setPaths(Collections.singletonList(datasetsFolder.getRoot().toURI().toString()));
    final DataSource dataSource = new DataSource();
    dataSource.setId(UUID.randomUUID().toString());
    dataSource.setTemplate(dataSourceTemplate);
    final DataSet dataSet = new DataSet();
    dataSet.setId(UUID.randomUUID().toString());
    dataSet.setDataSource(dataSource);
    return dataSet;
}