Examples with DataSource com.thinkbiganalytics.kylo.catalog.rest.model.DataSource used on opensource projects

Example 1 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class DataSetIT method testUpload.

/**
 * Verify uploading files.
 */
// TODO create local-files data source
@Ignore
@Test
public void testUpload() {
    // Create a data set
    final DataSet request = new DataSet();
    request.setDataSource(new DataSource());
    request.getDataSource().setId("local-files");
    final DataSet dataSet = given(DataSetController.BASE).when().body(request).post().then().statusCode(200).extract().as(DataSet.class);
    final String id = dataSet.getId();
    // Test empty data set
    List<DataSetFile> files = listUploads(id);
    Assert.assertEquals(0, files.size());
    // Upload sample files
    uploadFile(id, getSampleFile("userdata1.csv"));
    uploadFile(id, getSampleFile("userdata2.csv"));
    files = listUploads(id);
    Assert.assertThat(files, CoreMatchers.hasItem(nameEquals("userdata1.csv")));
    Assert.assertThat(files, CoreMatchers.hasItem(nameEquals("userdata2.csv")));
    Assert.assertEquals(2, files.size());
    // Delete a file
    given(DataSetController.BASE).when().delete(id + "/uploads/userdata1.csv").then().statusCode(204);
    files = listUploads(id);
    Assert.assertThat(files, CoreMatchers.hasItem(nameEquals("userdata2.csv")));
    Assert.assertEquals(1, files.size());
}

Example 2 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class DataSourceIT method testFindAll.

/**
 * Verifying retrieving all data sources.
 */
@Test
public void testFindAll() {
    // Create a feed data source
    final Connector[] connectors = listConnectors();
    Connector jdbcConnector = Arrays.asList(connectors).stream().filter(c -> c.getPluginId().equalsIgnoreCase("jdbc")).findFirst().orElse(null);
    DataSource ds = new DataSource();
    ds.setConnector(jdbcConnector);
    ds.setTitle("MySql Test");
    DefaultDataSetTemplate dsTemplate = new DefaultDataSetTemplate();
    Map<String, String> options = new HashMap<>();
    options.put("driver", "org.mariadb.jdbc.Driver");
    options.put("user", "root");
    options.put("password", "secret");
    options.put("url", "jdbc:mysql://localhost:3306/kylo");
    dsTemplate.setOptions(options);
    ds.setTemplate(dsTemplate);
    final DataSource jdbcDatasource = createDataSource(ds);
    // Find all data sources
    final SearchResult<DataSource> searchResult = given(DataSourceController.BASE).when().get().then().statusCode(200).extract().as(DataSourceSearchResult.class);
    final Matcher<DataSource> isHive = new CustomMatcher<DataSource>("is hive data source") {

        @Override
        public boolean matches(final Object item) {
            return (item instanceof DataSource && "Hive".equals(((DataSource) item).getTitle()));
        }
    };
    final Matcher<DataSource> isJdbc = new CustomMatcher<DataSource>("is jdbc data source") {

        @Override
        public boolean matches(final Object item) {
            final DataSource dataSource = (item instanceof DataSource) ? (DataSource) item : null;
            return (dataSource != null && jdbcDatasource.getId().equals(dataSource.getId()) && jdbcDatasource.getTitle().equals(dataSource.getTitle()));
        }
    };
    Assert.assertThat(searchResult.getData(), CoreMatchers.hasItem(isHive));
    Assert.assertThat(searchResult.getData(), CoreMatchers.hasItem(isJdbc));
    Assert.assertEquals(searchResult.getData().size(), searchResult.getRecordsTotal().longValue());
}

Example 3 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class FeedModelTransform method domainToFeedMetadata.

/**
 * Transforms the specified Metadata feed to a Feed Manager feed.
 *
 * @param domain       the Metadata feed
 * @param userFieldMap cache map from category to user-defined fields, or {@code null}
 * @return the Feed Manager feed
 */
@Nonnull
private FeedMetadata domainToFeedMetadata(@Nonnull final Feed domain, @Nullable final Map<Category, Set<UserFieldDescriptor>> userFieldMap, @Nullable ActionGroup actionGroup) {
    FeedMetadata feed = deserializeFeedMetadata(domain, false);
    feed.setId(domain.getId().toString());
    feed.setFeedId(domain.getId().toString());
    feed.setFeedName(domain.getDisplayName());
    feed.setSystemFeedName(domain.getName());
    feed.setDescription(domain.getDescription());
    feed.setAllowIndexing(domain.isAllowIndexing());
    feed.setHistoryReindexingStatus(domain.getCurrentHistoryReindexingStatus().getHistoryReindexingState().toString());
    feed.setOwner(domain.getOwner() != null ? new User(domain.getOwner().getName()) : null);
    if (domain.getCreatedTime() != null) {
        feed.setCreateDate(domain.getCreatedTime().toDate());
    }
    if (domain.getModifiedTime() != null) {
        feed.setUpdateDate(domain.getModifiedTime().toDate());
    }
    setFeedMetadataRegisteredTemplate(domain, feed);
    Category category = domain.getCategory();
    if (category != null) {
        feed.setCategory(categoryModelTransform.domainToFeedCategorySimple(category));
    }
    feed.setState(domain.getState() != null ? domain.getState().name() : null);
    // Set user-defined properties
    final Set<UserFieldDescriptor> userFields;
    if (userFieldMap == null) {
        userFields = getUserFields(category);
    } else if (userFieldMap.containsKey(category)) {
        userFields = userFieldMap.get(category);
    } else {
        userFields = getUserFields(category);
        userFieldMap.put(category, userFields);
    }
    @SuppressWarnings("unchecked") final Set<UserProperty> userProperties = UserPropertyTransform.toUserProperties(domain.getUserProperties(), userFields);
    feed.setUserProperties(userProperties);
    // Convert JCR securitygroup to DTO
    List<com.thinkbiganalytics.feedmgr.rest.model.HadoopSecurityGroup> restSecurityGroups = new ArrayList<>();
    if (domain.getSecurityGroups() != null && domain.getSecurityGroups().size() > 0) {
        for (Object group : domain.getSecurityGroups()) {
            HadoopSecurityGroup hadoopSecurityGroup = (HadoopSecurityGroup) group;
            com.thinkbiganalytics.feedmgr.rest.model.HadoopSecurityGroup restSecurityGroup = new com.thinkbiganalytics.feedmgr.rest.model.HadoopSecurityGroup();
            restSecurityGroup.setDescription(hadoopSecurityGroup.getDescription());
            restSecurityGroup.setId(hadoopSecurityGroup.getGroupId());
            restSecurityGroup.setName(hadoopSecurityGroup.getName());
            restSecurityGroups.add(restSecurityGroup);
        }
    }
    feed.setSecurityGroups(restSecurityGroups);
    feed.setTags(domain.getTags().stream().map(name -> new DefaultTag(name)).collect(Collectors.toList()));
    if (domain.getUsedByFeeds() != null) {
        final List<FeedSummary> usedByFeeds = domain.getUsedByFeeds().stream().map(this::domainToFeedSummary).collect(Collectors.toList());
        feed.setUsedByFeeds(usedByFeeds);
    }
    List<DataSet> srcDataSets = domain.getSources().stream().map(FeedConnection::getDataSet).filter(Optional::isPresent).map(Optional::get).map(dataSet -> {
        try {
            return catalogModelTransform.dataSetToRestModel().apply(dataSet);
        } catch (final AccessControlException e) {
            log.debug("Denied access to data set: {}: {}", domain.getId(), e);
            final DataSource dataSource = new DataSource();
            dataSource.setAllowedActions(new ActionGroup());
            dataSource.getAllowedActions().setActions(Collections.emptyList());
            final DataSet model = new DataSet();
            model.setId(domain.getId().toString());
            model.setDataSource(dataSource);
            return model;
        }
    }).collect(Collectors.toList());
    feed.setSourceDataSets(srcDataSets);
    if (actionGroup == null) {
        // add in access control items
        securityTransform.applyAccessControl(domain, feed);
    } else {
        feed.setAllowedActions(actionGroup);
    }
    return feed;
}

Example 4 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class FeedModelTransform method updateDataSets.

/**
 * Updates the data sets of the specified feed.
 */
public void updateDataSets(@Nullable final FeedMetadata feed) {
    if (feed != null && feed.getSourceDataSets() != null) {
        feed.setSourceDataSets(feed.getSourceDataSets().stream().peek(dataSet -> {
            com.thinkbiganalytics.metadata.api.catalog.DataSource dataSource = null;
            if (dataSet.getDataSource() != null && dataSet.getDataSource().getId() != null) {
                final com.thinkbiganalytics.metadata.api.catalog.DataSource.ID dataSourceId = dataSourceProvider.resolveId(dataSet.getDataSource().getId());
                dataSource = dataSourceProvider.find(dataSourceId).orElse(null);
            }
            // add empty datasource if user doesnt have access
            if (dataSource == null && !accessController.hasPermission(dataSource, DatasourceAccessControl.ACCESS_DATASOURCE, DatasourceAccessControl.ACCESS_DETAILS)) {
                dataSet.setDataSource(new com.thinkbiganalytics.kylo.catalog.rest.model.DataSource());
                dataSet.getDataSource().setAllowedActions(new ActionGroup());
            }
        }).collect(Collectors.toList()));
    }
}

Example 5 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class App method datasourceProvider.

/**
 * Creates a data source provider.
 *
 * @param datasourceProviderFactory the data source provider factory
 * @return the data source provider
 * @throws IOException if an I/O error occurs
 */
@Bean
public DatasourceProvider datasourceProvider(final DatasourceProviderFactory datasourceProviderFactory, @Named("decryptionModule") final Module decryptionModule) throws IOException {
    final List<Datasource> datasources;
    final String env = System.getenv("DATASOURCES");
    final List<DataSource> catalogDataSources;
    final String catalogDatasourcesEnv = System.getenv("CATALOG_DATASOURCES");
    if (env != null) {
        datasources = new ObjectMapper().registerModule(decryptionModule).readValue(env, TypeFactory.defaultInstance().constructCollectionType(List.class, Datasource.class));
    } else {
        datasources = Collections.emptyList();
    }
    if (catalogDatasourcesEnv != null) {
        catalogDataSources = new ObjectMapper().registerModule(decryptionModule).readValue(catalogDatasourcesEnv, TypeFactory.defaultInstance().constructCollectionType(List.class, DataSource.class));
    } else {
        catalogDataSources = Collections.emptyList();
    }
    return datasourceProviderFactory.getDatasourceProvider(datasources, catalogDataSources);
}