Examples with DerivedDatasource com.thinkbiganalytics.metadata.rest.model.data.DerivedDatasource used on opensource projects

Example 1 with DerivedDatasource

use of com.thinkbiganalytics.metadata.rest.model.data.DerivedDatasource in project kylo by Teradata.

the class DatasourceModelTransform method toDatasource.

/**
 * Transforms the specified domain object to a REST object.
 *
 * @param domain the domain object
 * @param level  the level of detail
 * @return the REST object
 * @throws IllegalArgumentException if the domain object cannot be converted
 */
public Datasource toDatasource(@Nonnull final com.thinkbiganalytics.metadata.api.datasource.Datasource domain, @Nonnull final Level level) {
    if (domain instanceof com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource) {
        final DerivedDatasource ds = new DerivedDatasource();
        updateDatasource(ds, (com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource) domain, level);
        return ds;
    } else if (domain instanceof com.thinkbiganalytics.metadata.api.datasource.UserDatasource) {
        return toDatasource((com.thinkbiganalytics.metadata.api.datasource.UserDatasource) domain, level);
    } else {
        throw new IllegalArgumentException("Not a supported datasource class: " + domain.getClass());
    }
}

Example 2 with DerivedDatasource

use of com.thinkbiganalytics.metadata.rest.model.data.DerivedDatasource in project kylo by Teradata.

the class DatasourceModelTransform method toDatasource.

/**
 * Transforms the specified domain data set to an equivalent legacy REST Datasource.
 *
 * @param domain the domain data set object
 * @param level  the level of detail
 * @return the REST object
 * @throws IllegalArgumentException if the domain object cannot be converted
 */
public Datasource toDatasource(@Nonnull final DataSet domainDataSet, @Nonnull final Level level) {
    String derivedDatasourceType = getDerivedDataSourceType(domainDataSet);
    String derivedDataSourceName = domainDataSet.getTitle();
    com.thinkbiganalytics.metadata.api.datasource.Datasource derivedDatasource = findDomainDatasource(domainDataSet);
    DerivedDatasource ds = null;
    if (derivedDatasource != null) {
        ds = (DerivedDatasource) toDatasource(derivedDatasource, Level.BASIC);
    } else {
        ds = new DerivedDatasource();
        ds.setId(domainDataSet.getId().toString());
        ds.setName(derivedDataSourceName);
        ds.setDatasourceType(derivedDatasourceType);
    }
    return ds;
}

Example 3 with DerivedDatasource

use of com.thinkbiganalytics.metadata.rest.model.data.DerivedDatasource in project kylo by Teradata.

the class DatasourceModelTransform method findDomainDatasource.

public com.thinkbiganalytics.metadata.api.datasource.Datasource findDomainDatasource(DataSet domainDataSet) {
    DataSetSparkParameters params = domainDataSet.getEffectiveSparkParameters();
    String derivedDatasourceType = getDerivedDataSourceType(domainDataSet);
    String derivedDataSourceName = domainDataSet.getTitle();
    if (StringUtils.isBlank(derivedDataSourceName) && params.getPaths() != null && !params.getPaths().isEmpty()) {
        derivedDataSourceName = params.getPaths().stream().collect(Collectors.joining(","));
    }
    DerivedDatasource ds = null;
    com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource derivedDatasource = datasourceProvider.findDerivedDatasource(derivedDatasourceType, derivedDataSourceName);
    return derivedDatasource;
}

Example 4 with DerivedDatasource

use of com.thinkbiganalytics.metadata.rest.model.data.DerivedDatasource in project kylo by Teradata.

the class FeedLineageBuilder method build.

private com.thinkbiganalytics.metadata.rest.model.feed.Feed build(Feed domainFeed) {
    com.thinkbiganalytics.metadata.rest.model.feed.Feed feed = restFeeds.containsKey(domainFeed.getId().toString()) ? restFeeds.get(domainFeed.getId().toString()) : model.domainToFeed(domainFeed);
    restFeeds.put(feed.getId(), feed);
    @SuppressWarnings("unchecked") List<? extends com.thinkbiganalytics.metadata.api.feed.FeedSource> sources = domainFeed.getSources();
    Set<FeedSource> feedSources = new HashSet<FeedSource>();
    if (sources != null) {
        boolean containsDerivedDatasource = false;
        for (com.thinkbiganalytics.metadata.api.feed.FeedSource feedSource : sources) {
            FeedSource src = new FeedSource();
            feedSource.getDatasource().ifPresent(datasource -> src.setDatasource(buildDatasource(datasource)));
            // only add datasets if the source is missing or if its not a derived datasource
            if (src.getDatasource() != null && src.getDatasource() instanceof DerivedDatasource) {
                containsDerivedDatasource = true;
            }
            if (!containsDerivedDatasource) {
                feedSource.getDataSet().ifPresent(dataSet -> src.setDatasource(buildDatasource(dataSet)));
            }
            feedSources.add(src);
        }
    }
    feed.setSources(feedSources);
    Set<FeedDestination> feedDestinations = new HashSet<FeedDestination>();
    List<? extends com.thinkbiganalytics.metadata.api.feed.FeedDestination> destinations = domainFeed.getDestinations();
    if (destinations != null) {
        destinations.stream().forEach(feedDestination -> {
            FeedDestination dest = new FeedDestination();
            feedDestination.getDatasource().ifPresent(datasource -> dest.setDatasource(buildDatasource(datasource)));
            feedDestination.getDataSet().ifPresent(dataSet -> dest.setDatasource(buildDatasource(dataSet)));
            feedDestinations.add(dest);
        });
    }
    feed.setDestinations(feedDestinations);
    if (domainFeed.getDependentFeeds() != null) {
        List<Feed> depFeeds = domainFeed.getDependentFeeds();
        depFeeds.stream().forEach(depFeed -> {
            com.thinkbiganalytics.metadata.rest.model.feed.Feed restFeed = restFeeds.get(depFeed.getId().toString());
            if (restFeed == null) {
                com.thinkbiganalytics.metadata.rest.model.feed.Feed depRestFeed = model.domainToFeed(depFeed);
                restFeeds.put(depRestFeed.getId(), depRestFeed);
                feed.getDependentFeeds().add(depRestFeed);
                build(depFeed);
            } else {
                feed.getDependentFeeds().add(restFeed);
            }
        });
    }
    if (domainFeed.getUsedByFeeds() != null) {
        List<Feed> usedByFeeds = domainFeed.getUsedByFeeds();
        usedByFeeds.stream().forEach(usedByFeed -> {
            com.thinkbiganalytics.metadata.rest.model.feed.Feed restFeed = restFeeds.get(usedByFeed.getId().toString());
            if (restFeed == null) {
                com.thinkbiganalytics.metadata.rest.model.feed.Feed usedByRestFeed = model.domainToFeed(usedByFeed);
                restFeeds.put(usedByRestFeed.getId(), usedByRestFeed);
                feed.getUsedByFeeds().add(usedByRestFeed);
                build(usedByFeed);
            } else {
                feed.getUsedByFeeds().add(restFeed);
            }
        });
    }
    return feed;
}