Examples with Datasource com.thinkbiganalytics.metadata.rest.model.data.Datasource used on opensource projects

Example 16 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class MetadataClientProviderTest method testEnsureDirectoryDatasource.

@Test
public void testEnsureDirectoryDatasource() {
    this.provider.ensureDirectoryDatasource("test6", "", Paths.get("aaa", "bbb"));
    Datasource ds = this.provider.getDatasourceByName("test6");
    assertThat(ds).isNotNull();
    assertThat(ds).isInstanceOf(DirectoryDatasource.class);
    String dsId = ds.getId();
    DirectoryDatasource dds = (DirectoryDatasource) ds;
    assertThat(dds.getPath()).contains("aaa/bbb");
    ds = this.provider.ensureDirectoryDatasource("test6", "", Paths.get("aaa", "bbb"));
    assertThat(ds).isNotNull();
    assertThat(ds.getId()).isEqualTo(dsId);
}

Example 17 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class AbstractTerminateFeed method onTrigger.

/* (non-Javadoc)
     * @see org.apache.nifi.processor.AbstractProcessor#onTrigger(org.apache.nifi.processor.ProcessContext, org.apache.nifi.processor.ProcessSession)
     */
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    try {
        FlowFile flowFile = session.get();
        if (flowFile != null) {
            MetadataProvider provider = getProviderService(context).getProvider();
            // TODO Remove when we do more intelligent handling when the feed and datasource info has been
            // removed from the metadata store.
            // FeedDestination destination = this.feedDestination.get();
            Datasource ds = ensureDatasourceMetadata(context);
            FeedDestination destination = ensureFeedDestination(context, flowFile, ds);
            // TODO Begin re-caching this when the correct error can be surfaced from the client
            // when the destination no longer exists.
            // if (destination == null) {
            // Datasource ds = this.destinationDatasource.get();
            // this.feedDestination.compareAndSet(null, destination);
            // }
            String timeStr = flowFile.getAttribute(OPERATON_START_PROP);
            DateTime opStart = timeStr != null ? Formatters.parseDateTime(timeStr) : new DateTime();
            DataOperation op = provider.beginOperation(destination, opStart);
            op = completeOperation(context, flowFile, ds, op, getState(context, op));
            updateFeedState(context, flowFile);
            flowFile = session.putAttribute(flowFile, OPERATON_STOP_PROP, Formatters.print(new DateTime()));
            session.remove(flowFile);
        // session.transfer(flowFile, SUCCESS);
        } else {
            context.yield();
        }
    } catch (Exception e) {
        context.yield();
        session.rollback();
        getLog().error("Unexpected error processing feed completion", e);
        throw new ProcessException(e);
    }
}

Example 18 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class DefaultFeedExporter method exportFeed.

/**
 * Export a feed as a zip file
 *
 * @param feedId the id {@link Feed#getId()} of the feed to export
 * @return object containing the zip file with data about the feed.
 */
@Override
public ExportFeed exportFeed(String feedId) throws IOException {
    this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.EXPORT_FEEDS);
    this.metadataService.checkFeedPermission(feedId, FeedAccessControl.EXPORT);
    // Prepare feed metadata
    final FeedMetadata feed = metadataService.getFeedById(feedId);
    if (feed == null) {
        // feed will not be found when user is allowed to export feeds but has no entity access to feed with feed id
        throw new NotFoundException("Feed not found for id " + feedId);
    }
    final List<Datasource> userDatasources = Optional.ofNullable(feed.getDataTransformation()).map(FeedDataTransformation::getDatasourceIds).map(datasourceIds -> metadataAccess.read(() -> datasourceIds.stream().map(datasourceProvider::resolve).map(datasourceProvider::getDatasource).map(domain -> datasourceTransform.toDatasource(domain, DatasourceModelTransform.Level.FULL)).map(datasource -> {
        // Clear sensitive fields
        datasource.getDestinationForFeeds().clear();
        datasource.getSourceForFeeds().clear();
        return datasource;
    }).collect(Collectors.toList()))).orElse(null);
    if (userDatasources != null && !userDatasources.isEmpty()) {
        this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
        feed.setUserDatasources(userDatasources);
    }
    // Add feed json to template zip file
    final ExportTemplate exportTemplate = templateExporter.exportTemplateForFeedExport(feed.getTemplateId());
    final String feedJson = ObjectMapperSerializer.serialize(feed);
    final byte[] zipFile = ZipFileUtil.addToZip(exportTemplate.getFile(), feedJson, ImportFeed.FEED_JSON_FILE);
    return new ExportFeed(feed.getSystemFeedName() + ".feed.zip", zipFile);
}

Example 19 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class DBCPConnectionPoolService method evaluateWithUserDefinedDatasources.

private boolean evaluateWithUserDefinedDatasources(PoolingDataSourceService.DataSourceProperties dataSourceProperties, AbstractControllerServiceRequest serviceProperties) {
    boolean valid = (StringUtils.isNotBlank(dataSourceProperties.getPassword()) && !dataSourceProperties.getPassword().startsWith("**"));
    if (!valid) {
        List<Datasource> matchingDatasources = metadataAccess.read(() -> {
            // attempt to get the properties from the stored datatsource
            return datasetProvider.getDatasources(datasetProvider.datasetCriteria().type(com.thinkbiganalytics.metadata.api.datasource.UserDatasource.class).name(serviceProperties.getControllerServiceName())).stream().map(ds -> datasourceTransform.toDatasource(ds, DatasourceModelTransform.Level.ADMIN)).filter(datasource -> datasource instanceof JdbcDatasource).collect(Collectors.toList());
        }, MetadataAccess.SERVICE);
        if (matchingDatasources != null && !matchingDatasources.isEmpty()) {
            JdbcDatasource userDatasource = (JdbcDatasource) matchingDatasources.stream().filter(ds -> ((JdbcDatasource) ds).getDatabaseUser().equalsIgnoreCase(dataSourceProperties.getUser())).findFirst().orElse(null);
            if (userDatasource == null) {
                userDatasource = (JdbcDatasource) matchingDatasources.get(0);
            }
            if (userDatasource != null) {
                dataSourceProperties.setUser(userDatasource.getDatabaseUser());
                dataSourceProperties.setPassword(userDatasource.getPassword());
                log.info("Returned user defined datasource for {} service and user {} ", serviceProperties.getControllerServiceName(), userDatasource.getDatabaseUser());
                valid = true;
            }
        }
        if (!valid) {
            String propertyKey = nifiControllerServiceProperties.getEnvironmentControllerServicePropertyPrefix(serviceProperties.getControllerServiceName()) + ".password";
            String example = propertyKey + "=PASSWORD";
            log.error("Unable to connect to Controller Service {}, {}.  You need to specify a configuration property as {} with the password for user: {}. ", serviceProperties.getControllerServiceName(), serviceProperties.getControllerServiceId(), example, dataSourceProperties.getUser());
        }
    }
    return valid;
}

Example 20 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class FeedLineageBuilder method build.

private com.thinkbiganalytics.metadata.rest.model.feed.Feed build(Feed domainFeed) {
    com.thinkbiganalytics.metadata.rest.model.feed.Feed feed = restFeeds.containsKey(domainFeed.getId().toString()) ? restFeeds.get(domainFeed.getId().toString()) : model.domainToFeed(domainFeed);
    restFeeds.put(feed.getId(), feed);
    @SuppressWarnings("unchecked") List<? extends com.thinkbiganalytics.metadata.api.feed.FeedSource> sources = domainFeed.getSources();
    Set<FeedSource> feedSources = new HashSet<FeedSource>();
    if (sources != null) {
        sources.stream().forEach(feedSource -> {
            FeedSource src = new FeedSource();
            Datasource ds = buildDatasource(feedSource.getDatasource());
            src.setDatasource(ds);
            feedSources.add(src);
        });
    }
    feed.setSources(feedSources);
    Set<FeedDestination> feedDestinations = new HashSet<FeedDestination>();
    List<? extends com.thinkbiganalytics.metadata.api.feed.FeedDestination> destinations = domainFeed.getDestinations();
    if (destinations != null) {
        destinations.stream().forEach(feedDestination -> {
            FeedDestination dest = new FeedDestination();
            Datasource ds = buildDatasource(feedDestination.getDatasource());
            dest.setDatasource(ds);
            feedDestinations.add(dest);
        });
    }
    feed.setDestinations(feedDestinations);
    if (domainFeed.getDependentFeeds() != null) {
        List<Feed> depFeeds = domainFeed.getDependentFeeds();
        depFeeds.stream().forEach(depFeed -> {
            com.thinkbiganalytics.metadata.rest.model.feed.Feed restFeed = restFeeds.get(depFeed.getId().toString());
            if (restFeed == null) {
                com.thinkbiganalytics.metadata.rest.model.feed.Feed depRestFeed = model.domainToFeed(depFeed);
                restFeeds.put(depRestFeed.getId(), depRestFeed);
                feed.getDependentFeeds().add(depRestFeed);
                build(depFeed);
            } else {
                feed.getDependentFeeds().add(restFeed);
            }
        });
    }
    if (domainFeed.getUsedByFeeds() != null) {
        List<Feed> usedByFeeds = domainFeed.getUsedByFeeds();
        usedByFeeds.stream().forEach(usedByFeed -> {
            com.thinkbiganalytics.metadata.rest.model.feed.Feed restFeed = restFeeds.get(usedByFeed.getId().toString());
            if (restFeed == null) {
                com.thinkbiganalytics.metadata.rest.model.feed.Feed usedByRestFeed = model.domainToFeed(usedByFeed);
                restFeeds.put(usedByRestFeed.getId(), usedByRestFeed);
                feed.getUsedByFeeds().add(usedByRestFeed);
                build(usedByFeed);
            } else {
                feed.getUsedByFeeds().add(restFeed);
            }
        });
    }
    return feed;
}