Examples with Datasource com.thinkbiganalytics.metadata.rest.model.data.Datasource used on opensource projects

Example 1 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class ExecuteSparkJob method getDatasources.

private Map<String, String> getDatasources(ProcessSession session, FlowFile flowFile, String PROVENANCE_JOB_STATUS_KEY, String datasourceIds, MetadataProviderService metadataService, List<String> extraJarPaths) throws JsonProcessingException {
    final Map<String, String> env = new HashMap<>();
    if (StringUtils.isNotBlank(datasourceIds)) {
        final StringBuilder datasources = new StringBuilder(10240);
        final ObjectMapper objectMapper = new ObjectMapper();
        final MetadataProvider provider = metadataService.getProvider();
        for (final String id : datasourceIds.split(",")) {
            datasources.append((datasources.length() == 0) ? '[' : ',');
            final Optional<Datasource> datasource = provider.getDatasource(id);
            if (datasource.isPresent()) {
                if (datasource.get() instanceof JdbcDatasource && StringUtils.isNotBlank(((JdbcDatasource) datasource.get()).getDatabaseDriverLocation())) {
                    final String[] databaseDriverLocations = ((JdbcDatasource) datasource.get()).getDatabaseDriverLocation().split(",");
                    extraJarPaths.addAll(Arrays.asList(databaseDriverLocations));
                }
                datasources.append(objectMapper.writeValueAsString(datasource.get()));
            } else {
                getLog().error("Required datasource {} is missing for Spark job: {}", new Object[] { id, flowFile });
                flowFile = session.putAttribute(flowFile, PROVENANCE_JOB_STATUS_KEY, "Invalid data source: " + id);
                session.transfer(flowFile, REL_FAILURE);
                return null;
            }
        }
        datasources.append(']');
        env.put("DATASOURCES", datasources.toString());
    }
    return env;
}

Example 2 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class BeginFeed method setupSource.

protected void setupSource(ProcessContext context, Feed feed, String datasourceName) {
    MetadataProvider provider = getProviderService(context).getProvider();
    Datasource datasource = getSourceDatasource(context, datasourceName);
    if (datasource != null) {
        getLog().debug("ensuring feed source - feed: {} datasource: {}", new Object[] { feed.getId(), datasource.getId() });
        provider.ensureFeedSource(feed.getId(), datasource.getId());
    // this.sourceDatasources.add(datasource);
    } else {
        throw new ProcessException("Source datasource does not exist: " + datasourceName);
    }
}

Example 3 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class AbstractTerminateFeed method ensureDestinationDatasource.

protected Datasource ensureDestinationDatasource(ProcessContext context) {
    String datasetName = context.getProperty(DEST_DATASET_NAME).getValue();
    Datasource dataset = findDatasource(context, datasetName);
    if (dataset != null) {
        return dataset;
    } else {
        getLog().info("No destinationDatasource exists with the givn name, creating: " + datasetName);
        return createDestinationDatasource(context, datasetName, "");
    }
}

Example 4 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class MetadataClientProvider method getDatasourceByName.

/* (non-Javadoc)
     * @see com.thinkbiganalytics.controller.metadata.MetadataProvider#getDatasourceByName(java.lang.String)
     */
@Override
public Datasource getDatasourceByName(String dsName) {
    DatasourceCriteria criteria = this.client.datasourceCriteria().name(dsName);
    List<Datasource> list = this.client.getDatasources(criteria);
    if (list.isEmpty()) {
        return null;
    } else {
        return list.get(0);
    }
}

Example 5 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class MetadataClientProviderTest method testBeginOperation.

@Test
public void testBeginOperation() {
    Feed feed = this.provider.ensureFeed("category", "test9", "");
    Datasource ds = this.provider.ensureDirectoryDatasource("test9", "", Paths.get("aaa", "bbb"));
    feed = this.provider.ensureFeedDestination(feed.getId(), ds.getId());
    FeedDestination dest = feed.getDestination(ds.getId());
    DataOperation op = this.provider.beginOperation(dest, new DateTime());
    assertThat(op).isNotNull();
    assertThat(op.getState()).isEqualTo(State.IN_PROGRESS);
}