Examples with DataSetSparkParameters com.thinkbiganalytics.metadata.api.catalog.DataSetSparkParameters used on opensource projects

Example 1 with DataSetSparkParameters

use of com.thinkbiganalytics.metadata.api.catalog.DataSetSparkParameters in project kylo by Teradata.

the class DefaultDataSourcesUpgradeAction method upgradeTo.

@Override
public void upgradeTo(final KyloVersion targetVersion) {
    log.info("Creating default catalog data sources: {}", targetVersion);
    Optional<ConnectorPlugin> plugin = pluginManager.getPlugin("hive");
    if (plugin.isPresent()) {
        Optional<Connector> connector = connectorProvider.findByPlugin(plugin.get().getId());
        if (connector.isPresent()) {
            List<DataSource> hiveSources = dataSourceProvider.findByConnector(connector.get().getId());
            // If at least one Hive data source exists then do nothing.
            if (hiveSources.size() == 0) {
                log.info("Creating default Hive data source");
                DataSource ds = dataSourceProvider.create(connector.get().getId(), "Hive");
                ds.setDescription("The default Hive data source");
                DataSetSparkParameters params = ds.getSparkParameters();
                params.setFormat("hive");
                params.addOption("driver", this.driver);
                params.addOption("url", this.url);
                params.addOption("user", this.user);
                params.addOption("password", this.password);
            } else {
                log.info("One or more Hive data sources already found: {}", hiveSources.stream().map(DataSource::toString).collect(Collectors.toList()));
            }
        } else {
            log.warn("No Hive connector found - cannot create a default Hive data source");
        }
    } else {
        log.warn("No Hive connector plugin found - cannot create a default Hive data source");
    }
}

Example 2 with DataSetSparkParameters

use of com.thinkbiganalytics.metadata.api.catalog.DataSetSparkParameters in project kylo by Teradata.

the class DatasourceModelTransform method getDerivedDataSourceType.

private String getDerivedDataSourceType(DataSet domainDataSet) {
    DataSetSparkParameters params = domainDataSet.getEffectiveSparkParameters();
    String datasourceType = params.getFormat();
    String derivedDatasourceType = "DatabaseDatasource";
    if (DATA_SET_SRC_TYPES.containsKey(datasourceType)) {
        derivedDatasourceType = DATA_SET_SRC_TYPES.get(datasourceType);
    } else if (domainDataSet.getDataSource().getConnector() != null) {
        datasourceType = domainDataSet.getDataSource().getConnector().getPluginId();
        derivedDatasourceType = DATA_SET_SRC_TYPES.getOrDefault(datasourceType, derivedDatasourceType);
    }
    return derivedDatasourceType;
}

Example 3 with DataSetSparkParameters

use of com.thinkbiganalytics.metadata.api.catalog.DataSetSparkParameters in project kylo by Teradata.

the class JcrDataSetProvider method generateTitle.

private String generateTitle(DataSource dataSource, String title, String format, Map<String, String> options, Set<String> paths) {
    String derivedTile = title;
    DataSetSparkParameters srcParams = dataSource.getEffectiveSparkParameters();
    String effectiveFormat = StringUtils.isEmpty(format) ? srcParams.getFormat() : format;
    if (StringUtils.isEmpty(title)) {
        if (effectiveFormat.equals("jdbc")) {
            derivedTile = Stream.concat(srcParams.getOptions().entrySet().stream(), options.entrySet().stream()).filter(e -> e.getKey().equals("dbtable")).map(Entry::getValue).findFirst().orElse(null);
        } else {
            derivedTile = Stream.concat(srcParams.getPaths().stream(), paths.stream()).collect(Collectors.joining(","));
        }
    }
    return StringUtils.isEmpty(derivedTile) ? dataSource.getSystemName() + "-" + UUID.randomUUID() : derivedTile;
}

Example 4 with DataSetSparkParameters

use of com.thinkbiganalytics.metadata.api.catalog.DataSetSparkParameters in project kylo by Teradata.

the class JcrDataSet method generateHashCode.

public long generateHashCode() {
    DataSetSparkParameters params = getEffectiveSparkParameters();
    // long hash = JcrDataSetProvider.generateHashCode(params.getFormat(), params.getPaths(), params.getJars(), params.getFiles(), params.getOptions());
    long hash = JcrDataSetProvider.generateHashCode(params.getFormat(), params.getPaths(), params.getOptions());
    setParamsHash(hash);
    return hash;
}

Example 5 with DataSetSparkParameters

use of com.thinkbiganalytics.metadata.api.catalog.DataSetSparkParameters in project kylo by Teradata.

the class JcrDataSetProvider method isJdbcSource.

private boolean isJdbcSource(DataSource dataSource, String format) {
    DataSetSparkParameters srcParams = dataSource.getEffectiveSparkParameters();
    String effectiveFormat = StringUtils.isEmpty(format) ? srcParams.getFormat() : format;
    return effectiveFormat.equals("jdbc");
}