Examples with DataSetTemplate com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate used on opensource projects

Example 1 with DataSetTemplate

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate in project kylo by Teradata.

the class SparkShellProxyController method addDataSourceInformation.

private DataSet addDataSourceInformation(@Nonnull DataSet dataSet) {
    DataSet fetchedDataSet = fetchDataSet(dataSet.getId());
    DataSetTemplate template = DataSetUtil.mergeTemplates(fetchedDataSet);
    fetchedDataSet.getDataSource().setTemplate(template);
    return new DataSet(fetchedDataSet);
}

Example 2 with DataSetTemplate

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate in project kylo by Teradata.

the class AbstractKyloCatalogClient method write.

@Nonnull
@Override
public KyloCatalogWriter<T> write(@Nonnull final T source, @Nonnull final String targetId) {
    final DataSetTemplate dataSet = (dataSets != null) ? dataSets.get(targetId) : null;
    if (dataSet != null) {
        final DefaultKyloCatalogWriter<T> writer = new DefaultKyloCatalogWriter<>(this, hadoopConfiguration, resourceLoader, source);
        writer.dataSet(dataSet);
        return writer;
    } else {
        throw new KyloCatalogException("Data set does not exist: " + targetId);
    }
}

Example 3 with DataSetTemplate

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate in project kylo by Teradata.

the class AbstractCatalogDataSetProvider method readDataSet.

public T readDataSet(@Nonnull final DataSet dataSet) {
    final DataSetTemplate dataSetTemplate = mergeTemplates(dataSet);
    KyloCatalogReader<T> reader = getClient().read().options(dataSetTemplate.getOptions()).addJars(dataSetTemplate.getJars()).addFiles(dataSetTemplate.getFiles()).format(dataSetTemplate.getFormat());
    T dataFrame;
    if (dataSet.getPaths() != null && !dataSet.getPaths().isEmpty()) {
        if (dataSet.getPaths().size() > 1) {
            dataFrame = reader.load(dataSet.getPaths().toArray(new String[0]));
        } else {
            dataFrame = reader.load(dataSet.getPaths().get(0));
        }
    } else {
        dataFrame = reader.load();
    }
    return dataFrame;
}

Example 4 with DataSetTemplate

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate in project kylo by Teradata.

the class DefaultCatalogTableManager method getTableNames.

@Nonnull
@Override
public List<String> getTableNames(@Nonnull final DataSource dataSource, @Nullable final String schemaName, @Nullable final String tableName) throws SQLException {
    final DataSetTemplate template = DataSourceUtil.mergeTemplates(dataSource);
    return isolatedFunction(template, schemaName, (connection, schemaParser) -> {
        final javax.sql.DataSource ds = new SingleConnectionDataSource(connection, true);
        final DBSchemaParser tableSchemaParser = new DBSchemaParser(ds, new KerberosTicketConfiguration());
        return tableSchemaParser.listTables(schemaName, tableName);
    });
}

Example 5 with DataSetTemplate

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate in project kylo by Teradata.

the class DefaultCatalogTableManager method describeTable.

@Nonnull
@Override
public CatalogTableSchema describeTable(@Nonnull final DataSource dataSource, @Nullable final String schemaName, @Nullable final String tableName) throws SQLException {
    final DataSetTemplate template = DataSourceUtil.mergeTemplates(dataSource);
    if (Objects.equals("hive", template.getFormat())) {
        final TableSchema tableSchema = hiveMetastoreService.getTable(schemaName, tableName);
        final CatalogTableSchema catalogTableSchema = new CatalogTableSchema(tableSchema);
        // Get table metadata
        if (StringUtils.isNotEmpty(tableSchema.getName())) {
            final DefaultJdbcTable jdbcTable = new DefaultJdbcTable(tableSchema.getName(), "TABLE");
            jdbcTable.setCatalog(tableSchema.getDatabaseName());
            jdbcTable.setCatalog(tableSchema.getDatabaseName());
            jdbcTable.setRemarks(tableSchema.getDescription());
            jdbcTable.setSchema(tableSchema.getSchemaName());
            jdbcTable.setCatalogSeparator(".");
            jdbcTable.setIdentifierQuoteString("`");
            catalogTableSchema.setTable(createTable(jdbcTable));
        }
        return catalogTableSchema;
    } else if (Objects.equals("jdbc", template.getFormat())) {
        return isolatedFunction(template, schemaName, (connection, schemaParser) -> {
            final javax.sql.DataSource ds = new SingleConnectionDataSource(connection, true);
            final DBSchemaParser tableSchemaParser = new DBSchemaParser(ds, new KerberosTicketConfiguration());
            final TableSchema tableSchema = tableSchemaParser.describeTable(schemaName, tableName);
            if (tableSchema != null) {
                // Get table metadata
                final DefaultJdbcTable jdbcTable = new DefaultJdbcTable(tableSchema.getName(), "TABLE");
                jdbcTable.setCatalog(tableSchema.getDatabaseName());
                jdbcTable.setCatalog(tableSchema.getDatabaseName());
                jdbcTable.setRemarks(tableSchema.getDescription());
                jdbcTable.setSchema(tableSchema.getSchemaName());
                jdbcTable.setMetaData(connection.getMetaData());
                // Return table schema
                final CatalogTableSchema catalogTableSchema = new CatalogTableSchema(tableSchema);
                catalogTableSchema.setTable(createTable(jdbcTable));
                return catalogTableSchema;
            } else {
                return null;
            }
        });
    } else {
        throw new IllegalArgumentException("Unsupported format: " + template.getFormat());
    }
}