Examples with DataSource com.thinkbiganalytics.kylo.catalog.rest.model.DataSource used on opensource projects

Example 6 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class NifiIntegrationRestController method getTableNames.

@GET
@Path("/controller-services/{serviceId}/tables")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Gets a list of table names from the specified database.", notes = "Connects to the database specified by the controller service using the password defined in Kylo's application.properties file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the table names.", response = String.class, responseContainer = "List"), @ApiResponse(code = 500, message = "Nifi or the database are unavailable.", response = RestResponseStatus.class) })
public Response getTableNames(@PathParam("serviceId") String serviceId, @QueryParam("serviceName") @DefaultValue("") String serviceName, @QueryParam("schema") String schema, @QueryParam("tableName") String tableName) {
    final Optional<DataSource> dataSource = findDataSourceForControllerServiceId(serviceId);
    final List<String> tables;
    if (dataSource.isPresent()) {
        log.info("Query for Table Names against data source: {}", dataSource.get().getId());
        try {
            tables = catalogTableManager.getTableNames(dataSource.get(), schema, tableName);
        } catch (final SQLException | InvalidControllerServiceLookupException e) {
            log.error("Unable to list table names for data source: {}", dataSource.get().getId(), e);
            throw new InternalServerErrorException("Unable to connect to the data source", e);
        }
    } else {
        log.info("Query for Table Names against service: {}({})", serviceName, serviceId);
        try {
            tables = dbcpConnectionPoolTableInfo.getTableNamesForControllerService(serviceId, serviceName, schema, tableName);
        } catch (final InvalidControllerServiceLookupException e) {
            log.error("Unable to list table names for controller service: {}", serviceName, e);
            throw new InternalServerErrorException("Unable to connect to the data source", e);
        }
    }
    return Response.ok(tables).build();
}

Example 7 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class NifiIntegrationRestController method describeTable.

@GET
@Path("/controller-services/{serviceId}/tables/{tableName}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Gets the schema of the specified table.", notes = "Connects to the database specified by the controller service using the password defined in Kylo's application.properties file.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the table schema.", response = TableSchema.class), @ApiResponse(code = 500, message = "Nifi or the database are unavailable.", response = RestResponseStatus.class) })
public Response describeTable(@PathParam("serviceId") String serviceId, @PathParam("tableName") String tableName, @QueryParam("serviceName") @DefaultValue("") String serviceName, @QueryParam("schema") String schema) {
    final Optional<DataSource> dataSource = findDataSourceForControllerServiceId(serviceId);
    final TableSchema tableSchema;
    if (dataSource.isPresent()) {
        log.info("Describe Table {} against data source: {}", tableName, dataSource);
        try {
            tableSchema = catalogTableManager.describeTable(dataSource.get(), schema, tableName);
        } catch (final SQLException e) {
            log.error("Unable to list table names for data source: {}", dataSource.get().getId(), e);
            throw new InternalServerErrorException("Unable to connect to data source. " + e.getMessage(), e);
        }
    } else {
        log.info("Describe Table {} against service: {}({})", tableName, serviceName, serviceId);
        tableSchema = dbcpConnectionPoolTableInfo.describeTableForControllerService(serviceId, serviceName, schema, tableName);
    }
    return Response.ok(tableSchema).build();
}

Example 8 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class AbstractCatalogDataSetProvider method mergeTemplates.

/**
 * Merges the data set, data source, and connector templates for the specified data set.
 */
@Nonnull
protected DefaultDataSetTemplate mergeTemplates(@Nonnull final DataSet dataSet) {
    final DefaultDataSetTemplate template;
    if (dataSet.getDataSource() != null) {
        final DataSource dataSource = dataSet.getDataSource();
        template = new DefaultDataSetTemplate();
        if (dataSource.getConnector() != null && dataSource.getConnector().getTemplate() != null) {
            mergeTemplates(template, dataSource.getConnector().getTemplate());
        }
        if (dataSource.getTemplate() != null) {
            mergeTemplates(template, dataSource.getTemplate());
        }
        mergeTemplates(template, dataSet);
    } else {
        template = new DefaultDataSetTemplate(dataSet);
    }
    return template;
}

Example 9 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class DefaultCatalogTableManager method describeTable.

@Nonnull
@Override
public CatalogTableSchema describeTable(@Nonnull final DataSource dataSource, @Nullable final String schemaName, @Nullable final String tableName) throws SQLException {
    final DataSetTemplate template = DataSourceUtil.mergeTemplates(dataSource);
    if (Objects.equals("hive", template.getFormat())) {
        final TableSchema tableSchema = hiveMetastoreService.getTable(schemaName, tableName);
        final CatalogTableSchema catalogTableSchema = new CatalogTableSchema(tableSchema);
        // Get table metadata
        if (StringUtils.isNotEmpty(tableSchema.getName())) {
            final DefaultJdbcTable jdbcTable = new DefaultJdbcTable(tableSchema.getName(), "TABLE");
            jdbcTable.setCatalog(tableSchema.getDatabaseName());
            jdbcTable.setCatalog(tableSchema.getDatabaseName());
            jdbcTable.setRemarks(tableSchema.getDescription());
            jdbcTable.setSchema(tableSchema.getSchemaName());
            jdbcTable.setCatalogSeparator(".");
            jdbcTable.setIdentifierQuoteString("`");
            catalogTableSchema.setTable(createTable(jdbcTable));
        }
        return catalogTableSchema;
    } else if (Objects.equals("jdbc", template.getFormat())) {
        return isolatedFunction(template, schemaName, (connection, schemaParser) -> {
            final javax.sql.DataSource ds = new SingleConnectionDataSource(connection, true);
            final DBSchemaParser tableSchemaParser = new DBSchemaParser(ds, new KerberosTicketConfiguration());
            final TableSchema tableSchema = tableSchemaParser.describeTable(schemaName, tableName);
            if (tableSchema != null) {
                // Get table metadata
                final DefaultJdbcTable jdbcTable = new DefaultJdbcTable(tableSchema.getName(), "TABLE");
                jdbcTable.setCatalog(tableSchema.getDatabaseName());
                jdbcTable.setCatalog(tableSchema.getDatabaseName());
                jdbcTable.setRemarks(tableSchema.getDescription());
                jdbcTable.setSchema(tableSchema.getSchemaName());
                jdbcTable.setMetaData(connection.getMetaData());
                // Return table schema
                final CatalogTableSchema catalogTableSchema = new CatalogTableSchema(tableSchema);
                catalogTableSchema.setTable(createTable(jdbcTable));
                return catalogTableSchema;
            } else {
                return null;
            }
        });
    } else {
        throw new IllegalArgumentException("Unsupported format: " + template.getFormat());
    }
}

Example 10 with DataSource

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSource in project kylo by Teradata.

the class DataSourceUtilTest method getPaths.

/**
 * Verify merging paths for a data source.
 */
@Test
public void getPaths() {
    // Create mock connector
    final DefaultDataSetTemplate connectorTemplate = new DefaultDataSetTemplate();
    connectorTemplate.setOptions(Collections.singletonMap("path", "connector1.txt"));
    connectorTemplate.setPaths(Collections.singletonList("connector2.txt"));
    final Connector connector = new Connector();
    connector.setTemplate(connectorTemplate);
    // Create mock data source
    final DefaultDataSetTemplate dataSourceTemplate = new DefaultDataSetTemplate();
    dataSourceTemplate.setOptions(Collections.singletonMap("path", "datasource1.txt"));
    dataSourceTemplate.setPaths(Collections.singletonList("datasource2.txt"));
    final DataSource dataSource = new DataSource();
    dataSource.setConnector(connector);
    dataSource.setTemplate(dataSourceTemplate);
    // Test retrieving data source paths
    Assert.assertEquals(Arrays.asList("datasource1.txt", "datasource2.txt"), DataSourceUtil.getPaths(dataSource).orElse(null));
    // Test retrieving connector paths
    dataSourceTemplate.setOptions(null);
    Assert.assertEquals(Arrays.asList("connector1.txt", "datasource2.txt"), DataSourceUtil.getPaths(dataSource).orElse(null));
    dataSourceTemplate.setPaths(null);
    Assert.assertEquals(Arrays.asList("connector1.txt", "connector2.txt"), DataSourceUtil.getPaths(dataSource).orElse(null));
    // Test retrieving empty paths
    connectorTemplate.setOptions(null);
    connectorTemplate.setPaths(null);
    Assert.assertEquals(Optional.empty(), DataSourceUtil.getPaths(dataSource));
}