Examples with DefaultDataSetTemplate com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate used on opensource projects

Example 6 with DefaultDataSetTemplate

use of com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate in project kylo by Teradata.

the class DataSourceController method createJdbcTableDataSet.

/**
 * Gets the schema of the specified table using the specified data source.
 *
 * @param dataSourceId the data source id
 * @param tableName    the table name
 * @param schema       the schema name, or {@code null} to search all schemas
 * @return the table and field details
 */
@POST
@Path("{id}/tables/{tableName}/dataset")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Gets the schema of the specified table.", notes = "Connects to the database specified by the data source.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the table schema.", response = DataSetWithTableSchema.class), @ApiResponse(code = 403, message = "Access denied.", response = RestResponseStatus.class), @ApiResponse(code = 404, message = "A JDBC data source with that id does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "NiFi or the database are unavailable.", response = RestResponseStatus.class) })
public Response createJdbcTableDataSet(@PathParam("id") final String dataSourceId, @PathParam("tableName") final String tableName, @QueryParam("schema") final String schema) {
    // TODO Verify user has access to data source
    // Require admin permission if the results should include unencrypted credentials.
    final boolean encryptCredentials = true;
    accessController.checkPermission(AccessController.SERVICES, encryptCredentials ? FeedServicesAccessControl.ACCESS_DATASOURCES : FeedServicesAccessControl.ADMIN_DATASOURCES);
    DataSetWithTableSchema dataSetWithTableSchema = null;
    boolean hasAccess = false;
    try {
        // ensure the user can read teh datasource
        DataSource encryptedSource = metadataService.read(() -> {
            return findDataSource(dataSourceId, true);
        });
        hasAccess = encryptedSource != null;
        if (hasAccess) {
            // fetch the datasource as a service account to get the creds.
            DataSource dataSource = metadataService.read(() -> {
                return findDataSource(dataSourceId, false);
            }, MetadataAccess.SERVICE);
            // describe the datasource and table
            CatalogTableSchema tableSchema = tableManager.describeTable(dataSource, schema, tableName);
            if (tableSchema != null) {
                DataSet dataSet = new DataSet();
                // assign the datasource to this dataset with encrypted credentials
                dataSet.setDataSource(encryptedSource);
                String fullTableName = (tableSchema.getTable() != null) ? tableSchema.getTable().getQualifiedIdentifier() : HiveUtils.quoteIdentifier(tableSchema.getSchemaName(), tableSchema.getName());
                dataSet.setTitle(tableSchema.getSchemaName() + "." + tableSchema.getName());
                DefaultDataSetTemplate defaultDataSetTemplate = DataSetUtil.mergeTemplates(dataSet);
                List<String> paths = defaultDataSetTemplate.getPaths();
                String format = defaultDataSetTemplate.getFormat();
                Map<String, String> options = defaultDataSetTemplate.getOptions();
                if (options == null) {
                    options = new HashMap<>();
                }
                if ("hive".equalsIgnoreCase(format.toLowerCase())) {
                    if (paths == null) {
                        paths = new ArrayList<>();
                    }
                    paths.add(fullTableName);
                }
                options.put("dbtable", fullTableName);
                dataSet.setFormat(format);
                dataSet.setPaths(paths);
                dataSet.setOptions(options);
                DataSet dataSet1 = dataSetService.findOrCreateDataSet(dataSet, encryptCredentials);
                dataSetWithTableSchema = new DataSetWithTableSchema(dataSet1, tableSchema);
            } else {
                if (log.isErrorEnabled()) {
                    log.error("Failed to describe tables for schema [" + schema + "], table [" + tableName + "], dataSource [" + dataSourceId + "] ");
                }
                final RestResponseStatus status = new RestResponseStatus.ResponseStatusBuilder().message(getMessage("catalog.datasource.describeTable.error", tableName, schema)).url(request.getRequestURI()).buildError();
                throw new InternalServerErrorException(Response.serverError().entity(status).build());
            }
        // });
        } else {
            // no acceess
            final RestResponseStatus status = new RestResponseStatus.ResponseStatusBuilder().message(getMessage("catalog.datasource.forbidden")).url(request.getRequestURI()).buildError();
            throw new InternalServerErrorException(Response.serverError().entity(status).build());
        }
    } catch (Exception e) {
        if (exceptionTransformer.causesInChain(e)) {
            throw new ThriftConnectionException(e);
        }
        final RestResponseStatus status = new RestResponseStatus.ResponseStatusBuilder().message(getMessage("catalog.datasource.describeTable.error", tableName, schema)).url(request.getRequestURI()).buildError();
        throw new InternalServerErrorException(Response.serverError().entity(status).build());
    }
    return Response.ok(dataSetWithTableSchema).build();
}

Example 7 with DefaultDataSetTemplate

use of com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate in project kylo by Teradata.

the class DataSourceProvider method toDataSource.

/**
 * Converts a feed data source to a REST data source.
 */
@Nullable
@SuppressWarnings("squid:S3655")
private DataSource toDataSource(@Nullable final Datasource feedDataSource, @Nonnull final DatasourceModelTransform.Level level) {
    // Transform to metadata data source
    final com.thinkbiganalytics.metadata.rest.model.data.Datasource metadataDataSource;
    if (feedDataSource != null) {
        metadataDataSource = feedDataSourceTransform.toDatasource(feedDataSource, level);
    } else {
        return null;
    }
    // Add properties to data source
    final DataSource dataSource = new DataSource();
    dataSource.setId(metadataDataSource.getId());
    dataSource.setTitle(metadataDataSource.getName());
    // Set properties based on type
    final Connector connector = new Connector();
    final DefaultDataSetTemplate template;
    if (metadataDataSource instanceof JdbcDatasource && getJdbcConnectorId().isPresent()) {
        connector.setId(getJdbcConnectorId().get());
        template = createTemplate((JdbcDatasource) metadataDataSource);
    } else {
        return null;
    }
    dataSource.setConnector(connector);
    dataSource.setTemplate(template);
    return dataSource;
}

Example 8 with DefaultDataSetTemplate

use of com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate in project kylo by Teradata.

the class DataSourceProvider method createTemplate.

/**
 * Creates a data set template for the specified JDBC data source.
 */
@Nullable
private DefaultDataSetTemplate createTemplate(@Nonnull final JdbcDatasource jdbcDatasource) {
    final DefaultDataSetTemplate template = new DefaultDataSetTemplate();
    template.setOptions(new HashMap<>());
    if (jdbcDatasource.getDatabaseConnectionUrl() != null) {
        template.getOptions().put("url", jdbcDatasource.getDatabaseConnectionUrl());
    }
    if (jdbcDatasource.getDatabaseDriverClassName() != null) {
        template.getOptions().put("driver", jdbcDatasource.getDatabaseDriverClassName());
    }
    if (jdbcDatasource.getDatabaseDriverLocation() != null) {
        template.setJars(Arrays.asList(jdbcDatasource.getDatabaseDriverLocation().split(",")));
    }
    if (jdbcDatasource.getDatabaseUser() != null) {
        template.getOptions().put("user", jdbcDatasource.getDatabaseUser());
    }
    if (jdbcDatasource.getPassword() != null) {
        template.getOptions().put("password", jdbcDatasource.getPassword());
    }
    return (template.getJars() != null || !template.getOptions().isEmpty()) ? template : null;
}

Example 9 with DefaultDataSetTemplate

use of com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate in project kylo by Teradata.

the class DataSetUtil method mergeTemplates.

/**
 * Merges the data set, data source, and connector templates for the specified data set.
 */
@Nonnull
public static DefaultDataSetTemplate mergeTemplates(@Nonnull final DataSet dataSet) {
    final DefaultDataSetTemplate template;
    if (dataSet.getDataSource() != null) {
        template = DataSourceUtil.mergeTemplates(dataSet.getDataSource());
        mergeTemplates(template, dataSet);
    } else {
        template = new DefaultDataSetTemplate(dataSet);
    }
    return template;
}

Example 10 with DefaultDataSetTemplate

use of com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate in project kylo by Teradata.

the class DataSourceIT method testListFilesS3.

/**
 * Verify listing files from the Amazon S3 connector.
 */
@Test
public void testListFilesS3() {
    Assume.assumeNotNull(awsAccessKeyId, awsSecretAccessKey);
    // Create an S3 data source
    final Connector connector = new Connector();
    connector.setId("amazon-s3");
    final DefaultDataSetTemplate template = new DefaultDataSetTemplate();
    template.setOptions(new HashMap<>());
    template.getOptions().put("spark.hadoop.fs.s3a.access.key", awsAccessKeyId);
    template.getOptions().put("spark.hadoop.fs.s3a.secret.key", awsSecretAccessKey);
    final DataSource request = new DataSource();
    request.setConnector(connector);
    request.setTemplate(template);
    request.setTitle("test list files s3");
    final DataSource dataSource = given(DataSourceController.BASE).when().body(request).post().then().statusCode(200).extract().as(DataSource.class);
    // Test listing buckets
    final List<DataSetFile> buckets = given(DataSourceController.BASE).when().pathParam("id", dataSource.getId()).queryParam("path", "s3a:/").get("{id}/files").then().statusCode(200).extract().as(DataSetFileList.class);
    Assert.assertThat(buckets, CoreMatchers.hasItem(new CustomMatcher<DataSetFile>("DataSetFile name=thinkbig.greg directory=true") {

        @Override
        public boolean matches(final Object item) {
            return (item instanceof DataSetFile) && Objects.equals("thinkbig.greg", ((DataSetFile) item).getName()) && Objects.equals("s3a://thinkbig.greg/", ((DataSetFile) item).getPath()) && ((DataSetFile) item).isDirectory();
        }
    }));
    // Test listing files
    final List<DataSetFile> files = given(DataSourceController.BASE).when().pathParam("id", dataSource.getId()).queryParam("path", "s3a://thinkbig.greg/").get("{id}/files").then().statusCode(200).extract().as(DataSetFileList.class);
    Assert.assertThat(files, CoreMatchers.hasItem(new CustomMatcher<DataSetFile>("DataSetFile name=userdata1.csv directory=false") {

        @Override
        public boolean matches(Object item) {
            return (item instanceof DataSetFile) && Objects.equals("userdata1.csv", ((DataSetFile) item).getName()) && Objects.equals("s3a://thinkbig.greg/userdata1.csv", ((DataSetFile) item).getPath()) && !((DataSetFile) item).isDirectory();
        }
    }));
}