Examples with DataSet com.thinkbiganalytics.kylo.catalog.rest.model.DataSet used on opensource projects

Example 11 with DataSet

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSet in project kylo by Teradata.

the class CatalogFileManagerTest method createDataSet.

/**
 * Creates a new data set.
 */
@Nonnull
private DataSet createDataSet() {
    final DefaultDataSetTemplate dataSourceTemplate = new DefaultDataSetTemplate();
    dataSourceTemplate.setPaths(Collections.singletonList(datasetsFolder.getRoot().toURI().toString()));
    final DataSource dataSource = new DataSource();
    dataSource.setId(UUID.randomUUID().toString());
    dataSource.setTemplate(dataSourceTemplate);
    final DataSet dataSet = new DataSet();
    dataSet.setId(UUID.randomUUID().toString());
    dataSet.setDataSource(dataSource);
    return dataSet;
}

Example 12 with DataSet

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSet in project kylo by Teradata.

the class DataSourceController method createDataSet.

@POST
@Path("{id}/dataset")
@ApiOperation("creates a new dataset for a datasource")
public Response createDataSet(@PathParam("id") final String datasourceId) {
    log.entry(datasourceId);
    final DataSource dataSource = findDataSource(datasourceId, true);
    final DataSet dataSet = new DataSet();
    dataSet.setDataSource(dataSource);
    return dataSetController.createDataSet(dataSet);
}

Example 13 with DataSet

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSet in project kylo by Teradata.

the class DataSourceController method createJdbcTableDataSet.

/**
 * Gets the schema of the specified table using the specified data source.
 *
 * @param dataSourceId the data source id
 * @param tableName    the table name
 * @param schema       the schema name, or {@code null} to search all schemas
 * @return the table and field details
 */
@POST
@Path("{id}/tables/{tableName}/dataset")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Gets the schema of the specified table.", notes = "Connects to the database specified by the data source.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the table schema.", response = DataSetWithTableSchema.class), @ApiResponse(code = 403, message = "Access denied.", response = RestResponseStatus.class), @ApiResponse(code = 404, message = "A JDBC data source with that id does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "NiFi or the database are unavailable.", response = RestResponseStatus.class) })
public Response createJdbcTableDataSet(@PathParam("id") final String dataSourceId, @PathParam("tableName") final String tableName, @QueryParam("schema") final String schema) {
    // TODO Verify user has access to data source
    // Require admin permission if the results should include unencrypted credentials.
    final boolean encryptCredentials = true;
    accessController.checkPermission(AccessController.SERVICES, encryptCredentials ? FeedServicesAccessControl.ACCESS_DATASOURCES : FeedServicesAccessControl.ADMIN_DATASOURCES);
    DataSetWithTableSchema dataSetWithTableSchema = null;
    boolean hasAccess = false;
    try {
        // ensure the user can read teh datasource
        DataSource encryptedSource = metadataService.read(() -> {
            return findDataSource(dataSourceId, true);
        });
        hasAccess = encryptedSource != null;
        if (hasAccess) {
            // fetch the datasource as a service account to get the creds.
            DataSource dataSource = metadataService.read(() -> {
                return findDataSource(dataSourceId, false);
            }, MetadataAccess.SERVICE);
            // describe the datasource and table
            CatalogTableSchema tableSchema = tableManager.describeTable(dataSource, schema, tableName);
            if (tableSchema != null) {
                DataSet dataSet = new DataSet();
                // assign the datasource to this dataset with encrypted credentials
                dataSet.setDataSource(encryptedSource);
                String fullTableName = (tableSchema.getTable() != null) ? tableSchema.getTable().getQualifiedIdentifier() : HiveUtils.quoteIdentifier(tableSchema.getSchemaName(), tableSchema.getName());
                dataSet.setTitle(tableSchema.getSchemaName() + "." + tableSchema.getName());
                DefaultDataSetTemplate defaultDataSetTemplate = DataSetUtil.mergeTemplates(dataSet);
                List<String> paths = defaultDataSetTemplate.getPaths();
                String format = defaultDataSetTemplate.getFormat();
                Map<String, String> options = defaultDataSetTemplate.getOptions();
                if (options == null) {
                    options = new HashMap<>();
                }
                if ("hive".equalsIgnoreCase(format.toLowerCase())) {
                    if (paths == null) {
                        paths = new ArrayList<>();
                    }
                    paths.add(fullTableName);
                }
                options.put("dbtable", fullTableName);
                dataSet.setFormat(format);
                dataSet.setPaths(paths);
                dataSet.setOptions(options);
                DataSet dataSet1 = dataSetService.findOrCreateDataSet(dataSet, encryptCredentials);
                dataSetWithTableSchema = new DataSetWithTableSchema(dataSet1, tableSchema);
            } else {
                if (log.isErrorEnabled()) {
                    log.error("Failed to describe tables for schema [" + schema + "], table [" + tableName + "], dataSource [" + dataSourceId + "] ");
                }
                final RestResponseStatus status = new RestResponseStatus.ResponseStatusBuilder().message(getMessage("catalog.datasource.describeTable.error", tableName, schema)).url(request.getRequestURI()).buildError();
                throw new InternalServerErrorException(Response.serverError().entity(status).build());
            }
        // });
        } else {
            // no acceess
            final RestResponseStatus status = new RestResponseStatus.ResponseStatusBuilder().message(getMessage("catalog.datasource.forbidden")).url(request.getRequestURI()).buildError();
            throw new InternalServerErrorException(Response.serverError().entity(status).build());
        }
    } catch (Exception e) {
        if (exceptionTransformer.causesInChain(e)) {
            throw new ThriftConnectionException(e);
        }
        final RestResponseStatus status = new RestResponseStatus.ResponseStatusBuilder().message(getMessage("catalog.datasource.describeTable.error", tableName, schema)).url(request.getRequestURI()).buildError();
        throw new InternalServerErrorException(Response.serverError().entity(status).build());
    }
    return Response.ok(dataSetWithTableSchema).build();
}

Example 14 with DataSet

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSet in project kylo by Teradata.

the class DataSetController method postUpload.

@POST
@Path("{id}/uploads")
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Uploads a file for the data set.")
@ApiResponses({ @ApiResponse(code = 200, message = "Uploaded file info", response = DataSetFile.class), @ApiResponse(code = 400, message = "Invalid filename", response = RestResponseStatus.class), @ApiResponse(code = 404, message = "Data set does not exist", response = RestResponseStatus.class), @ApiResponse(code = 409, message = "A file already exists with the same name", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "Failed to upload file", response = RestResponseStatus.class) })
public Response postUpload(@PathParam("id") @UUID final String dataSetId, @Nonnull final FormDataMultiPart form) {
    log.entry(dataSetId, form);
    final List<BodyPart> bodyParts = form.getBodyParts();
    if (bodyParts.size() != 1) {
        log.debug("Wrong number of form parts for uploading to dataset {}: {}", dataSetId, bodyParts.size());
        throw new BadRequestException(getMessage("catalog.dataset.postUpload.missingBodyPart"));
    }
    final DataSet dataSet = findDataSet(dataSetId, true);
    final DataSetFile file;
    try {
        final BodyPart part = bodyParts.get(0);
        log.debug("Upload file [{}] for dataset {}", part.getContentDisposition().getFileName(), dataSetId);
        file = fileManager.createUpload(dataSet, part.getContentDisposition().getFileName(), part.getEntityAs(BodyPartEntity.class).getInputStream());
    } catch (final FileAlreadyExistsException e) {
        log.debug("Filename conflict for uploaded file [{}] for dataset {}: {}", bodyParts.get(0).getContentDisposition().getFileName(), dataSetId, e, e);
        throw new WebApplicationException(getMessage("catalog.dataset.postUpload.fileExists"), Response.Status.CONFLICT);
    } catch (final IllegalArgumentException e) {
        log.debug("Invalid filename [{}] for uploaded file for dataset {}: {}", bodyParts.get(0).getContentDisposition().getFileName(), dataSetId, e, e);
        throw new BadRequestException(getMessage("catalog.dataset.invalidFileName"));
    } catch (final Exception e) {
        log.error("Failed to save file for dataset {}: {}", dataSetId, e, e);
        throw new InternalServerErrorException(getMessage("catalog.dataset.postUpload.error"));
    }
    return log.exit(Response.ok(file).build());
}

Example 15 with DataSet

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSet in project kylo by Teradata.

the class DataSetController method getUploads.

@GET
@Path("{id}/uploads")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Lists uploaded files for a data set.")
@ApiResponses({ @ApiResponse(code = 200, message = "List of uploaded files", response = DataSetFile.class, responseContainer = "List"), @ApiResponse(code = 404, message = "Data set does not exist", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "Failed to list uploaded files", response = RestResponseStatus.class) })
public Response getUploads(@PathParam("id") @UUID final String dataSetId) {
    log.entry(dataSetId);
    final DataSet dataSet = findDataSet(dataSetId, true);
    final List<DataSetFile> files;
    try {
        log.debug("Listing uploaded files for dataset {}", dataSetId);
        files = fileManager.listUploads(dataSet);
    } catch (final Exception e) {
        log.error("Unable to retrieve dataset uploads: {}", e, e);
        throw new InternalServerErrorException(getMessage("catalog.dataset.getUploads.error"));
    }
    return log.exit(Response.ok(files).build());
}