Examples with DataSet com.thinkbiganalytics.kylo.catalog.rest.model.DataSet used on opensource projects

Example 16 with DataSet

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSet in project kylo by Teradata.

the class DataSetController method createDataSet.

@POST
@ApiOperation("Creates a new data set")
@ApiResponses({ @ApiResponse(code = 200, message = "Data set created", response = DataSet.class), @ApiResponse(code = 400, message = "Invalid data source", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "Internal server error", response = RestResponseStatus.class) })
@Consumes(MediaType.APPLICATION_JSON)
public Response createDataSet(@Nonnull final DataSet source) {
    log.entry(source);
    final boolean encryptCredentials = true;
    DataSet dataSet;
    try {
        dataSet = dataSetService.findOrCreateDataSet(source, encryptCredentials);
    } catch (final CatalogException e) {
        log.debug("Cannot create data set from request: {}", source, e);
        throw new BadRequestException(getMessage(e));
    }
    return Response.ok(log.exit(dataSet)).build();
}

Example 17 with DataSet

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSet in project kylo by Teradata.

the class DataSetController method deleteUpload.

@DELETE
@Path("{id}/uploads/{name}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Deletes an upload file from a data set.")
@ApiResponses({ @ApiResponse(code = 204, message = "The file was deleted successfully"), @ApiResponse(code = 404, message = "Data set or file does not exist", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "Failed to delete file", response = RestResponseStatus.class) })
public Response deleteUpload(@PathParam("id") @UUID final String dataSetId, @PathParam("name") final String fileName) {
    log.entry(dataSetId, fileName);
    final DataSet dataSet = findDataSet(dataSetId, true);
    try {
        log.debug("Deleting uploaded file [{}] from dataset {}", fileName, dataSetId);
        fileManager.deleteUpload(dataSet, fileName);
    } catch (final IllegalArgumentException e) {
        log.debug("Invalid name for deleting file [{}] from dataset {}: {}", fileName, dataSetId, e, e);
        throw new NotFoundException(getMessage("catalog.dataset.invalidFileName"));
    } catch (final Exception e) {
        log.error("Failed do delete file [{}] from dataset {}: {}", fileName, dataSetId, e, e);
        throw new InternalServerErrorException(getMessage("catalog.dataset.deleteUpload.error"));
    }
    return log.exit(Response.noContent().build());
}

Example 18 with DataSet

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSet in project kylo by Teradata.

the class DataSetUtil method getPaths.

/**
 * Gets the paths for the specified data set.
 */
@Nonnull
public static Optional<List<String>> getPaths(@Nonnull final DataSet dataSet) {
    final DataSetTemplate connectorTemplate = Optional.of(dataSet).map(DataSet::getDataSource).map(DataSource::getConnector).map(Connector::getTemplate).orElse(null);
    final DataSetTemplate dataSetTemplate = Optional.of(dataSet).map(DataSet::getDataSource).map(DataSource::getTemplate).orElse(null);
    List<String> paths = new ArrayList<>();
    // Add "path" option
    if (dataSet.getOptions() != null && dataSet.getOptions().get("path") != null) {
        paths.add(dataSet.getOptions().get("path"));
    } else if (dataSetTemplate != null && dataSetTemplate.getOptions() != null && dataSetTemplate.getOptions().get("path") != null) {
        paths.add(dataSetTemplate.getOptions().get("path"));
    } else if (connectorTemplate != null && connectorTemplate.getOptions() != null && connectorTemplate.getOptions().get("path") != null) {
        paths.add(connectorTemplate.getOptions().get("path"));
    }
    // Add paths list
    if (dataSet.getPaths() != null) {
        paths.addAll(dataSet.getPaths());
    } else if (dataSetTemplate != null && dataSetTemplate.getPaths() != null) {
        paths.addAll(dataSetTemplate.getPaths());
    } else if (connectorTemplate != null && connectorTemplate.getPaths() != null) {
        paths.addAll(connectorTemplate.getPaths());
    } else if (paths.isEmpty()) {
        paths = null;
    }
    return Optional.ofNullable(paths);
}

Example 19 with DataSet

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSet in project kylo by Teradata.

the class CatalogFileManagerTest method listUploads.

/**
 * Verify listing uploaded files.
 */
@Test
public void listUploads() throws IOException {
    // Create data set including files
    final DataSet dataSet = createDataSet();
    final File dataSetFolder = datasetsFolder.newFolder(dataSet.getId());
    Files.write("data1", new File(dataSetFolder, "file1.txt"), StandardCharsets.UTF_8);
    Files.write("data2", new File(dataSetFolder, "file2.txt"), StandardCharsets.UTF_8);
    Files.write("data3", new File(dataSetFolder, "file3.txt"), StandardCharsets.UTF_8);
    // Test listing files
    final CatalogFileManager fileManager = new MockCatalogFileManager();
    final List<DataSetFile> files = fileManager.listUploads(dataSet);
    Assert.assertThat(files, CoreMatchers.hasItem(equalTo("file1.txt", new Path(dataSetFolder.toPath().resolve("file1.txt").toUri()).toString(), false, 5, "data1")));
    Assert.assertThat(files, CoreMatchers.hasItem(equalTo("file2.txt", new Path(dataSetFolder.toPath().resolve("file2.txt").toUri()).toString(), false, 5, "data2")));
    Assert.assertThat(files, CoreMatchers.hasItem(equalTo("file3.txt", new Path(dataSetFolder.toPath().resolve("file3.txt").toUri()).toString(), false, 5, "data3")));
    Assert.assertEquals(3, files.size());
}

Example 20 with DataSet

use of com.thinkbiganalytics.kylo.catalog.rest.model.DataSet in project kylo by Teradata.

the class SparkShellProxyController method fileMetadata.

@POST
@Path(FILE_METADATA)
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("returns filemetadata based upon the list of file paths in the dataset.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the status of the file-metadata job.", response = TransformResponse.class), @ApiResponse(code = 400, message = "The requested data source does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "There was a problem processing the data.", response = RestResponseStatus.class) })
public Response fileMetadata(com.thinkbiganalytics.kylo.catalog.rest.model.DataSet dataSet) {
    TransformRequest request = new TransformRequest();
    DataSet decrypted = catalogModelTransform.decryptOptions(dataSet);
    request.setScript(FileMetadataScalaScriptGenerator.getScript(DataSetUtil.getPaths(decrypted).orElseGet(Collections::emptyList), DataSetUtil.mergeTemplates(decrypted).getOptions()));
    final SparkShellProcess process = getSparkShellProcess();
    return getModifiedTransformResponse(() -> Optional.of(restClient.transform(process, request)), new FileMetadataTransformResponseModifier(fileMetadataTrackerService));
}