Examples with TrainingDatasetDTO io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO used on opensource projects

Example 1 with TrainingDatasetDTO

use of io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO in project hopsworks by logicalclocks.

the class TrainingDatasetService method getByName.

/**
 * Endpoint for getting a list of training dataset based on the name
 *
 * @param name name of the training dataset to get
 * @return return a JSON representation of the training dataset with the given id
 * @throws FeaturestoreException
 */
@GET
@Path("/{name: [a-z0-9_]*(?=[a-z])[a-z0-9_]+}")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Get a list of training datasets with a specific name, filter by version", response = List.class)
public Response getByName(@ApiParam(value = "Name of the training dataset", required = true) @PathParam("name") String name, @ApiParam(value = "Filter by a specific version") @QueryParam("version") Integer version, @Context SecurityContext sc) throws FeaturestoreException, ServiceException {
    verifyNameProvided(name);
    Users user = jWTHelper.getUserPrincipal(sc);
    List<TrainingDatasetDTO> trainingDatasetDTO;
    if (version == null) {
        trainingDatasetDTO = trainingDatasetController.getWithNameAndFeaturestore(user, project, featurestore, name);
    } else {
        trainingDatasetDTO = Arrays.asList(trainingDatasetController.getWithNameVersionAndFeaturestore(user, project, featurestore, name, version));
    }
    GenericEntity<List<TrainingDatasetDTO>> trainingDatasetGeneric = new GenericEntity<List<TrainingDatasetDTO>>(trainingDatasetDTO) {
    };
    return Response.ok().entity(trainingDatasetGeneric).build();
}

Example 2 with TrainingDatasetDTO

use of io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO in project hopsworks by logicalclocks.

the class TrainingDatasetService method getAll.

/**
 * Endpoint for getting a list of all training datasets in the feature store.
 *
 * @return a JSON representation of the training datasets in the features store
 */
@GET
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Get the list of training datasets for a featurestore", response = TrainingDatasetDTO.class, responseContainer = "List")
public Response getAll(@Context SecurityContext sc) throws ServiceException, FeaturestoreException {
    Users user = jWTHelper.getUserPrincipal(sc);
    List<TrainingDatasetDTO> trainingDatasetDTOs = trainingDatasetController.getTrainingDatasetsForFeaturestore(user, project, featurestore);
    GenericEntity<List<TrainingDatasetDTO>> trainingDatasetsGeneric = new GenericEntity<List<TrainingDatasetDTO>>(trainingDatasetDTOs) {
    };
    return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(trainingDatasetsGeneric).build();
}

Example 3 with TrainingDatasetDTO

use of io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO in project hopsworks by logicalclocks.

the class TestStatisticColumnController method testVerifyStatisticColumnsExistTD.

@Test
public void testVerifyStatisticColumnsExistTD() throws Exception {
    TrainingDataset trainingDataset = new TrainingDataset();
    StatisticsConfigDTO statisticsConfig = new StatisticsConfigDTO();
    statisticsConfig.setColumns(Arrays.asList("ft1", "ft4"));
    trainingDataset.setFeatures(Arrays.asList(new TrainingDatasetFeature(trainingDataset, "ft1", null, null, false, null), new TrainingDatasetFeature(trainingDataset, "ft2", null, null, false, null), new TrainingDatasetFeature(trainingDataset, "ft3", null, null, false, null)));
    TrainingDatasetDTO trainingDatasetDTO = new TrainingDatasetDTO();
    trainingDatasetDTO.setName("td1");
    trainingDatasetDTO.setVersion(1);
    trainingDatasetDTO.setStatisticsConfig(statisticsConfig);
    // should throw exception
    thrown.expect(FeaturestoreException.class);
    statisticColumnController.verifyStatisticColumnsExist(trainingDatasetDTO, trainingDataset);
    // should not throw exception
    statisticsConfig.setColumns(Arrays.asList("ft1", "ft2"));
    statisticColumnController.verifyStatisticColumnsExist(trainingDatasetDTO, trainingDataset);
}

Example 4 with TrainingDatasetDTO

use of io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO in project hopsworks by logicalclocks.

the class TrainingDatasetService method create.

/**
 * Endpoint for creating a new trainingDataset
 *
 * @param trainingDatasetDTO the JSON payload with the data of the new trainingDataset
 * @return JSON representation of the created trainingDataset
 */
@POST
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Create training dataset for a featurestore", response = TrainingDatasetDTO.class)
public Response create(@Context SecurityContext sc, TrainingDatasetDTO trainingDatasetDTO) throws FeaturestoreException, ProvenanceException, IOException, ServiceException {
    if (trainingDatasetDTO == null) {
        throw new IllegalArgumentException("Input JSON for creating a new Training Dataset cannot be null");
    }
    Users user = jWTHelper.getUserPrincipal(sc);
    TrainingDatasetDTO createdTrainingDatasetDTO = trainingDatasetController.createTrainingDataset(user, project, featurestore, trainingDatasetDTO);
    activityFacade.persistActivity(ActivityFacade.CREATED_TRAINING_DATASET + createdTrainingDatasetDTO.getName(), project, user, ActivityFlag.SERVICE);
    GenericEntity<TrainingDatasetDTO> createdTrainingDatasetDTOGeneric = new GenericEntity<TrainingDatasetDTO>(createdTrainingDatasetDTO) {
    };
    return noCacheResponse.getNoCacheResponseBuilder(Response.Status.CREATED).entity(createdTrainingDatasetDTOGeneric).build();
}

Example 5 with TrainingDatasetDTO

use of io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO in project hopsworks by logicalclocks.

the class TrainingDatasetService method getById.

/**
 * Endpoint for getting a training dataset with a particular id
 *
 * @param trainingdatasetid id of the training dataset to get
 * @return return a JSON representation of the training dataset with the given id
 * @throws FeaturestoreException
 *
 * @deprecated : use getTrainingDatasetByName instead
 */
@Deprecated
@GET
@Path("/{trainingdatasetid: [0-9]+}")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Get a training datasets with a specific id from a featurestore", response = TrainingDatasetDTO.class)
public Response getById(@ApiParam(value = "Id of the training dataset", required = true) @PathParam("trainingdatasetid") Integer trainingdatasetid, @Context SecurityContext sc) throws FeaturestoreException, ServiceException {
    verifyIdProvided(trainingdatasetid);
    Users user = jWTHelper.getUserPrincipal(sc);
    TrainingDatasetDTO trainingDatasetDTO = trainingDatasetController.getTrainingDatasetWithIdAndFeaturestore(user, project, featurestore, trainingdatasetid);
    GenericEntity<TrainingDatasetDTO> trainingDatasetGeneric = new GenericEntity<TrainingDatasetDTO>(trainingDatasetDTO) {
    };
    return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(trainingDatasetGeneric).build();
}