Examples with FeaturegroupDTO io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO used on opensource projects

Example 1 with FeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO in project hopsworks by logicalclocks.

the class TrainingDatasetController method convertTrainingDatasetToDTO.

/**
 * Converts a trainingDataset entity to a TrainingDataset DTO
 *
 * @param user
 * @param project
 * @param trainingDataset trainingDataset entity
 * @return JSON/XML DTO of the trainingDataset
 * @throws ServiceException
 * @throws FeaturestoreException
 */
private TrainingDatasetDTO convertTrainingDatasetToDTO(Users user, Project project, TrainingDataset trainingDataset) throws ServiceException, FeaturestoreException {
    TrainingDatasetDTO trainingDatasetDTO = new TrainingDatasetDTO(trainingDataset);
    String featurestoreName = featurestoreFacade.getHiveDbName(trainingDataset.getFeaturestore().getHiveDbId());
    trainingDatasetDTO.setFeaturestoreName(featurestoreName);
    // Set features
    List<TrainingDatasetFeature> tdFeatures = getFeaturesSorted(trainingDataset, true);
    Map<Integer, String> fsLookupTable = getFsLookupTableFeatures(tdFeatures);
    trainingDatasetDTO.setFeatures(tdFeatures.stream().map(f -> new TrainingDatasetFeatureDTO(checkPrefix(f), f.getType(), f.getFeatureGroup() != null ? new FeaturegroupDTO(f.getFeatureGroup().getFeaturestore().getId(), fsLookupTable.get(f.getFeatureGroup().getFeaturestore().getId()), f.getFeatureGroup().getId(), f.getFeatureGroup().getName(), f.getFeatureGroup().getVersion(), onlineFeaturegroupController.onlineFeatureGroupTopicName(project.getId(), f.getFeatureGroup().getId(), Utils.getFeaturegroupName(f.getFeatureGroup()))) : null, f.getIndex(), f.isLabel())).collect(Collectors.toList()));
    switch(trainingDataset.getTrainingDatasetType()) {
        case HOPSFS_TRAINING_DATASET:
            return hopsfsTrainingDatasetController.convertHopsfsTrainingDatasetToDTO(trainingDatasetDTO, trainingDataset);
        case EXTERNAL_TRAINING_DATASET:
            return externalTrainingDatasetController.convertExternalTrainingDatasetToDTO(user, project, trainingDatasetDTO, trainingDataset);
        default:
            throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_TRAINING_DATASET_TYPE.getMessage() + ", Recognized training dataset types are: " + TrainingDatasetType.HOPSFS_TRAINING_DATASET + ", and: " + TrainingDatasetType.EXTERNAL_TRAINING_DATASET + ". The provided training dataset type was not recognized: " + trainingDataset.getTrainingDatasetType());
    }
}

Example 2 with FeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO in project hopsworks by logicalclocks.

the class FeaturegroupService method updateFeaturegroup.

/**
 * Endpoint for updating the featuregroup metadata without changing the schema.
 * Since the schema is not changed, the data does not need to be dropped.
 *
 * @param featuregroupId id of the featuregroup to update
 * @param featuregroupDTO updated metadata
 * @return JSON representation of the updated featuregroup
 * @throws FeaturestoreException
 */
@PUT
@Path("/{featuregroupId: [0-9]+}")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Update featuregroup contents", response = FeaturegroupDTO.class)
public Response updateFeaturegroup(@Context SecurityContext sc, @ApiParam(value = "Id of the featuregroup", required = true) @PathParam("featuregroupId") Integer featuregroupId, @ApiParam(value = "updateMetadata", example = "true") @QueryParam("updateMetadata") @DefaultValue("false") Boolean updateMetadata, @ApiParam(value = "enableOnline", example = "true") @QueryParam("enableOnline") @DefaultValue("false") Boolean enableOnline, @ApiParam(value = "disableOnline", example = "true") @QueryParam("disableOnline") @DefaultValue("false") Boolean disableOnline, @ApiParam(value = "updateStatsConfig", example = "true") @QueryParam("updateStatsConfig") @DefaultValue("false") Boolean updateStatsConfig, @ApiParam(value = "validationType", example = "NONE") @QueryParam("validationType") ValidationType validationType, FeaturegroupDTO featuregroupDTO) throws FeaturestoreException, SQLException, ProvenanceException, ServiceException, SchemaException, KafkaException, ProjectException, UserException, IOException, HopsSecurityException {
    if (featuregroupDTO == null) {
        throw new IllegalArgumentException("Input JSON for updating Feature Group cannot be null");
    }
    verifyIdProvided(featuregroupId);
    featuregroupDTO.setId(featuregroupId);
    Users user = jWTHelper.getUserPrincipal(sc);
    Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId);
    FeaturegroupDTO updatedFeaturegroupDTO = null;
    if (updateMetadata) {
        updatedFeaturegroupDTO = featuregroupController.updateFeaturegroupMetadata(project, user, featurestore, featuregroupDTO);
    }
    if (enableOnline && featuregroup.getFeaturegroupType() == FeaturegroupType.CACHED_FEATURE_GROUP && !(featuregroup.getCachedFeaturegroup().isOnlineEnabled())) {
        updatedFeaturegroupDTO = featuregroupController.enableFeaturegroupOnline(featurestore, featuregroupDTO, project, user);
    }
    if (disableOnline && featuregroup.getFeaturegroupType() == FeaturegroupType.CACHED_FEATURE_GROUP && featuregroup.getCachedFeaturegroup().isOnlineEnabled()) {
        updatedFeaturegroupDTO = featuregroupController.disableFeaturegroupOnline(featuregroup, project, user);
    }
    if (updateStatsConfig) {
        updatedFeaturegroupDTO = featuregroupController.updateFeatureGroupStatsConfig(featurestore, featuregroupDTO, project, user);
    }
    if (validationType != null) {
        updatedFeaturegroupDTO = featuregroupController.updateValidationType(featuregroup, validationType, project, user);
    }
    if (updatedFeaturegroupDTO != null) {
        GenericEntity<FeaturegroupDTO> featuregroupGeneric = new GenericEntity<FeaturegroupDTO>(updatedFeaturegroupDTO) {
        };
        return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(featuregroupGeneric).build();
    } else {
        GenericEntity<FeaturegroupDTO> featuregroupGeneric = new GenericEntity<FeaturegroupDTO>(featuregroupDTO) {
        };
        return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(featuregroupGeneric).build();
    }
}

Example 3 with FeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO in project hopsworks by logicalclocks.

the class FeaturegroupService method deleteFeaturegroupContents.

/**
 * Endpoint for deleting the contents of the featuregroup.
 * As HopsHive do not support ACID transactions the way to delete the contents of a table is to drop the table and
 * re-create it, which also will drop the featuregroup metadata due to ON DELETE CASCADE foreign key rule.
 * This method stores the metadata of the featuregroup before deleting it and then re-creates the featuregroup with
 * the same metadata.
 * <p>
 * This endpoint is typically used when the user wants to insert data into a featuregroup with the write-mode
 * 'overwrite' instead of default mode 'append'
 *
 * @param featuregroupId the id of the featuregroup
 * @throws FeaturestoreException
 * @throws HopsSecurityException
 */
@POST
@Path("/{featuregroupId}/clear")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Delete featuregroup contents")
public Response deleteFeaturegroupContents(@Context SecurityContext sc, @ApiParam(value = "Id of the featuregroup", required = true) @PathParam("featuregroupId") Integer featuregroupId) throws FeaturestoreException, ServiceException, KafkaException, SchemaException, ProjectException, UserException {
    verifyIdProvided(featuregroupId);
    Users user = jWTHelper.getUserPrincipal(sc);
    // Verify that the user has the data-owner role or is the creator of the featuregroup
    Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId);
    try {
        FeaturegroupDTO newFeatureGroup = featuregroupController.clearFeaturegroup(featuregroup, project, user);
        return Response.ok().entity(newFeatureGroup).build();
    } catch (SQLException | IOException | ProvenanceException | HopsSecurityException e) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_CLEAR_FEATUREGROUP, Level.SEVERE, "project: " + project.getName() + ", featurestoreId: " + featurestore.getId() + ", featuregroupId: " + featuregroupId, e.getMessage(), e);
    }
}

Example 4 with FeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO in project hopsworks by logicalclocks.

the class FeaturegroupService method createFeaturegroup.

/**
 * Endpoint for creating a new featuregroup in a featurestore
 *
 * @param featuregroupDTO JSON payload for the new featuregroup
 * @return JSON information about the created featuregroup
 * @throws HopsSecurityException
 */
@POST
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Create feature group in a featurestore", response = FeaturegroupDTO.class)
public Response createFeaturegroup(@Context SecurityContext sc, FeaturegroupDTO featuregroupDTO) throws FeaturestoreException, ServiceException, KafkaException, SchemaException, ProjectException, UserException {
    Users user = jWTHelper.getUserPrincipal(sc);
    if (featuregroupDTO == null) {
        throw new IllegalArgumentException("Input JSON for creating a new Feature Group cannot be null");
    }
    try {
        if (featuregroupController.featuregroupExists(featurestore, featuregroupDTO)) {
            throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATUREGROUP_EXISTS, Level.INFO, "project: " + project.getName() + ", featurestoreId: " + featurestore.getId());
        }
        FeaturegroupDTO createdFeaturegroup = featuregroupController.createFeaturegroup(featurestore, featuregroupDTO, project, user);
        GenericEntity<FeaturegroupDTO> featuregroupGeneric = new GenericEntity<FeaturegroupDTO>(createdFeaturegroup) {
        };
        return noCacheResponse.getNoCacheResponseBuilder(Response.Status.CREATED).entity(featuregroupGeneric).build();
    } catch (SQLException | ProvenanceException | IOException | HopsSecurityException e) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_CREATE_FEATUREGROUP, Level.SEVERE, "project: " + project.getName() + ", featurestoreId: " + featurestore.getId(), e.getMessage(), e);
    }
}

Example 5 with FeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO in project hopsworks by logicalclocks.

the class TestPitJoinController method testIsPitEnabledTrueQueryDTO.

@Test
public void testIsPitEnabledTrueQueryDTO() {
    FeaturegroupDTO fg1 = new FeaturegroupDTO("ts");
    FeaturegroupDTO fg2 = new FeaturegroupDTO("ts");
    JoinDTO join1 = new JoinDTO(new QueryDTO(fg1, null), null, null);
    JoinDTO join2 = new JoinDTO(new QueryDTO(fg2, null), null, null);
    List<JoinDTO> joins = Arrays.asList(join1, join2);
    QueryDTO leftQuery = new QueryDTO(new FeaturegroupDTO("ts"), new ArrayList<>(), joins);
    Assert.assertEquals(true, pitJoinController.isPitEnabled(leftQuery));
}