Examples with Featuregroup io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup used on opensource projects

Example 1 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class TestCommitBuilder method setup.

@Before
public void setup() {
    featureGroupCommitController = Mockito.mock(FeatureGroupCommitController.class);
    Mockito.when(featureGroupCommitController.getCommitDetails(Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anySet(), Mockito.anySet())).thenReturn(new AbstractFacade.CollectionInfo(0L, new ArrayList()));
    // throws exception if limit or offset is null
    Mockito.when(featureGroupCommitController.getCommitDetails(Mockito.anyInt(), Mockito.anyInt(), Mockito.isNull(), Mockito.anySet(), Mockito.anySet())).thenThrow(new NullPointerException());
    Mockito.when(featureGroupCommitController.getCommitDetails(Mockito.anyInt(), Mockito.isNull(), Mockito.anyInt(), Mockito.anySet(), Mockito.anySet())).thenThrow(new NullPointerException());
    commitBuilder = new CommitBuilder(featureGroupCommitController);
    mockUriInfo = Mockito.mock(UriInfo.class);
    UriBuilder mockUriBuilder = Mockito.mock(UriBuilder.class);
    Mockito.when(mockUriBuilder.path(Mockito.anyString())).thenReturn(mockUriBuilder);
    Mockito.when(mockUriInfo.getBaseUriBuilder()).thenReturn(mockUriBuilder);
    featuregroup = new Featuregroup();
    featuregroup.setId(0);
    Featurestore featurestore = new Featurestore();
    featurestore.setId(0);
    featuregroup.setFeaturestore(featurestore);
    project = new Project();
    project.setId(0);
}

Example 2 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class FeaturegroupService method deleteTag.

@ApiOperation(value = "Delete tag attached to featuregroup")
@DELETE
@Path("/{featuregroupId}/tags/{name}")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response deleteTag(@Context SecurityContext sc, @ApiParam(value = "Id of the featuregroup", required = true) @PathParam("featuregroupId") Integer featuregroupId, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name) throws DatasetException, MetadataException, FeaturestoreException {
    verifyIdProvided(featuregroupId);
    Users user = jWTHelper.getUserPrincipal(sc);
    Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId);
    tagController.delete(project, user, featurestore, featuregroup, name);
    return Response.noContent().build();
}

Example 3 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class FeaturegroupService method updateFeaturegroup.

/**
 * Endpoint for updating the featuregroup metadata without changing the schema.
 * Since the schema is not changed, the data does not need to be dropped.
 *
 * @param featuregroupId id of the featuregroup to update
 * @param featuregroupDTO updated metadata
 * @return JSON representation of the updated featuregroup
 * @throws FeaturestoreException
 */
@PUT
@Path("/{featuregroupId: [0-9]+}")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Update featuregroup contents", response = FeaturegroupDTO.class)
public Response updateFeaturegroup(@Context SecurityContext sc, @ApiParam(value = "Id of the featuregroup", required = true) @PathParam("featuregroupId") Integer featuregroupId, @ApiParam(value = "updateMetadata", example = "true") @QueryParam("updateMetadata") @DefaultValue("false") Boolean updateMetadata, @ApiParam(value = "enableOnline", example = "true") @QueryParam("enableOnline") @DefaultValue("false") Boolean enableOnline, @ApiParam(value = "disableOnline", example = "true") @QueryParam("disableOnline") @DefaultValue("false") Boolean disableOnline, @ApiParam(value = "updateStatsConfig", example = "true") @QueryParam("updateStatsConfig") @DefaultValue("false") Boolean updateStatsConfig, @ApiParam(value = "validationType", example = "NONE") @QueryParam("validationType") ValidationType validationType, FeaturegroupDTO featuregroupDTO) throws FeaturestoreException, SQLException, ProvenanceException, ServiceException, SchemaException, KafkaException, ProjectException, UserException, IOException, HopsSecurityException {
    if (featuregroupDTO == null) {
        throw new IllegalArgumentException("Input JSON for updating Feature Group cannot be null");
    }
    verifyIdProvided(featuregroupId);
    featuregroupDTO.setId(featuregroupId);
    Users user = jWTHelper.getUserPrincipal(sc);
    Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId);
    FeaturegroupDTO updatedFeaturegroupDTO = null;
    if (updateMetadata) {
        updatedFeaturegroupDTO = featuregroupController.updateFeaturegroupMetadata(project, user, featurestore, featuregroupDTO);
    }
    if (enableOnline && featuregroup.getFeaturegroupType() == FeaturegroupType.CACHED_FEATURE_GROUP && !(featuregroup.getCachedFeaturegroup().isOnlineEnabled())) {
        updatedFeaturegroupDTO = featuregroupController.enableFeaturegroupOnline(featurestore, featuregroupDTO, project, user);
    }
    if (disableOnline && featuregroup.getFeaturegroupType() == FeaturegroupType.CACHED_FEATURE_GROUP && featuregroup.getCachedFeaturegroup().isOnlineEnabled()) {
        updatedFeaturegroupDTO = featuregroupController.disableFeaturegroupOnline(featuregroup, project, user);
    }
    if (updateStatsConfig) {
        updatedFeaturegroupDTO = featuregroupController.updateFeatureGroupStatsConfig(featurestore, featuregroupDTO, project, user);
    }
    if (validationType != null) {
        updatedFeaturegroupDTO = featuregroupController.updateValidationType(featuregroup, validationType, project, user);
    }
    if (updatedFeaturegroupDTO != null) {
        GenericEntity<FeaturegroupDTO> featuregroupGeneric = new GenericEntity<FeaturegroupDTO>(updatedFeaturegroupDTO) {
        };
        return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(featuregroupGeneric).build();
    } else {
        GenericEntity<FeaturegroupDTO> featuregroupGeneric = new GenericEntity<FeaturegroupDTO>(featuregroupDTO) {
        };
        return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(featuregroupGeneric).build();
    }
}

Example 4 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class FeaturegroupService method deleteFeaturegroupContents.

/**
 * Endpoint for deleting the contents of the featuregroup.
 * As HopsHive do not support ACID transactions the way to delete the contents of a table is to drop the table and
 * re-create it, which also will drop the featuregroup metadata due to ON DELETE CASCADE foreign key rule.
 * This method stores the metadata of the featuregroup before deleting it and then re-creates the featuregroup with
 * the same metadata.
 * <p>
 * This endpoint is typically used when the user wants to insert data into a featuregroup with the write-mode
 * 'overwrite' instead of default mode 'append'
 *
 * @param featuregroupId the id of the featuregroup
 * @throws FeaturestoreException
 * @throws HopsSecurityException
 */
@POST
@Path("/{featuregroupId}/clear")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Delete featuregroup contents")
public Response deleteFeaturegroupContents(@Context SecurityContext sc, @ApiParam(value = "Id of the featuregroup", required = true) @PathParam("featuregroupId") Integer featuregroupId) throws FeaturestoreException, ServiceException, KafkaException, SchemaException, ProjectException, UserException {
    verifyIdProvided(featuregroupId);
    Users user = jWTHelper.getUserPrincipal(sc);
    // Verify that the user has the data-owner role or is the creator of the featuregroup
    Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId);
    try {
        FeaturegroupDTO newFeatureGroup = featuregroupController.clearFeaturegroup(featuregroup, project, user);
        return Response.ok().entity(newFeatureGroup).build();
    } catch (SQLException | IOException | ProvenanceException | HopsSecurityException e) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_CLEAR_FEATUREGROUP, Level.SEVERE, "project: " + project.getName() + ", featurestoreId: " + featurestore.getId() + ", featuregroupId: " + featuregroupId, e.getMessage(), e);
    }
}

Example 5 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class FeaturegroupService method ingestionJob.

@POST
@Path("/{featuregroupId}/ingestion")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Prepares environment for uploading data to ingest into the feature group", response = IngestionJobDTO.class)
public Response ingestionJob(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Id of the featuregroup", required = true) @PathParam("featuregroupId") Integer featuregroupId, IngestionJobConf ingestionJobConf) throws DatasetException, HopsSecurityException, FeaturestoreException, JobException {
    Users user = jWTHelper.getUserPrincipal(sc);
    verifyIdProvided(featuregroupId);
    Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId);
    Map<String, String> dataOptions = null;
    if (ingestionJobConf.getDataOptions() != null) {
        dataOptions = ingestionJobConf.getDataOptions().stream().collect(Collectors.toMap(OptionDTO::getName, OptionDTO::getValue));
    }
    Map<String, String> writeOptions = null;
    if (ingestionJobConf.getWriteOptions() != null) {
        writeOptions = ingestionJobConf.getWriteOptions().stream().collect(Collectors.toMap(OptionDTO::getName, OptionDTO::getValue));
    }
    IngestionJob ingestionJob = fsJobManagerController.setupIngestionJob(project, user, featuregroup, ingestionJobConf.getSparkJobConfiguration(), ingestionJobConf.getDataFormat(), writeOptions, dataOptions);
    IngestionJobDTO ingestionJobDTO = ingestionJobBuilder.build(uriInfo, project, featuregroup, ingestionJob);
    return Response.ok().entity(ingestionJobDTO).build();
}