Examples with FeatureGroupCommit io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.FeatureGroupCommit used on opensource projects

Example 1 with FeatureGroupCommit

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.FeatureGroupCommit in project hopsworks by logicalclocks.

the class CommitResource method commitFeatureGroup.

@POST
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Commit to Feature Group", response = CommitDTO.class)
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response commitFeatureGroup(@Context UriInfo uriInfo, CommitDTO commitDTO, @Context SecurityContext sc) throws FeaturestoreException {
    Users user = jwtHelper.getUserPrincipal(sc);
    FeatureGroupCommit featureGroupCommit = featureGroupCommitController.createHudiFeatureGroupCommit(user, featuregroup, commitDTO.getCommitDateString(), commitDTO.getCommitTime(), commitDTO.getRowsUpdated(), commitDTO.getRowsInserted(), commitDTO.getRowsDeleted(), commitDTO.getValidationId());
    CommitDTO builtCommitDTO = commitBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.COMMITS), project, featuregroup, featureGroupCommit);
    return Response.ok().entity(builtCommitDTO).build();
}

Example 2 with FeatureGroupCommit

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.FeatureGroupCommit in project hopsworks by logicalclocks.

the class CodeController method registerCode.

public FeaturestoreCode registerCode(Project project, Users user, Long codeCommitTimeStamp, Long fgCommitId, String applicationId, Featuregroup featuregroup, String entityId, String databricksNotebook, byte[] databricksArchive, CodeActions.RunType type) throws ServiceException, FeaturestoreException {
    Inode codeInode = saveCode(project, user, applicationId, featuregroup, entityId, databricksNotebook, databricksArchive, type);
    Timestamp commitTime = new Timestamp(codeCommitTimeStamp);
    FeaturestoreCode featurestoreCode = new FeaturestoreCode(commitTime, codeInode, featuregroup, applicationId);
    if (featuregroup.getFeaturegroupType() == FeaturegroupType.CACHED_FEATURE_GROUP && featuregroup.getCachedFeaturegroup().getTimeTravelFormat() == TimeTravelFormat.HUDI) {
        FeatureGroupCommit featureGroupCommit = featureGroupCommitCommitController.findCommitByDate(featuregroup, fgCommitId);
        featurestoreCode.setFeatureGroupCommit(featureGroupCommit);
    }
    return featurestoreCodeFacade.update(featurestoreCode);
}

Example 3 with FeatureGroupCommit

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.FeatureGroupCommit in project hopsworks by logicalclocks.

the class FeatureGroupCommitFacade method getCommitDetailsByDate.

public CollectionInfo getCommitDetailsByDate(Integer featureGroupId, Integer limit, Integer offset, Set<? extends SortBy> sort, Set<? extends FilterBy> filters) {
    String queryStr = buildQuery("SELECT fgc FROM FeatureGroupCommit fgc ", filters, sort, "fgc.featureGroupCommitPK.featureGroupId = :featureGroupId");
    Query query = em.createQuery(queryStr, FeatureGroupCommit.class).setParameter("featureGroupId", featureGroupId);
    String queryCountStr = buildQuery("SELECT COUNT(fgc.featureGroupCommitPK.commitId) FROM " + "FeatureGroupCommit fgc ", filters, sort, "fgc.featureGroupCommitPK.featureGroupId = :featureGroupId");
    Query queryCount = em.createQuery(queryCountStr, FeatureGroupCommit.class).setParameter("featureGroupId", featureGroupId);
    setFilter(filters, query);
    setFilter(filters, queryCount);
    setOffsetAndLim(offset, limit, query);
    return new CollectionInfo((Long) queryCount.getSingleResult(), query.getResultList());
}

Example 4 with FeatureGroupCommit

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.FeatureGroupCommit in project hopsworks by logicalclocks.

the class FeatureGroupCommitFacade method getCommitDetails.

public CollectionInfo getCommitDetails(Integer featureGroupId, Integer limit, Integer offset, Set<? extends SortBy> sort) {
    String queryStr = buildQuery("SELECT fgc FROM FeatureGroupCommit fgc ", null, sort, "fgc.featureGroupCommitPK.featureGroupId = :featureGroupId");
    Query query = em.createQuery(queryStr, FeatureGroupCommit.class).setParameter("featureGroupId", featureGroupId);
    String queryCountStr = buildQuery("SELECT COUNT(fgc.featureGroupCommitPK.commitId) FROM " + "FeatureGroupCommit fgc ", null, sort, "fgc.featureGroupCommitPK.featureGroupId = :featureGroupId");
    Query queryCount = em.createQuery(queryCountStr, FeatureGroupCommit.class).setParameter("featureGroupId", featureGroupId);
    query.setFirstResult(offset);
    if (limit != null) {
        query.setMaxResults(limit);
    }
    return new CollectionInfo((Long) queryCount.getSingleResult(), query.getResultList());
}

Example 5 with FeatureGroupCommit

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.FeatureGroupCommit in project hopsworks by logicalclocks.

the class FeatureGroupCommitController method createHudiFeatureGroupCommit.

public FeatureGroupCommit createHudiFeatureGroupCommit(Users user, Featuregroup featuregroup, String commitDateString, Long commitTime, Long rowsUpdated, Long rowsInserted, Long rowsDeleted, Integer validationId) throws FeaturestoreException {
    // Compute HUDI commit path
    String commitPath = computeHudiCommitPath(featuregroup, commitDateString);
    Inode inode = inodeController.getInodeAtPath(commitPath);
    // commit id will be timestamp
    FeatureGroupCommit featureGroupCommit = new FeatureGroupCommit(featuregroup.getId(), commitTime);
    featureGroupCommit.setInode(inode);
    featureGroupCommit.setCommittedOn(new Timestamp(commitTime));
    featureGroupCommit.setNumRowsUpdated(rowsUpdated);
    featureGroupCommit.setNumRowsInserted(rowsInserted);
    featureGroupCommit.setNumRowsDeleted(rowsDeleted);
    // Find validation
    if (validationId != null && validationId > 0) {
        featureGroupCommit.setValidation(featureGroupValidationFacade.findById(validationId));
    }
    featureGroupCommit = featureGroupCommitFacade.update(featureGroupCommit);
    fsActivityFacade.logCommitActivity(user, featuregroup, featureGroupCommit);
    return featureGroupCommit;
}