Examples with OnDemandFeaturegroupDTO io.hops.hopsworks.common.featurestore.featuregroup.ondemand.OnDemandFeaturegroupDTO used on opensource projects

Example 1 with OnDemandFeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.ondemand.OnDemandFeaturegroupDTO in project hopsworks by logicalclocks.

the class FeaturegroupController method createFeaturegroupNoValidation.

public FeaturegroupDTO createFeaturegroupNoValidation(Featurestore featurestore, FeaturegroupDTO featuregroupDTO, Project project, Users user) throws FeaturestoreException, SQLException, ProvenanceException, ServiceException, KafkaException, SchemaException, ProjectException, UserException, IOException, HopsSecurityException {
    // Persist specific feature group metadata (cached fg or on-demand fg)
    OnDemandFeaturegroup onDemandFeaturegroup = null;
    CachedFeaturegroup cachedFeaturegroup = null;
    List<FeatureGroupFeatureDTO> featuresNoHudi = null;
    if (featuregroupDTO instanceof CachedFeaturegroupDTO) {
        // make copy of schema without hudi columns
        featuresNoHudi = new ArrayList<>(featuregroupDTO.getFeatures());
        cachedFeaturegroup = cachedFeaturegroupController.createCachedFeaturegroup(featurestore, (CachedFeaturegroupDTO) featuregroupDTO, project, user);
    } else {
        onDemandFeaturegroup = onDemandFeaturegroupController.createOnDemandFeaturegroup(featurestore, (OnDemandFeaturegroupDTO) featuregroupDTO, project, user);
    }
    // Persist basic feature group metadata
    Featuregroup featuregroup = persistFeaturegroupMetadata(featurestore, user, featuregroupDTO, cachedFeaturegroup, onDemandFeaturegroup);
    // online feature group needs to be set up after persisting metadata in order to get feature group id
    if (featuregroupDTO instanceof CachedFeaturegroupDTO && settings.isOnlineFeaturestore() && featuregroup.getCachedFeaturegroup().isOnlineEnabled()) {
        onlineFeaturegroupController.setupOnlineFeatureGroup(featurestore, featuregroup, featuresNoHudi, project, user);
    }
    FeaturegroupDTO completeFeaturegroupDTO = convertFeaturegrouptoDTO(featuregroup, project, user);
    // Extract metadata
    String hdfsUsername = hdfsUsersController.getHdfsUserName(project, user);
    DistributedFileSystemOps udfso = dfs.getDfsOps(hdfsUsername);
    try {
        String fgPath = Utils.getFeaturestorePath(featurestore.getProject(), settings) + "/" + Utils.getFeaturegroupName(featuregroup);
        fsController.featuregroupAttachXAttrs(fgPath, completeFeaturegroupDTO, udfso);
    } finally {
        dfs.closeDfsClient(udfso);
    }
    // Log activity
    fsActivityFacade.logMetadataActivity(user, featuregroup, FeaturestoreActivityMeta.FG_CREATED, null);
    return completeFeaturegroupDTO;
}

Example 2 with OnDemandFeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.ondemand.OnDemandFeaturegroupDTO in project hopsworks by logicalclocks.

the class FeaturegroupController method convertFeaturegrouptoDTO.

/**
 * Convert a featuregroup entity to a DTO representation
 *
 * @param featuregroup the entity to convert
 * @return a DTO representation of the entity
 */
private FeaturegroupDTO convertFeaturegrouptoDTO(Featuregroup featuregroup, Project project, Users user) throws FeaturestoreException, ServiceException {
    String featurestoreName = featurestoreFacade.getHiveDbName(featuregroup.getFeaturestore().getHiveDbId());
    switch(featuregroup.getFeaturegroupType()) {
        case CACHED_FEATURE_GROUP:
            CachedFeaturegroupDTO cachedFeaturegroupDTO = cachedFeaturegroupController.convertCachedFeaturegroupToDTO(featuregroup, project, user);
            cachedFeaturegroupDTO.setFeaturestoreName(featurestoreName);
            return cachedFeaturegroupDTO;
        case ON_DEMAND_FEATURE_GROUP:
            FeaturestoreStorageConnectorDTO storageConnectorDTO = connectorController.convertToConnectorDTO(user, project, featuregroup.getOnDemandFeaturegroup().getFeaturestoreConnector());
            OnDemandFeaturegroupDTO onDemandFeaturegroupDTO = new OnDemandFeaturegroupDTO(featurestoreName, featuregroup, storageConnectorDTO);
            try {
                String path = getFeatureGroupLocation(featuregroup);
                String location = featurestoreUtils.prependNameNode(path);
                onDemandFeaturegroupDTO.setLocation(location);
            } catch (ServiceDiscoveryException e) {
                throw new ServiceException(RESTCodes.ServiceErrorCode.SERVICE_NOT_FOUND, Level.SEVERE);
            }
            return onDemandFeaturegroupDTO;
        default:
            throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_FEATUREGROUP_TYPE.getMessage() + ", Recognized Feature group types are: " + FeaturegroupType.ON_DEMAND_FEATURE_GROUP + ", and: " + FeaturegroupType.CACHED_FEATURE_GROUP + ". The provided feature group type was not recognized: " + featuregroup.getFeaturegroupType());
    }
}

Example 3 with OnDemandFeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.ondemand.OnDemandFeaturegroupDTO in project hopsworks by logicalclocks.

the class FeaturegroupController method updateFeaturegroupMetadata.

/**
 * Updates metadata for a featuregroup
 *
 * @param featurestore    the featurestore where the featuregroup resides
 * @param featuregroupDTO the updated featuregroup metadata
 * @return DTO of the updated feature group
 * @throws FeaturestoreException
 */
public FeaturegroupDTO updateFeaturegroupMetadata(Project project, Users user, Featurestore featurestore, FeaturegroupDTO featuregroupDTO) throws FeaturestoreException, SQLException, ProvenanceException, ServiceException, SchemaException, KafkaException {
    Featuregroup featuregroup = getFeaturegroupById(featurestore, featuregroupDTO.getId());
    // currently supports updating:
    // adding new features
    // feature group description
    // feature descriptions
    // Verify general entity related information
    featurestoreInputValidation.verifyDescription(featuregroupDTO);
    featureGroupInputValidation.verifyFeatureGroupFeatureList(featuregroupDTO.getFeatures());
    // Update on-demand feature group metadata
    if (featuregroup.getFeaturegroupType() == FeaturegroupType.CACHED_FEATURE_GROUP) {
        cachedFeaturegroupController.updateMetadata(project, user, featuregroup, (CachedFeaturegroupDTO) featuregroupDTO);
    } else if (featuregroup.getFeaturegroupType() == FeaturegroupType.ON_DEMAND_FEATURE_GROUP) {
        onDemandFeaturegroupController.updateOnDemandFeaturegroupMetadata(featuregroup.getOnDemandFeaturegroup(), (OnDemandFeaturegroupDTO) featuregroupDTO);
    }
    // get feature group object again after alter table
    featuregroup = getFeaturegroupById(featurestore, featuregroupDTO.getId());
    featuregroupDTO = convertFeaturegrouptoDTO(featuregroup, project, user);
    String hdfsUsername = hdfsUsersController.getHdfsUserName(project, user);
    DistributedFileSystemOps udfso = dfs.getDfsOps(hdfsUsername);
    try {
        String fgPath = Utils.getFeaturestorePath(featurestore.getProject(), settings) + "/" + Utils.getFeaturegroupName(featuregroup);
        fsController.featuregroupAttachXAttrs(fgPath, featuregroupDTO, udfso);
    } finally {
        dfs.closeDfsClient(udfso);
    }
    return featuregroupDTO;
}

Example 4 with OnDemandFeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.ondemand.OnDemandFeaturegroupDTO in project hopsworks by logicalclocks.

the class HopsFSProvenanceController method fromFeaturegroup.

private FeaturegroupXAttr.FullDTO fromFeaturegroup(FeaturegroupDTO featuregroup) {
    List<FeaturegroupXAttr.SimpleFeatureDTO> features = new LinkedList<>();
    for (FeatureGroupFeatureDTO feature : featuregroup.getFeatures()) {
        features.add(new FeaturegroupXAttr.SimpleFeatureDTO(feature.getName(), feature.getDescription()));
    }
    FeaturegroupXAttr.FullDTO fullDTO = new FeaturegroupXAttr.FullDTO(featuregroup.getFeaturestoreId(), featuregroup.getDescription(), featuregroup.getCreated(), featuregroup.getCreator().getEmail(), features);
    if (featuregroup instanceof CachedFeaturegroupDTO) {
        fullDTO.setFgType(FeaturegroupXAttr.FGType.CACHED);
    } else if (featuregroup instanceof OnDemandFeaturegroupDTO) {
        fullDTO.setFgType(FeaturegroupXAttr.FGType.ON_DEMAND);
    }
    return fullDTO;
}

Example 5 with OnDemandFeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.ondemand.OnDemandFeaturegroupDTO in project hopsworks by logicalclocks.

the class ConstructorController method getOnDemandAliases.

// TODO(Fabio): does it make sense to this in the same pass as where we generate the table nodes?
// or does the code becomes even more complicated?
public List<OnDemandFeatureGroupAliasDTO> getOnDemandAliases(Users user, Project project, Query query, List<OnDemandFeatureGroupAliasDTO> aliases) throws FeaturestoreException, ServiceException {
    if (query.getFeaturegroup().getFeaturegroupType() == FeaturegroupType.ON_DEMAND_FEATURE_GROUP) {
        FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO = storageConnectorController.convertToConnectorDTO(user, project, query.getFeaturegroup().getOnDemandFeaturegroup().getFeaturestoreConnector());
        OnDemandFeaturegroupDTO onDemandFeaturegroupDTO = new OnDemandFeaturegroupDTO(query.getFeaturegroup(), featurestoreStorageConnectorDTO);
        try {
            String path = featuregroupController.getFeatureGroupLocation(query.getFeaturegroup());
            onDemandFeaturegroupDTO.setLocation(featurestoreUtils.prependNameNode(path));
        } catch (ServiceDiscoveryException e) {
            throw new ServiceException(RESTCodes.ServiceErrorCode.SERVICE_NOT_FOUND, Level.SEVERE);
        }
        aliases.add(new OnDemandFeatureGroupAliasDTO(query.getAs(), onDemandFeaturegroupDTO));
    }
    if (query.getJoins() != null && !query.getJoins().isEmpty()) {
        for (Join join : query.getJoins()) {
            getOnDemandAliases(user, project, join.getRightQuery(), aliases);
        }
    }
    return aliases;
}