Examples with OnDemandOption io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand.OnDemandOption used on opensource projects

Example 1 with OnDemandOption

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand.OnDemandOption in project hopsworks by logicalclocks.

the class OnDemandFeaturegroupController method createOnDemandFeaturegroup.

/**
 * Persists an on demand feature group
 *
 * @param project
 * @param user
 * @param featurestore
 * @param onDemandFeaturegroupDTO the user input data to use when creating the feature group
 * @return the created entity
 */
public OnDemandFeaturegroup createOnDemandFeaturegroup(Featurestore featurestore, OnDemandFeaturegroupDTO onDemandFeaturegroupDTO, Project project, Users user) throws FeaturestoreException {
    // Verify User Input specific for on demand feature groups
    FeaturestoreConnector connector = getStorageConnector(onDemandFeaturegroupDTO.getStorageConnector().getId());
    // We allow users to read an entire S3 bucket for instance and they don't need to provide us with a query
    // However if you are running against a JDBC database, you need to provide a query
    boolean isJDBCType = (connector.getConnectorType() == FeaturestoreConnectorType.JDBC || connector.getConnectorType() == FeaturestoreConnectorType.REDSHIFT || connector.getConnectorType() == FeaturestoreConnectorType.SNOWFLAKE);
    if (connector.getConnectorType() == FeaturestoreConnectorType.KAFKA) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_CREATE_ON_DEMAND_FEATUREGROUP, Level.FINE, connector.getConnectorType() + " storage connectors are not supported as source for on demand " + "feature groups");
    } else if (Strings.isNullOrEmpty(onDemandFeaturegroupDTO.getQuery()) && isJDBCType) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.INVALID_SQL_QUERY, Level.FINE, "SQL Query cannot be empty");
    } else if (!Strings.isNullOrEmpty(onDemandFeaturegroupDTO.getQuery()) && !isJDBCType) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.INVALID_SQL_QUERY, Level.FINE, "SQL query not supported when specifying " + connector.getConnectorType() + " storage connectors");
    } else if (onDemandFeaturegroupDTO.getDataFormat() == null && !isJDBCType) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_ON_DEMAND_DATA_FORMAT, Level.FINE, "Data format required when specifying " + connector.getConnectorType() + " storage connectors");
    }
    // Persist on-demand featuregroup
    OnDemandFeaturegroup onDemandFeaturegroup = new OnDemandFeaturegroup();
    onDemandFeaturegroup.setDescription(onDemandFeaturegroupDTO.getDescription());
    onDemandFeaturegroup.setFeaturestoreConnector(connector);
    onDemandFeaturegroup.setQuery(onDemandFeaturegroupDTO.getQuery());
    onDemandFeaturegroup.setFeatures(convertOnDemandFeatures(onDemandFeaturegroupDTO, onDemandFeaturegroup));
    onDemandFeaturegroup.setInode(createFile(project, user, featurestore, onDemandFeaturegroupDTO));
    onDemandFeaturegroup.setDataFormat(onDemandFeaturegroupDTO.getDataFormat());
    onDemandFeaturegroup.setPath(onDemandFeaturegroupDTO.getPath());
    if (onDemandFeaturegroupDTO.getOptions() != null) {
        onDemandFeaturegroup.setOptions(onDemandFeaturegroupDTO.getOptions().stream().map(o -> new OnDemandOption(onDemandFeaturegroup, o.getName(), o.getValue())).collect(Collectors.toList()));
    }
    onDemandFeaturegroupFacade.persist(onDemandFeaturegroup);
    return onDemandFeaturegroup;
}