Examples with FeaturestoreConnector io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector used on opensource projects

Example 1 with FeaturestoreConnector

use of io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector in project hopsworks by logicalclocks.

the class FeaturestoreStorageConnectorController method deleteConnectorWithName.

// The transaction here is required otherwise when calling the remove the entity is not going to be managed anymore
@TransactionAttribute(TransactionAttributeType.REQUIRED)
public void deleteConnectorWithName(Users user, Project project, String connectorName, Featurestore featurestore) throws UserException {
    validateUser(user, featurestore);
    Optional<FeaturestoreConnector> featurestoreConnectorOptional = featurestoreConnectorFacade.findByFeaturestoreName(featurestore, connectorName);
    if (!featurestoreConnectorOptional.isPresent()) {
        return;
    }
    FeaturestoreConnector featurestoreConnector = featurestoreConnectorOptional.get();
    featurestoreConnectorFacade.remove(featurestoreConnector);
    activityFacade.persistActivity(ActivityFacade.REMOVED_FEATURESTORE_STORAGE_CONNECTOR + featurestoreConnector.getName(), project, user, ActivityFlag.SERVICE);
}

Example 2 with FeaturestoreConnector

use of io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector in project hopsworks by logicalclocks.

the class OnDemandFeaturegroupController method createOnDemandFeaturegroup.

/**
 * Persists an on demand feature group
 *
 * @param project
 * @param user
 * @param featurestore
 * @param onDemandFeaturegroupDTO the user input data to use when creating the feature group
 * @return the created entity
 */
public OnDemandFeaturegroup createOnDemandFeaturegroup(Featurestore featurestore, OnDemandFeaturegroupDTO onDemandFeaturegroupDTO, Project project, Users user) throws FeaturestoreException {
    // Verify User Input specific for on demand feature groups
    FeaturestoreConnector connector = getStorageConnector(onDemandFeaturegroupDTO.getStorageConnector().getId());
    // We allow users to read an entire S3 bucket for instance and they don't need to provide us with a query
    // However if you are running against a JDBC database, you need to provide a query
    boolean isJDBCType = (connector.getConnectorType() == FeaturestoreConnectorType.JDBC || connector.getConnectorType() == FeaturestoreConnectorType.REDSHIFT || connector.getConnectorType() == FeaturestoreConnectorType.SNOWFLAKE);
    if (connector.getConnectorType() == FeaturestoreConnectorType.KAFKA) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_CREATE_ON_DEMAND_FEATUREGROUP, Level.FINE, connector.getConnectorType() + " storage connectors are not supported as source for on demand " + "feature groups");
    } else if (Strings.isNullOrEmpty(onDemandFeaturegroupDTO.getQuery()) && isJDBCType) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.INVALID_SQL_QUERY, Level.FINE, "SQL Query cannot be empty");
    } else if (!Strings.isNullOrEmpty(onDemandFeaturegroupDTO.getQuery()) && !isJDBCType) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.INVALID_SQL_QUERY, Level.FINE, "SQL query not supported when specifying " + connector.getConnectorType() + " storage connectors");
    } else if (onDemandFeaturegroupDTO.getDataFormat() == null && !isJDBCType) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_ON_DEMAND_DATA_FORMAT, Level.FINE, "Data format required when specifying " + connector.getConnectorType() + " storage connectors");
    }
    // Persist on-demand featuregroup
    OnDemandFeaturegroup onDemandFeaturegroup = new OnDemandFeaturegroup();
    onDemandFeaturegroup.setDescription(onDemandFeaturegroupDTO.getDescription());
    onDemandFeaturegroup.setFeaturestoreConnector(connector);
    onDemandFeaturegroup.setQuery(onDemandFeaturegroupDTO.getQuery());
    onDemandFeaturegroup.setFeatures(convertOnDemandFeatures(onDemandFeaturegroupDTO, onDemandFeaturegroup));
    onDemandFeaturegroup.setInode(createFile(project, user, featurestore, onDemandFeaturegroupDTO));
    onDemandFeaturegroup.setDataFormat(onDemandFeaturegroupDTO.getDataFormat());
    onDemandFeaturegroup.setPath(onDemandFeaturegroupDTO.getPath());
    if (onDemandFeaturegroupDTO.getOptions() != null) {
        onDemandFeaturegroup.setOptions(onDemandFeaturegroupDTO.getOptions().stream().map(o -> new OnDemandOption(onDemandFeaturegroup, o.getName(), o.getValue())).collect(Collectors.toList()));
    }
    onDemandFeaturegroupFacade.persist(onDemandFeaturegroup);
    return onDemandFeaturegroup;
}

Example 3 with FeaturestoreConnector

use of io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector in project hopsworks by logicalclocks.

the class OnlineFeaturestoreController method createJdbcConnectorForOnlineFeaturestore.

/**
 * Utility function for create a JDBC connection to the online featurestore for a particular user.
 *
 * @param onlineDbUsername the db-username of the connection
 * @param featurestore the featurestore metadata
 * @param dbName name of the MySQL database
 * @return DTO of the newly created connector
 * @throws FeaturestoreException
 */
public void createJdbcConnectorForOnlineFeaturestore(String onlineDbUsername, Featurestore featurestore, String dbName) throws FeaturestoreException {
    String connectorName = onlineDbUsername + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX;
    if (featurestoreConnectorFacade.findByFeaturestoreName(featurestore, connectorName).isPresent()) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, "a storage connector with that name already exists");
    }
    FeaturestoreConnector featurestoreConnector = new FeaturestoreConnector();
    featurestoreConnector.setName(connectorName);
    featurestoreConnector.setDescription("JDBC connection to Hopsworks Project Online " + "Feature Store NDB Database for user: " + onlineDbUsername);
    featurestoreConnector.setFeaturestore(featurestore);
    featurestoreConnector.setConnectorType(FeaturestoreConnectorType.JDBC);
    FeaturestoreJdbcConnector featurestoreJdbcConnector = new FeaturestoreJdbcConnector();
    featurestoreJdbcConnector.setConnectionString(settings.getFeaturestoreJdbcUrl() + dbName + "?useSSL=false&allowPublicKeyRetrieval=true");
    List<OptionDTO> arguments = new ArrayList<>();
    arguments.add(new OptionDTO(FeaturestoreConstants.ONLINE_FEATURE_STORE_JDBC_PASSWORD_ARG, FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_PASSWORD_TEMPLATE));
    arguments.add(new OptionDTO(FeaturestoreConstants.ONLINE_FEATURE_STORE_JDBC_USER_ARG, onlineDbUsername));
    arguments.add(new OptionDTO(FeaturestoreConstants.ONLINE_FEATURE_STORE_JDBC_DRIVER_ARG, MYSQL_DRIVER));
    arguments.add(new OptionDTO("isolationLevel", "NONE"));
    arguments.add(new OptionDTO("batchsize", "500"));
    featurestoreJdbcConnector.setArguments(storageConnectorUtil.fromOptions(arguments));
    featurestoreConnector.setJdbcConnector(featurestoreJdbcConnector);
    featurestoreConnectorFacade.update(featurestoreConnector);
}

Example 4 with FeaturestoreConnector

use of io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector in project hopsworks by logicalclocks.

the class TrainingDatasetController method createTrainingDataset.

public TrainingDatasetDTO createTrainingDataset(Users user, Project project, Featurestore featurestore, TrainingDatasetDTO trainingDatasetDTO) throws FeaturestoreException, ProvenanceException, IOException, ServiceException {
    // if version not provided, get latest and increment
    if (trainingDatasetDTO.getVersion() == null) {
        // returns ordered list by desc version
        List<TrainingDataset> tdPrevious = trainingDatasetFacade.findByNameAndFeaturestoreOrderedDescVersion(trainingDatasetDTO.getName(), featurestore);
        if (tdPrevious != null && !tdPrevious.isEmpty()) {
            trainingDatasetDTO.setVersion(tdPrevious.get(0).getVersion() + 1);
        } else {
            trainingDatasetDTO.setVersion(1);
        }
    }
    // Check that training dataset doesn't already exists
    if (trainingDatasetFacade.findByNameVersionAndFeaturestore(trainingDatasetDTO.getName(), trainingDatasetDTO.getVersion(), featurestore).isPresent()) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRAINING_DATASET_ALREADY_EXISTS, Level.FINE, "Training Dataset: " + trainingDatasetDTO.getName() + ", version: " + trainingDatasetDTO.getVersion());
    }
    // If the training dataset is constructed from a query, verify that it compiles correctly
    Query query = null;
    if (trainingDatasetDTO.getQueryDTO() != null) {
        query = constructQuery(trainingDatasetDTO.getQueryDTO(), project, user);
    } else if (trainingDatasetDTO.getFeatures() == null) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRAINING_DATASET_NO_SCHEMA, Level.FINE, "The training dataset doesn't have any feature");
    }
    // Verify input
    inputValidation.validate(trainingDatasetDTO, query);
    Inode inode = null;
    FeaturestoreConnector featurestoreConnector;
    if (trainingDatasetDTO.getTrainingDatasetType() == TrainingDatasetType.HOPSFS_TRAINING_DATASET) {
        if (trainingDatasetDTO.getStorageConnector() != null && trainingDatasetDTO.getStorageConnector().getId() != null) {
            featurestoreConnector = featurestoreConnectorFacade.findByIdType(trainingDatasetDTO.getStorageConnector().getId(), FeaturestoreConnectorType.HOPSFS).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.HOPSFS_CONNECTOR_NOT_FOUND, Level.FINE, "HOPSFS Connector: " + trainingDatasetDTO.getStorageConnector().getId()));
        } else {
            featurestoreConnector = getDefaultHopsFSTrainingDatasetConnector(featurestore);
        }
    } else {
        if (trainingDatasetDTO.getStorageConnector() == null) {
            throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_NOT_FOUND, Level.FINE, "Storage connector is empty");
        }
        featurestoreConnector = featurestoreConnectorFacade.findById(trainingDatasetDTO.getStorageConnector().getId()).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_NOT_FOUND, Level.FINE, "Connector: " + trainingDatasetDTO.getStorageConnector().getId()));
    }
    // for HopsFS TD it will either be the default connector already or it will be a connector pointing to another
    // HopsFS Directory
    // for external TD we will use default connector
    Dataset trainingDatasetsFolder;
    if (featurestoreConnector.getHopsfsConnector() != null) {
        trainingDatasetsFolder = featurestoreConnector.getHopsfsConnector().getHopsfsDataset();
    } else {
        trainingDatasetsFolder = getDefaultHopsFSTrainingDatasetConnector(featurestore).getHopsfsConnector().getHopsfsDataset();
    }
    // TODO(Fabio) account for path
    // we allow specifying the path in the training dataset dir, but it is not really used, this option will be
    // deprecated for hopsfs training datasets.
    String trainingDatasetPath = getTrainingDatasetPath(inodeController.getPath(trainingDatasetsFolder.getInode()), trainingDatasetDTO.getName(), trainingDatasetDTO.getVersion());
    DistributedFileSystemOps udfso = null;
    String username = hdfsUsersBean.getHdfsUserName(project, user);
    try {
        udfso = dfs.getDfsOps(username);
        udfso.mkdir(trainingDatasetPath);
        inode = inodeController.getInodeAtPath(trainingDatasetPath);
        TrainingDatasetDTO completeTrainingDatasetDTO = createTrainingDatasetMetadata(user, project, featurestore, trainingDatasetDTO, query, featurestoreConnector, inode);
        fsProvenanceController.trainingDatasetAttachXAttr(trainingDatasetPath, completeTrainingDatasetDTO, udfso);
        return completeTrainingDatasetDTO;
    } finally {
        if (udfso != null) {
            dfs.closeDfsClient(udfso);
        }
    }
}

Example 5 with FeaturestoreConnector

use of io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector in project hopsworks by logicalclocks.

the class TransformationFunctionController method getOrCreatePath.

private String getOrCreatePath(Featurestore featurestore, DistributedFileSystemOps udfso) throws IOException, FeaturestoreException {
    String connectorName = featurestore.getProject().getName() + "_" + Settings.ServiceDataset.TRAININGDATASETS.getName();
    FeaturestoreConnector featurestoreConnector = featurestoreConnectorFacade.findByFeaturestoreName(featurestore, connectorName).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.HOPSFS_CONNECTOR_NOT_FOUND, Level.FINE, "HOPSFS Connector: " + FeaturestoreConnectorType.HOPSFS.name()));
    Dataset trainingDatasetsFolder = featurestoreConnector.getHopsfsConnector().getHopsfsDataset();
    String transformationFunctionDatasetPath = inodeController.getPath(trainingDatasetsFolder.getInode()) + "/" + TRANSFORMATIONFUNCTIONS_FOLDER;
    // This is the case of an old project without TRANSFORMATIONFUNCTION dataset, create it.
    if (!udfso.exists(transformationFunctionDatasetPath)) {
        udfso.mkdir(transformationFunctionDatasetPath);
    }
    return transformationFunctionDatasetPath;
}