Examples with FeaturestoreHopsfsConnectorDTO io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO used on opensource projects

Example 1 with FeaturestoreHopsfsConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO in project hopsworks by logicalclocks.

the class FeaturestoreController method hopsfsTrainingDatasetConnector.

public FeaturestoreStorageConnectorDTO hopsfsTrainingDatasetConnector(Dataset hopsfsDataset) {
    String name = hopsfsDataset.getName();
    String description = "HOPSFS backend for storing Training Datasets of the Hopsworks Feature Store";
    FeaturestoreHopsfsConnectorDTO featurestoreHopsfsConnectorDTO = new FeaturestoreHopsfsConnectorDTO();
    featurestoreHopsfsConnectorDTO.setStorageConnectorType(FeaturestoreConnectorType.HOPSFS);
    featurestoreHopsfsConnectorDTO.setName(name);
    featurestoreHopsfsConnectorDTO.setDescription(description);
    featurestoreHopsfsConnectorDTO.setDatasetName(hopsfsDataset.getName());
    return featurestoreHopsfsConnectorDTO;
}

Example 2 with FeaturestoreHopsfsConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO in project hopsworks by logicalclocks.

the class HopsfsTrainingDatasetController method convertHopsfsTrainingDatasetToDTO.

/**
 * Converts a Hopsfs Training Dataset entity into a DTO representation
 *
 * @param trainingDataset the entity to convert
 * @return the converted DTO representation
 */
public TrainingDatasetDTO convertHopsfsTrainingDatasetToDTO(TrainingDatasetDTO trainingDatasetDTO, TrainingDataset trainingDataset) throws ServiceException {
    Service namenodeService;
    try {
        namenodeService = serviceDiscoveryController.getAnyAddressOfServiceWithDNS(ServiceDiscoveryController.HopsworksService.RPC_NAMENODE);
    } catch (ServiceDiscoveryException e) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.SERVICE_NOT_FOUND, Level.SEVERE, "Could not find namenode service", e.getMessage(), e);
    }
    HopsfsTrainingDataset hopsfsTrainingDataset = trainingDataset.getHopsfsTrainingDataset();
    trainingDatasetDTO.setLocation(new Path(DistributedFileSystemOps.HOPSFS_SCHEME, namenodeService.getAddress() + ":" + namenodeService.getPort(), inodeController.getPath(hopsfsTrainingDataset.getInode())).toString());
    trainingDatasetDTO.setInodeId(hopsfsTrainingDataset.getInode().getId());
    FeaturestoreHopsfsConnectorDTO hopsfsConnectorDTO = new FeaturestoreHopsfsConnectorDTO(hopsfsTrainingDataset.getFeaturestoreConnector());
    trainingDatasetDTO.setStorageConnector(hopsfsConnectorDTO);
    return trainingDatasetDTO;
}

Example 3 with FeaturestoreHopsfsConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO in project hopsworks by logicalclocks.

the class FeaturestoreStorageConnectorController method createStorageConnector.

/**
 * Creates a new Storage Connector of a specific type in a feature store
 *
 * @param user the user making the request
 * @param featurestore the featurestore to create the new connector
 * @param featurestoreStorageConnectorDTO the data to use when creating the storage connector
 * @return A JSON/XML DTOs representation of the created storage connector
 * @throws FeaturestoreException
 */
public FeaturestoreStorageConnectorDTO createStorageConnector(Users user, Project project, Featurestore featurestore, FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO) throws FeaturestoreException, UserException, ProjectException {
    validateUser(user, featurestore);
    if (featurestoreConnectorFacade.findByFeaturestoreName(featurestore, featurestoreStorageConnectorDTO.getName()).isPresent()) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, "Storage connector with the same name already exists. Name=" + featurestoreStorageConnectorDTO.getName());
    }
    FeaturestoreConnector featurestoreConnector = new FeaturestoreConnector();
    verifyName(featurestoreStorageConnectorDTO);
    featurestoreConnector.setName(featurestoreStorageConnectorDTO.getName());
    verifyDescription(featurestoreStorageConnectorDTO);
    featurestoreConnector.setDescription(featurestoreStorageConnectorDTO.getDescription());
    featurestoreConnector.setFeaturestore(featurestore);
    switch(featurestoreStorageConnectorDTO.getStorageConnectorType()) {
        case HOPSFS:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.HOPSFS);
            featurestoreConnector.setHopsfsConnector(hopsfsConnectorController.createFeaturestoreHopsfsConnector(featurestore, (FeaturestoreHopsfsConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case S3:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.S3);
            featurestoreConnector.setS3Connector(s3ConnectorController.createFeaturestoreS3Connector(user, featurestore, (FeaturestoreS3ConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case JDBC:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.JDBC);
            featurestoreConnector.setJdbcConnector(jdbcConnectorController.createFeaturestoreJdbcConnector((FeaturestoreJdbcConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case REDSHIFT:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.REDSHIFT);
            featurestoreConnector.setRedshiftConnector(redshiftConnectorController.createFeaturestoreRedshiftConnector(user, featurestore, (FeaturestoreRedshiftConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case ADLS:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.ADLS);
            featurestoreConnector.setAdlsConnector(adlsConnectorController.createADLConnector(user, project, featurestore, (FeaturestoreADLSConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case SNOWFLAKE:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.SNOWFLAKE);
            featurestoreConnector.setSnowflakeConnector(snowflakeConnectorController.createConnector(user, featurestore, (FeaturestoreSnowflakeConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case KAFKA:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.KAFKA);
            featurestoreConnector.setKafkaConnector(kafkaConnectorController.createConnector(user, featurestore, (FeatureStoreKafkaConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        default:
            // We should not reach this point
            throw new IllegalArgumentException("Feature Store connector type not recognized");
    }
    // Update object to populate id (auto-increment) information
    featurestoreConnector = featurestoreConnectorFacade.update(featurestoreConnector);
    activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + featurestoreConnector.getName(), project, user, ActivityFlag.SERVICE);
    return convertToConnectorDTO(user, project, featurestoreConnector);
}

Example 4 with FeaturestoreHopsfsConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO in project hopsworks by logicalclocks.

the class FeaturestoreStorageConnectorController method updateStorageConnector.

@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Transactional(rollbackOn = FeaturestoreException.class)
public void updateStorageConnector(Users user, Project project, Featurestore featurestore, FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO, String connectorName) throws FeaturestoreException, UserException, ProjectException {
    validateUser(user, featurestore);
    if (!connectorName.equalsIgnoreCase(featurestoreStorageConnectorDTO.getName())) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, Level.FINE, "Can not update connector name.");
    }
    FeaturestoreConnector featurestoreConnector = featurestoreConnectorFacade.findByFeaturestoreName(featurestore, connectorName).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_NOT_FOUND, Level.FINE, "Cannot find storage connector with name: " + connectorName));
    verifyDescription(featurestoreStorageConnectorDTO);
    featurestoreConnector.setDescription(featurestoreStorageConnectorDTO.getDescription());
    switch(featurestoreConnector.getConnectorType()) {
        case HOPSFS:
            featurestoreConnector.setHopsfsConnector(hopsfsConnectorController.updateFeaturestoreHopsfsConnector(featurestore, (FeaturestoreHopsfsConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getHopsfsConnector()));
            break;
        case S3:
            featurestoreConnector.setS3Connector(s3ConnectorController.updateFeaturestoreS3Connector(user, featurestore, (FeaturestoreS3ConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getS3Connector()));
            break;
        case JDBC:
            featurestoreConnector.setJdbcConnector(jdbcConnectorController.updateFeaturestoreJdbcConnector((FeaturestoreJdbcConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getJdbcConnector()));
            break;
        case REDSHIFT:
            featurestoreConnector.setRedshiftConnector(redshiftConnectorController.updateFeaturestoreRedshiftConnector(user, featurestore, (FeaturestoreRedshiftConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getRedshiftConnector()));
            break;
        case ADLS:
            featurestoreConnector.setAdlsConnector(adlsConnectorController.updateAdlConnector(user, featurestore, (FeaturestoreADLSConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getAdlsConnector()));
            break;
        case SNOWFLAKE:
            featurestoreConnector.setSnowflakeConnector(snowflakeConnectorController.updateConnector(user, (FeaturestoreSnowflakeConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getSnowflakeConnector()));
            break;
        case KAFKA:
            featurestoreConnector.setKafkaConnector((kafkaConnectorController.updateConnector(user, featurestore, (FeatureStoreKafkaConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getKafkaConnector())));
            break;
        default:
            // We should not reach this point
            throw new IllegalArgumentException("Feature Store connector type not recognized");
    }
    featurestoreConnector = featurestoreConnectorFacade.update(featurestoreConnector);
    activityFacade.persistActivity(ActivityFacade.UPDATED_FEATURESTORE_STORAGE_CONNECTOR + featurestoreConnector.getName(), project, user, ActivityFlag.SERVICE);
}