Examples with FeaturestoreS3ConnectorDTO io.hops.hopsworks.common.featurestore.storageconnectors.s3.FeaturestoreS3ConnectorDTO used on opensource projects

Example 1 with FeaturestoreS3ConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.s3.FeaturestoreS3ConnectorDTO in project hopsworks by logicalclocks.

the class FeaturestoreStorageConnectorController method createStorageConnector.

/**
 * Creates a new Storage Connector of a specific type in a feature store
 *
 * @param user the user making the request
 * @param featurestore the featurestore to create the new connector
 * @param featurestoreStorageConnectorDTO the data to use when creating the storage connector
 * @return A JSON/XML DTOs representation of the created storage connector
 * @throws FeaturestoreException
 */
public FeaturestoreStorageConnectorDTO createStorageConnector(Users user, Project project, Featurestore featurestore, FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO) throws FeaturestoreException, UserException, ProjectException {
    validateUser(user, featurestore);
    if (featurestoreConnectorFacade.findByFeaturestoreName(featurestore, featurestoreStorageConnectorDTO.getName()).isPresent()) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, "Storage connector with the same name already exists. Name=" + featurestoreStorageConnectorDTO.getName());
    }
    FeaturestoreConnector featurestoreConnector = new FeaturestoreConnector();
    verifyName(featurestoreStorageConnectorDTO);
    featurestoreConnector.setName(featurestoreStorageConnectorDTO.getName());
    verifyDescription(featurestoreStorageConnectorDTO);
    featurestoreConnector.setDescription(featurestoreStorageConnectorDTO.getDescription());
    featurestoreConnector.setFeaturestore(featurestore);
    switch(featurestoreStorageConnectorDTO.getStorageConnectorType()) {
        case HOPSFS:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.HOPSFS);
            featurestoreConnector.setHopsfsConnector(hopsfsConnectorController.createFeaturestoreHopsfsConnector(featurestore, (FeaturestoreHopsfsConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case S3:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.S3);
            featurestoreConnector.setS3Connector(s3ConnectorController.createFeaturestoreS3Connector(user, featurestore, (FeaturestoreS3ConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case JDBC:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.JDBC);
            featurestoreConnector.setJdbcConnector(jdbcConnectorController.createFeaturestoreJdbcConnector((FeaturestoreJdbcConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case REDSHIFT:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.REDSHIFT);
            featurestoreConnector.setRedshiftConnector(redshiftConnectorController.createFeaturestoreRedshiftConnector(user, featurestore, (FeaturestoreRedshiftConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case ADLS:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.ADLS);
            featurestoreConnector.setAdlsConnector(adlsConnectorController.createADLConnector(user, project, featurestore, (FeaturestoreADLSConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case SNOWFLAKE:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.SNOWFLAKE);
            featurestoreConnector.setSnowflakeConnector(snowflakeConnectorController.createConnector(user, featurestore, (FeaturestoreSnowflakeConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        case KAFKA:
            featurestoreConnector.setConnectorType(FeaturestoreConnectorType.KAFKA);
            featurestoreConnector.setKafkaConnector(kafkaConnectorController.createConnector(user, featurestore, (FeatureStoreKafkaConnectorDTO) featurestoreStorageConnectorDTO));
            break;
        default:
            // We should not reach this point
            throw new IllegalArgumentException("Feature Store connector type not recognized");
    }
    // Update object to populate id (auto-increment) information
    featurestoreConnector = featurestoreConnectorFacade.update(featurestoreConnector);
    activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + featurestoreConnector.getName(), project, user, ActivityFlag.SERVICE);
    return convertToConnectorDTO(user, project, featurestoreConnector);
}

Example 2 with FeaturestoreS3ConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.s3.FeaturestoreS3ConnectorDTO in project hopsworks by logicalclocks.

the class FeaturestoreStorageConnectorController method updateStorageConnector.

@TransactionAttribute(TransactionAttributeType.REQUIRED)
@Transactional(rollbackOn = FeaturestoreException.class)
public void updateStorageConnector(Users user, Project project, Featurestore featurestore, FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO, String connectorName) throws FeaturestoreException, UserException, ProjectException {
    validateUser(user, featurestore);
    if (!connectorName.equalsIgnoreCase(featurestoreStorageConnectorDTO.getName())) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, Level.FINE, "Can not update connector name.");
    }
    FeaturestoreConnector featurestoreConnector = featurestoreConnectorFacade.findByFeaturestoreName(featurestore, connectorName).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_NOT_FOUND, Level.FINE, "Cannot find storage connector with name: " + connectorName));
    verifyDescription(featurestoreStorageConnectorDTO);
    featurestoreConnector.setDescription(featurestoreStorageConnectorDTO.getDescription());
    switch(featurestoreConnector.getConnectorType()) {
        case HOPSFS:
            featurestoreConnector.setHopsfsConnector(hopsfsConnectorController.updateFeaturestoreHopsfsConnector(featurestore, (FeaturestoreHopsfsConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getHopsfsConnector()));
            break;
        case S3:
            featurestoreConnector.setS3Connector(s3ConnectorController.updateFeaturestoreS3Connector(user, featurestore, (FeaturestoreS3ConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getS3Connector()));
            break;
        case JDBC:
            featurestoreConnector.setJdbcConnector(jdbcConnectorController.updateFeaturestoreJdbcConnector((FeaturestoreJdbcConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getJdbcConnector()));
            break;
        case REDSHIFT:
            featurestoreConnector.setRedshiftConnector(redshiftConnectorController.updateFeaturestoreRedshiftConnector(user, featurestore, (FeaturestoreRedshiftConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getRedshiftConnector()));
            break;
        case ADLS:
            featurestoreConnector.setAdlsConnector(adlsConnectorController.updateAdlConnector(user, featurestore, (FeaturestoreADLSConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getAdlsConnector()));
            break;
        case SNOWFLAKE:
            featurestoreConnector.setSnowflakeConnector(snowflakeConnectorController.updateConnector(user, (FeaturestoreSnowflakeConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getSnowflakeConnector()));
            break;
        case KAFKA:
            featurestoreConnector.setKafkaConnector((kafkaConnectorController.updateConnector(user, featurestore, (FeatureStoreKafkaConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getKafkaConnector())));
            break;
        default:
            // We should not reach this point
            throw new IllegalArgumentException("Feature Store connector type not recognized");
    }
    featurestoreConnector = featurestoreConnectorFacade.update(featurestoreConnector);
    activityFacade.persistActivity(ActivityFacade.UPDATED_FEATURESTORE_STORAGE_CONNECTOR + featurestoreConnector.getName(), project, user, ActivityFlag.SERVICE);
}