Examples with FeaturestoreStorageConnectorDTO io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO used on opensource projects

Example 1 with FeaturestoreStorageConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO in project hopsworks by logicalclocks.

the class FeaturestoreStorageConnectorService method createNewStorageConnector.

/**
 * Endpoint for creating a storage connector with a particular type in a feature store
 *
 * @return a JSON representation of the connector
 * @throws FeaturestoreException
 */
@POST
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Create a new storage connector for the feature store", response = FeaturestoreStorageConnectorDTO.class)
public Response createNewStorageConnector(@Context SecurityContext sc, FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO) throws FeaturestoreException, UserException, ProjectException {
    if (featurestoreStorageConnectorDTO == null) {
        throw new IllegalArgumentException("Input JSON for creating a new Feature Store Storage Connector Cannot be " + "null");
    }
    Users user = jWTHelper.getUserPrincipal(sc);
    FeaturestoreStorageConnectorDTO createdFeaturestoreStorageConnectorDTO = storageConnectorController.createStorageConnector(user, project, featurestore, featurestoreStorageConnectorDTO);
    GenericEntity<FeaturestoreStorageConnectorDTO> featurestoreStorageConnectorDTOGenericEntity = new GenericEntity<FeaturestoreStorageConnectorDTO>(createdFeaturestoreStorageConnectorDTO) {
    };
    return noCacheResponse.getNoCacheResponseBuilder(Response.Status.CREATED).entity(featurestoreStorageConnectorDTOGenericEntity).build();
}

Example 2 with FeaturestoreStorageConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO in project hopsworks by logicalclocks.

the class FeaturestoreStorageConnectorService method getStorageConnectors.

/**
 * Endpoint for getting all storage connectors in a featurestore
 *
 * @return a JSON representation of all the storage connectors in the feature store
 */
@GET
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.FEATURESTORE }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Get all storage connectors of a feature store", response = FeaturestoreStorageConnectorDTO.class, responseContainer = "List")
public Response getStorageConnectors(@Context SecurityContext sc) throws FeaturestoreException {
    Users user = jWTHelper.getUserPrincipal(sc);
    List<FeaturestoreStorageConnectorDTO> featurestoreStorageConnectorDTOS = storageConnectorController.getConnectorsForFeaturestore(user, project, featurestore);
    GenericEntity<List<FeaturestoreStorageConnectorDTO>> featurestoreStorageConnectorsGeneric = new GenericEntity<List<FeaturestoreStorageConnectorDTO>>(featurestoreStorageConnectorDTOS) {
    };
    return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(featurestoreStorageConnectorsGeneric).build();
}

Example 3 with FeaturestoreStorageConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO in project hopsworks by logicalclocks.

the class FeaturegroupController method convertFeaturegrouptoDTO.

/**
 * Convert a featuregroup entity to a DTO representation
 *
 * @param featuregroup the entity to convert
 * @return a DTO representation of the entity
 */
private FeaturegroupDTO convertFeaturegrouptoDTO(Featuregroup featuregroup, Project project, Users user) throws FeaturestoreException, ServiceException {
    String featurestoreName = featurestoreFacade.getHiveDbName(featuregroup.getFeaturestore().getHiveDbId());
    switch(featuregroup.getFeaturegroupType()) {
        case CACHED_FEATURE_GROUP:
            CachedFeaturegroupDTO cachedFeaturegroupDTO = cachedFeaturegroupController.convertCachedFeaturegroupToDTO(featuregroup, project, user);
            cachedFeaturegroupDTO.setFeaturestoreName(featurestoreName);
            return cachedFeaturegroupDTO;
        case ON_DEMAND_FEATURE_GROUP:
            FeaturestoreStorageConnectorDTO storageConnectorDTO = connectorController.convertToConnectorDTO(user, project, featuregroup.getOnDemandFeaturegroup().getFeaturestoreConnector());
            OnDemandFeaturegroupDTO onDemandFeaturegroupDTO = new OnDemandFeaturegroupDTO(featurestoreName, featuregroup, storageConnectorDTO);
            try {
                String path = getFeatureGroupLocation(featuregroup);
                String location = featurestoreUtils.prependNameNode(path);
                onDemandFeaturegroupDTO.setLocation(location);
            } catch (ServiceDiscoveryException e) {
                throw new ServiceException(RESTCodes.ServiceErrorCode.SERVICE_NOT_FOUND, Level.SEVERE);
            }
            return onDemandFeaturegroupDTO;
        default:
            throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_FEATUREGROUP_TYPE.getMessage() + ", Recognized Feature group types are: " + FeaturegroupType.ON_DEMAND_FEATURE_GROUP + ", and: " + FeaturegroupType.CACHED_FEATURE_GROUP + ". The provided feature group type was not recognized: " + featuregroup.getFeaturegroupType());
    }
}

Example 4 with FeaturestoreStorageConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO in project hopsworks by logicalclocks.

the class ConstructorController method getOnDemandAliases.

// TODO(Fabio): does it make sense to this in the same pass as where we generate the table nodes?
// or does the code becomes even more complicated?
public List<OnDemandFeatureGroupAliasDTO> getOnDemandAliases(Users user, Project project, Query query, List<OnDemandFeatureGroupAliasDTO> aliases) throws FeaturestoreException, ServiceException {
    if (query.getFeaturegroup().getFeaturegroupType() == FeaturegroupType.ON_DEMAND_FEATURE_GROUP) {
        FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO = storageConnectorController.convertToConnectorDTO(user, project, query.getFeaturegroup().getOnDemandFeaturegroup().getFeaturestoreConnector());
        OnDemandFeaturegroupDTO onDemandFeaturegroupDTO = new OnDemandFeaturegroupDTO(query.getFeaturegroup(), featurestoreStorageConnectorDTO);
        try {
            String path = featuregroupController.getFeatureGroupLocation(query.getFeaturegroup());
            onDemandFeaturegroupDTO.setLocation(featurestoreUtils.prependNameNode(path));
        } catch (ServiceDiscoveryException e) {
            throw new ServiceException(RESTCodes.ServiceErrorCode.SERVICE_NOT_FOUND, Level.SEVERE);
        }
        aliases.add(new OnDemandFeatureGroupAliasDTO(query.getAs(), onDemandFeaturegroupDTO));
    }
    if (query.getJoins() != null && !query.getJoins().isEmpty()) {
        for (Join join : query.getJoins()) {
            getOnDemandAliases(user, project, join.getRightQuery(), aliases);
        }
    }
    return aliases;
}

Example 5 with FeaturestoreStorageConnectorDTO

use of io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO in project hopsworks by logicalclocks.

the class FeaturestoreController method createOfflineJdbcConnector.

public FeaturestoreStorageConnectorDTO createOfflineJdbcConnector(String databaseName) throws FeaturestoreException {
    String hiveEndpoint = "";
    try {
        hiveEndpoint = serviceDiscoveryController.constructServiceFQDNWithPort(ServiceDiscoveryController.HopsworksService.HIVE_SERVER_TLS);
    } catch (ServiceDiscoveryException e) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_NOT_FOUND, Level.SEVERE, "Could not create Hive connection string", e.getMessage(), e);
    }
    String connectionString = HiveController.HIVE_JDBC_PREFIX + hiveEndpoint + "/" + databaseName + ";auth=noSasl;ssl=true;twoWay=true;";
    List<OptionDTO> arguments = FeaturestoreConstants.OFFLINE_JDBC_CONNECTOR_ARGS.stream().map(arg -> new OptionDTO(arg, null)).collect(Collectors.toList());
    FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO = new FeaturestoreJdbcConnectorDTO();
    featurestoreJdbcConnectorDTO.setStorageConnectorType(FeaturestoreConnectorType.JDBC);
    featurestoreJdbcConnectorDTO.setName(databaseName);
    featurestoreJdbcConnectorDTO.setDescription("JDBC connector for the Offline Feature Store");
    featurestoreJdbcConnectorDTO.setConnectionString(connectionString);
    featurestoreJdbcConnectorDTO.setArguments(arguments);
    return featurestoreJdbcConnectorDTO;
}