Examples with FeaturestoreADLSConnector io.hops.hopsworks.persistence.entity.featurestore.storageconnector.adls.FeaturestoreADLSConnector used on opensource projects

Example 1 with FeaturestoreADLSConnector

use of io.hops.hopsworks.persistence.entity.featurestore.storageconnector.adls.FeaturestoreADLSConnector in project hopsworks by logicalclocks.

the class FeaturestoreADLSConnectorController method getADLConnectorDTO.

public FeaturestoreADLSConnectorDTO getADLConnectorDTO(FeaturestoreConnector featurestoreConnector) throws FeaturestoreException {
    FeaturestoreADLSConnectorDTO adlsConnectorDTO = new FeaturestoreADLSConnectorDTO(featurestoreConnector);
    String serviceCredential = storageConnectorUtil.getSecret(featurestoreConnector.getAdlsConnector().getServiceCredentialSecret(), String.class);
    adlsConnectorDTO.setServiceCredential(serviceCredential);
    // Spark/Hadoop-Azure requries these options for setting up the reader/writer
    FeaturestoreADLSConnector adlsConnector = featurestoreConnector.getAdlsConnector();
    if (adlsConnector.getGeneration() == 1) {
        adlsConnectorDTO.setSparkOptions(getSparkOptionsGen1(adlsConnector, serviceCredential));
    } else if (adlsConnector.getGeneration() == 2) {
        adlsConnectorDTO.setSparkOptions(getSparkOptionsGen2(adlsConnector, serviceCredential));
    }
    return adlsConnectorDTO;
}

Example 2 with FeaturestoreADLSConnector

use of io.hops.hopsworks.persistence.entity.featurestore.storageconnector.adls.FeaturestoreADLSConnector in project hopsworks by logicalclocks.

the class ExternalTrainingDatasetController method buildDatasetPathADL.

private String buildDatasetPathADL(TrainingDataset trainingDataset) {
    FeaturestoreADLSConnector adlsConnector = trainingDataset.getExternalTrainingDataset().getFeaturestoreConnector().getAdlsConnector();
    String directory = Strings.isNullOrEmpty(trainingDataset.getExternalTrainingDataset().getPath()) ? "/" : trainingDataset.getExternalTrainingDataset().getPath();
    String scheme = adlsConnector.getGeneration() == 1 ? ADL_SCHEME : ABFSS_SCHEME;
    String hostname = adlsConnector.getGeneration() == 1 ? adlsConnector.getAccountName() + ADL_URI_SUFFIX : adlsConnector.getContainerName() + "@" + adlsConnector.getAccountName() + ABFSS_URI_SUFFIX;
    return scheme + hostname + directory + trainingDataset.getName() + "_" + trainingDataset.getVersion();
}

Example 3 with FeaturestoreADLSConnector

use of io.hops.hopsworks.persistence.entity.featurestore.storageconnector.adls.FeaturestoreADLSConnector in project hopsworks by logicalclocks.

the class FeaturestoreADLSConnectorController method createADLConnector.

public FeaturestoreADLSConnector createADLConnector(Users user, Project project, Featurestore featurestore, FeaturestoreADLSConnectorDTO adlConnectorDTO) throws FeaturestoreException, ProjectException, UserException {
    verifyConnectorDTO(adlConnectorDTO);
    String secretName = storageConnectorUtil.createSecretName(featurestore.getId(), adlConnectorDTO.getName(), adlConnectorDTO.getStorageConnectorType());
    Secret secret = storageConnectorUtil.createProjectSecret(user, secretName, featurestore, adlConnectorDTO.getServiceCredential());
    FeaturestoreADLSConnector adlConnector = new FeaturestoreADLSConnector();
    adlConnector.setGeneration(adlConnectorDTO.getGeneration());
    adlConnector.setDirectoryId(adlConnectorDTO.getDirectoryId());
    adlConnector.setApplicationId(adlConnectorDTO.getApplicationId());
    adlConnector.setServiceCredentialSecret(secret);
    adlConnector.setAccountName(adlConnectorDTO.getAccountName());
    adlConnector.setContainerName(adlConnectorDTO.getContainerName());
    return adlConnector;
}