Search in sources :

Example 11 with UserCerts

use of io.hops.hopsworks.persistence.entity.certificates.UserCerts in project hopsworks by logicalclocks.

the class HopsUtils method copyProjectUserCerts.

/**
 * Utility method that copies project user certificates from the Database, to
 * either hdfs to be passed as LocalResources to the YarnJob or to used
 * by another method.
 *
 * @param project
 * @param username
 * @param localTmpDir
 * @param remoteTmpDir
 * @param jobType
 * @param dfso
 * @param projectLocalResources
 * @param applicationId
 */
public static void copyProjectUserCerts(Project project, String username, String localTmpDir, String remoteTmpDir, JobType jobType, DistributedFileSystemOps dfso, List<LocalResourceDTO> projectLocalResources, String applicationId, CertificateMaterializer certMat) {
    // Let the Certificate Materializer handle the certificates
    UserCerts userCert = new UserCerts(project.getName(), username);
    try {
        certMat.materializeCertificatesLocal(username, project.getName());
        CertificateMaterializer.CryptoMaterial material = certMat.getUserMaterial(username, project.getName());
        userCert.setUserKey(material.getKeyStore().array());
        userCert.setUserCert(material.getTrustStore().array());
        userCert.setUserKeyPwd(new String(material.getPassword()));
    } catch (IOException | CryptoPasswordNotFoundException ex) {
        throw new RuntimeException("Could not materialize user certificates", ex);
    }
    // Check if the user certificate was actually retrieved
    if (userCert.getUserCert() != null && userCert.getUserCert().length > 0 && userCert.getUserKey() != null && userCert.getUserKey().length > 0) {
        Map<String, byte[]> certFiles = new HashMap<>();
        certFiles.put(Settings.T_CERTIFICATE, userCert.getUserCert());
        certFiles.put(Settings.K_CERTIFICATE, userCert.getUserKey());
        try {
            String kCertName = HopsUtils.getProjectKeystoreName(project.getName(), username);
            String tCertName = HopsUtils.getProjectTruststoreName(project.getName(), username);
            String passName = getProjectMaterialPasswordName(project.getName(), username);
            try {
                if (jobType != null) {
                    switch(jobType) {
                        case PYSPARK:
                        case SPARK:
                            Map<String, File> certs = new HashMap<>();
                            certs.put(Settings.K_CERTIFICATE, new File(localTmpDir + File.separator + kCertName));
                            certs.put(Settings.T_CERTIFICATE, new File(localTmpDir + File.separator + tCertName));
                            certs.put(Settings.CRYPTO_MATERIAL_PASSWORD, new File(localTmpDir + File.separator + passName));
                            for (Map.Entry<String, File> entry : certs.entrySet()) {
                                // by the YarnJob
                                if (!dfso.exists(remoteTmpDir)) {
                                    Path remoteTmpDirPath = new Path(remoteTmpDir);
                                    dfso.mkdir(remoteTmpDirPath, FsPermission.getDirDefault());
                                    dfso.setPermission(remoteTmpDirPath, FsPermissions.rwxrwxrwx);
                                }
                                // Put project certificates in its own dir
                                String certUser = project.getName() + "__" + username;
                                String remoteTmpProjDir = remoteTmpDir + File.separator + certUser;
                                if (!dfso.exists(remoteTmpProjDir)) {
                                    Path remoteTmpProjDirPath = new Path(remoteTmpProjDir);
                                    dfso.mkdir(remoteTmpProjDirPath, FsPermission.getDirDefault());
                                    dfso.setPermission(remoteTmpProjDirPath, FsPermissions.rwxrwx___);
                                    dfso.setOwner(remoteTmpProjDirPath, certUser, certUser);
                                }
                                String remoteProjAppDir = remoteTmpProjDir + File.separator + applicationId;
                                Path remoteProjAppPath = new Path(remoteProjAppDir);
                                if (!dfso.exists(remoteProjAppDir)) {
                                    dfso.mkdir(remoteProjAppPath, FsPermission.getDirDefault());
                                    dfso.setPermission(remoteProjAppPath, FsPermissions.rwxrwx___);
                                    dfso.setOwner(remoteProjAppPath, certUser, certUser);
                                }
                                dfso.copyToHDFSFromLocal(false, entry.getValue().getAbsolutePath(), remoteProjAppDir + File.separator + entry.getValue().getName());
                                dfso.setPermission(new Path(remoteProjAppDir + File.separator + entry.getValue().getName()), FsPermissions.rwx______);
                                dfso.setOwner(new Path(remoteProjAppDir + File.separator + entry.getValue().getName()), certUser, certUser);
                                projectLocalResources.add(new LocalResourceDTO(entry.getKey(), "hdfs://" + remoteProjAppDir + File.separator + entry.getValue().getName(), LocalResourceVisibility.APPLICATION.toString(), LocalResourceType.FILE.toString(), null));
                            }
                            break;
                        default:
                            break;
                    }
                }
            } catch (IOException ex) {
                LOG.log(Level.SEVERE, "Error writing project user certificates to local fs", ex);
            }
        } finally {
            if (jobType != null) {
                certMat.removeCertificatesLocal(username, project.getName());
            }
        }
    }
}
Also used : Path(org.apache.hadoop.fs.Path) HashMap(java.util.HashMap) CertificateMaterializer(io.hops.hopsworks.common.security.CertificateMaterializer) IOException(java.io.IOException) LocalResourceDTO(io.hops.hopsworks.persistence.entity.jobs.configuration.yarn.LocalResourceDTO) CryptoPasswordNotFoundException(io.hops.hopsworks.exceptions.CryptoPasswordNotFoundException) UserCerts(io.hops.hopsworks.persistence.entity.certificates.UserCerts) File(java.io.File) Map(java.util.Map) HashMap(java.util.HashMap)

Aggregations

UserCerts (io.hops.hopsworks.persistence.entity.certificates.UserCerts)11 HashMap (java.util.HashMap)3 Map (java.util.Map)3 UserException (io.hops.hopsworks.exceptions.UserException)2 File (java.io.File)2 IOException (java.io.IOException)2 CertificateMaterializer (io.hops.hopsworks.common.security.CertificateMaterializer)1 CryptoPasswordNotFoundException (io.hops.hopsworks.exceptions.CryptoPasswordNotFoundException)1 EncryptionMasterPasswordException (io.hops.hopsworks.exceptions.EncryptionMasterPasswordException)1 KafkaException (io.hops.hopsworks.exceptions.KafkaException)1 ProjectException (io.hops.hopsworks.exceptions.ProjectException)1 SchemaException (io.hops.hopsworks.exceptions.SchemaException)1 LocalResourceDTO (io.hops.hopsworks.persistence.entity.jobs.configuration.yarn.LocalResourceDTO)1 Project (io.hops.hopsworks.persistence.entity.project.Project)1 Users (io.hops.hopsworks.persistence.entity.user.Users)1 FileOutputStream (java.io.FileOutputStream)1 ByteBuffer (java.nio.ByteBuffer)1 InvalidKeyException (java.security.InvalidKeyException)1 KeyStore (java.security.KeyStore)1 NoSuchAlgorithmException (java.security.NoSuchAlgorithmException)1