use of io.hops.hopsworks.exceptions.SchemaException in project hopsworks by logicalclocks.
the class KafkaResource method deleteSubjectsVersion.
@ApiOperation(value = "Delete a specific version of a subject (and its compatibility).")
@DELETE
@Path("/subjects/{subject}/versions/{version}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.KAFKA }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response deleteSubjectsVersion(@PathParam("subject") String subject, @PathParam("version") String version, @Context SecurityContext sc) throws KafkaException {
try {
Integer deleted = subjectsController.deleteSubjectsVersion(project, subject, version);
GenericEntity<Integer> entity = new GenericEntity<Integer>(deleted) {
};
return Response.ok().entity(entity).build();
} catch (SchemaException e) {
SchemaRegistryError error = new SchemaRegistryError(e.getErrorCode().getCode(), e.getErrorCode().getMessage());
return Response.status(e.getErrorCode().getRespStatus()).entity(error).build();
}
}
use of io.hops.hopsworks.exceptions.SchemaException in project hopsworks by logicalclocks.
the class ProjectService method example.
@POST
@Path("starterProject/{type}")
@Produces(MediaType.APPLICATION_JSON)
public Response example(@PathParam("type") String type, @Context HttpServletRequest req, @Context SecurityContext sc) throws DatasetException, GenericException, KafkaException, ProjectException, UserException, ServiceException, HopsSecurityException, FeaturestoreException, JobException, IOException, ElasticException, SchemaException, ProvenanceException {
TourProjectType demoType;
try {
demoType = TourProjectType.fromString(type);
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("Type must be one of: " + Arrays.toString(TourProjectType.values()));
}
ProjectDTO projectDTO = new ProjectDTO();
Project project = null;
projectDTO.setDescription("A demo project for getting started with " + demoType.getDescription());
Users user = jWTHelper.getUserPrincipal(sc);
String username = user.getUsername();
List<String> projectServices = new ArrayList<>();
// save the project
String readMeMessage = null;
switch(demoType) {
case KAFKA:
// It's a Kafka guide
projectDTO.setProjectName("demo_" + TourProjectType.KAFKA.getTourName() + "_" + username);
populateActiveServices(projectServices, TourProjectType.KAFKA);
readMeMessage = "jar file to demonstrate Kafka streaming";
break;
case SPARK:
// It's a Spark guide
projectDTO.setProjectName("demo_" + TourProjectType.SPARK.getTourName() + "_" + username);
populateActiveServices(projectServices, TourProjectType.SPARK);
readMeMessage = "jar file to demonstrate the creation of a spark batch job";
break;
case FS:
// It's a Featurestore guide
projectDTO.setProjectName("demo_" + TourProjectType.FS.getTourName() + "_" + username);
populateActiveServices(projectServices, TourProjectType.FS);
readMeMessage = "Dataset containing a jar file and data that can be used to run a sample spark-job for " + "inserting data in the feature store.";
break;
case ML:
// It's a TensorFlow guide
projectDTO.setProjectName("demo_" + TourProjectType.ML.getTourName() + "_" + username);
populateActiveServices(projectServices, TourProjectType.ML);
readMeMessage = "Jupyter notebooks and training data for demonstrating how to run Deep Learning";
break;
default:
throw new IllegalArgumentException("Type must be one of: " + Arrays.toString(TourProjectType.values()));
}
projectDTO.setServices(projectServices);
DistributedFileSystemOps dfso = null;
DistributedFileSystemOps udfso = null;
try {
project = projectController.createProject(projectDTO, user, req.getSession().getId());
dfso = dfs.getDfsOps();
username = hdfsUsersBean.getHdfsUserName(project, user);
udfso = dfs.getDfsOps(username);
ProvTypeDTO projectMetaStatus = fsProvenanceController.getProjectProvType(user, project);
String tourFilesDataset = projectController.addTourFilesToProject(user.getEmail(), project, dfso, dfso, demoType, projectMetaStatus);
// TestJob dataset
datasetController.generateReadme(udfso, tourFilesDataset, readMeMessage, project.getName());
} catch (Exception ex) {
projectController.cleanup(project, req.getSession().getId());
throw ex;
} finally {
if (dfso != null) {
dfso.close();
}
if (udfso != null) {
dfs.closeDfsClient(udfso);
}
}
return noCacheResponse.getNoCacheResponseBuilder(Response.Status.CREATED).entity(project).build();
}
use of io.hops.hopsworks.exceptions.SchemaException in project hopsworks by logicalclocks.
the class SubjectsCompatibilityController method setSubjectCompatibility.
public Compatibility setSubjectCompatibility(Project project, String subject, SchemaCompatibility sc) throws SchemaException {
if (sc == null) {
throw new SchemaException(RESTCodes.SchemaRegistryErrorCode.INVALID_COMPATIBILITY, Level.WARNING, "Compatibility cannot be null");
}
if (subject == null || subject.equals(Settings.PROJECT_COMPATIBILITY_SUBJECT) || subjectsFacade.findSubjectByName(project, subject).isEmpty()) {
throw new SchemaException(RESTCodes.SchemaRegistryErrorCode.SUBJECT_NOT_FOUND, Level.WARNING, "Incorrect subject");
}
subjectsCompatibilityFacade.updateSubjectCompatibility(project, subject, sc);
CompatibilityLevel levelDto = getSubjectCompatibility(project, subject);
return new Compatibility(levelDto.getCompatibilityLevel());
}
use of io.hops.hopsworks.exceptions.SchemaException in project hopsworks by logicalclocks.
the class ProjectController method createProject.
/**
* Creates a new project(project), the related DIR, the different services in
* the project, and the master of the
* project.
* <p>
* This needs to be an atomic operation (all or nothing) REQUIRES_NEW will
* make sure a new transaction is created even
* if this method is called from within a transaction.
*
* @param projectDTO
* @param owner
* @param sessionId
* @return
*/
public Project createProject(ProjectDTO projectDTO, Users owner, String sessionId) throws DatasetException, GenericException, KafkaException, ProjectException, UserException, HopsSecurityException, ServiceException, FeaturestoreException, ElasticException, SchemaException, IOException {
Long startTime = System.currentTimeMillis();
// check that the project name is ok
String projectName = projectDTO.getProjectName();
FolderNameValidator.isValidProjectName(projectUtils, projectName);
List<ProjectServiceEnum> projectServices = new ArrayList<>();
if (projectDTO.getServices() != null) {
for (String s : projectDTO.getServices()) {
ProjectServiceEnum se = ProjectServiceEnum.valueOf(s.toUpperCase());
projectServices.add(se);
}
}
LOGGER.log(Level.FINE, () -> "PROJECT CREATION TIME. Step 1: " + (System.currentTimeMillis() - startTime));
DistributedFileSystemOps dfso = null;
Project project = null;
try {
dfso = dfs.getDfsOps();
/*
* create a project in the database
* if the creation go through it means that there is no other project with
* the same name.
* this project creation act like a lock, no other project can be created
* with the same name
* until this project is removed from the database
*/
try {
project = createProject(projectName, owner, projectDTO.getDescription(), dfso);
} catch (EJBException ex) {
LOGGER.log(Level.WARNING, null, ex);
Path dummy = new Path("/tmp/" + projectName);
try {
dfso.rm(dummy, true);
} catch (IOException e) {
LOGGER.log(Level.SEVERE, null, e);
}
throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_EXISTS, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
}
LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 2 (hdfs): {0}", System.currentTimeMillis() - startTime);
verifyProject(project, dfso, sessionId);
LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 3 (verify): {0}", System.currentTimeMillis() - startTime);
// Run the handlers.
try {
ProjectHandler.runProjectPreCreateHandlers(projectHandlers, project);
} catch (ProjectException ex) {
cleanup(project, sessionId, null, true, owner);
throw ex;
}
List<Future<?>> projectCreationFutures = new ArrayList<>();
// This is an async call
try {
projectCreationFutures.add(certificatesController.generateCertificates(project, owner));
} catch (Exception ex) {
cleanup(project, sessionId, projectCreationFutures, true, owner);
throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername(), ex.getMessage(), ex);
}
String username = hdfsUsersController.getHdfsUserName(project, owner);
if (username == null || username.isEmpty()) {
cleanup(project, sessionId, projectCreationFutures, true, owner);
throw new UserException(RESTCodes.UserErrorCode.USER_WAS_NOT_FOUND, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername());
}
LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 4 (certs): {0}", System.currentTimeMillis() - startTime);
// all the verifications have passed, we can now create the project
// create the project folder
ProvTypeDTO provType = settings.getProvType().dto;
try {
mkProjectDIR(projectName, dfso);
fsProvController.updateProjectProvType(project, provType, dfso);
} catch (IOException | EJBException | ProvenanceException ex) {
cleanup(project, sessionId, projectCreationFutures, true, owner);
throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_FOLDER_NOT_CREATED, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
}
LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 5 (folders): {0}", System.currentTimeMillis() - startTime);
// update the project with the project folder inode
try {
setProjectInode(project, dfso);
} catch (IOException | EJBException ex) {
cleanup(project, sessionId, projectCreationFutures, true, owner);
throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_INODE_CREATION_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
}
LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 6 (inodes): {0}", System.currentTimeMillis() - startTime);
// set payment and quotas
try {
setProjectOwnerAndQuotas(project, dfso, owner);
} catch (IOException | EJBException ex) {
cleanup(project, sessionId, projectCreationFutures, true, owner);
throw new ProjectException(RESTCodes.ProjectErrorCode.QUOTA_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
}
LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 7 (quotas): {0}", System.currentTimeMillis() - startTime);
try {
hdfsUsersController.addProjectFolderOwner(project, dfso);
createProjectLogResources(owner, project, dfso);
} catch (IOException | EJBException ex) {
cleanup(project, sessionId, projectCreationFutures);
throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_SET_PERMISSIONS_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
}
LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 8 (logs): {0}", System.currentTimeMillis() - startTime);
// inconsistencies
try {
elasticController.deleteProjectIndices(project);
elasticController.deleteProjectSavedObjects(projectName);
LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 9 (elastic cleanup): {0}", System.currentTimeMillis() - startTime);
} catch (ElasticException ex) {
LOGGER.log(Level.FINE, "Error while cleaning old project indices", ex);
}
logProject(project, OperationType.Add);
// enable services
for (ProjectServiceEnum service : projectServices) {
try {
projectCreationFutures.addAll(addService(project, service, owner, dfso, provType));
} catch (RESTException | IOException ex) {
cleanup(project, sessionId, projectCreationFutures);
throw ex;
}
}
try {
for (Future f : projectCreationFutures) {
if (f != null) {
f.get();
}
}
} catch (InterruptedException | ExecutionException ex) {
LOGGER.log(Level.SEVERE, "Error while waiting for the certificate generation thread to finish. Will try to " + "cleanup...", ex);
cleanup(project, sessionId, projectCreationFutures);
throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE);
}
// Run the handlers.
try {
ProjectHandler.runProjectPostCreateHandlers(projectHandlers, project);
} catch (ProjectException ex) {
cleanup(project, sessionId, projectCreationFutures);
throw ex;
}
try {
project = environmentController.createEnv(project, owner);
} catch (PythonException | EJBException ex) {
cleanup(project, sessionId, projectCreationFutures);
throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_ANACONDA_ENABLE_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
}
LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 10 (env): {0}", System.currentTimeMillis() - startTime);
return project;
} finally {
if (dfso != null) {
dfso.close();
}
LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 11 (close): {0}", System.currentTimeMillis() - startTime);
}
}
use of io.hops.hopsworks.exceptions.SchemaException in project hopsworks by logicalclocks.
the class SubjectsController method deleteSubjectsVersion.
public Integer deleteSubjectsVersion(Project project, String subject, String version) throws SchemaException, KafkaException {
validateSubject(subject, false);
validateVersion(version);
if (!subjectsFacade.getListOfSubjects(project).contains(subject)) {
throw new SchemaException(RESTCodes.SchemaRegistryErrorCode.SUBJECT_NOT_FOUND, Level.FINE, "subject=" + subject);
}
Optional<Subjects> optional;
if (version.equals("latest")) {
optional = subjectsFacade.findSubjectLatestVersion(project, subject);
} else {
optional = subjectsFacade.findSubjectByNameAndVersion(project, subject, Integer.valueOf(version));
}
if (!optional.isPresent()) {
throw new SchemaException(RESTCodes.SchemaRegistryErrorCode.VERSION_NOT_FOUND, Level.FINE, "project=" + project.getName() + ", subject=" + subject + ", version=" + version);
}
Integer versionToDelete = optional.get().getVersion();
if (!projectTopicsFacade.findTopiscBySubjectAndVersion(project, subject, versionToDelete).isEmpty()) {
throw new KafkaException(RESTCodes.KafkaErrorCode.SCHEMA_IN_USE, Level.FINE, "project=" + project.getName() + ", subject=" + subject + ", version=" + versionToDelete);
}
subjectsFacade.remove(optional.get());
subjectsCompatibilityFacade.getSubjectCompatibility(project, subject).ifPresent(sc -> subjectsCompatibilityFacade.remove(sc));
return versionToDelete;
}
Aggregations