Examples with YarnApplicationstate io.hops.hopsworks.persistence.entity.jobs.history.YarnApplicationstate used on opensource projects

Example 1 with YarnApplicationstate

use of io.hops.hopsworks.persistence.entity.jobs.history.YarnApplicationstate in project hopsworks by logicalclocks.

the class GrafanaProxyServlet method service.

@Override
protected void service(HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws ServletException, IOException {
    if (servletRequest.getUserPrincipal() == null || (!servletRequest.isUserInRole("HOPS_ADMIN") && !servletRequest.isUserInRole("HOPS_USER"))) {
        servletResponse.sendError(403, "User is not logged in");
        return;
    }
    if (servletRequest.getRequestURI().contains("query")) {
        String email = servletRequest.getUserPrincipal().getName();
        Users user = userFacade.findByEmail(email);
        Matcher matcher = pattern.matcher(servletRequest.getQueryString());
        if (matcher.find()) {
            String appId = matcher.group(1);
            YarnApplicationstate appState = yarnApplicationstateFacade.findByAppId(appId);
            if (appState == null) {
                servletResponse.sendError(Response.Status.BAD_REQUEST.getStatusCode(), "You don't have the access right for this application");
                return;
            }
            String projectName = hdfsUsersBean.getProjectName(appState.getAppuser());
            Project project = projectFacade.findByName(projectName);
            if (project == null) {
                servletResponse.sendError(Response.Status.BAD_REQUEST.getStatusCode(), "Project does not exists");
                return;
            }
            if (!projectTeamFacade.isUserMemberOfProject(project, user)) {
                servletResponse.sendError(Response.Status.BAD_REQUEST.getStatusCode(), "You don't have the access right for this application");
                return;
            }
        } else {
            boolean userRole = servletRequest.isUserInRole("HOPS_ADMIN");
            if (!userRole) {
                servletResponse.sendError(Response.Status.BAD_REQUEST.getStatusCode(), "You don't have the access right for this application");
                return;
            }
        }
    }
    super.service(servletRequest, servletResponse);
}

Example 2 with YarnApplicationstate

use of io.hops.hopsworks.persistence.entity.jobs.history.YarnApplicationstate in project hopsworks by logicalclocks.

the class TensorBoardProxyServlet method service.

// A request will come in with the format:
// http://127.0.0.1:8080/hopsworks-api/tensorboard/application_1507065031551_0005/hopsworks0:59460/#graphs
// 
@Override
protected void service(HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws ServletException, IOException {
    String email = servletRequest.getUserPrincipal().getName();
    if (Strings.isNullOrEmpty(email)) {
        servletResponse.sendError(Response.Status.FORBIDDEN.getStatusCode(), "You don't have access to this TensorBoard");
        return;
    }
    LOGGER.log(Level.FINE, "Request URL: {0}", servletRequest.getRequestURL());
    String uri = servletRequest.getRequestURI();
    // valid hostname regex:
    // https://stackoverflow.com/questions/106179/regular-expression-to-match-dns-hostname-or-ip-address
    Pattern urlPattern = Pattern.compile("([a-zA-Z0-9\\-\\.]{2,255}:[0-9]{4,6})(/.*$)");
    Matcher urlMatcher = urlPattern.matcher(uri);
    String hostPortPair = "";
    String uriToFinish = "/";
    if (urlMatcher.find()) {
        hostPortPair = urlMatcher.group(1);
        uriToFinish = urlMatcher.group(2);
    }
    if (hostPortPair.isEmpty()) {
        servletResponse.sendError(Response.Status.FORBIDDEN.getStatusCode(), "This TensorBoard is not accessible right now");
        return;
    }
    Pattern appPattern = Pattern.compile("(application_.*?_\\d*)");
    Matcher appMatcher = appPattern.matcher(servletRequest.getRequestURI());
    Pattern elasticPattern = Pattern.compile("(experiments)");
    Matcher elasticMatcher = elasticPattern.matcher(servletRequest.getRequestURI());
    if (elasticMatcher.find()) {
        List<TensorBoard> TBList = tensorBoardFacade.findByUserEmail(email);
        if (TBList == null) {
            servletResponse.sendError(Response.Status.FORBIDDEN.getStatusCode(), "This TensorBoard is not running right now");
            return;
        }
        boolean foundTB = false;
        for (TensorBoard tb : TBList) {
            if (tb.getEndpoint().equals(hostPortPair)) {
                foundTB = true;
                break;
            }
        }
        if (!foundTB) {
            servletResponse.sendError(Response.Status.FORBIDDEN.getStatusCode(), "This TensorBoard is not running right now");
            return;
        }
        targetUri = uriToFinish;
        String theHost = "http://" + hostPortPair;
        URI targetUriHost;
        try {
            targetUriObj = new URI(targetUri);
            targetUriHost = new URI(theHost);
        } catch (Exception e) {
            LOGGER.log(Level.FINE, "An error occurred serving the request", e);
            return;
        }
        targetHost = URIUtils.extractHost(targetUriHost);
        servletRequest.setAttribute(ATTR_TARGET_URI, targetUri);
        servletRequest.setAttribute(ATTR_TARGET_HOST, targetHost);
        servletRequest.setAttribute(ATTR_URI_FINISH, uriToFinish);
        servletRequest.setAttribute(ATTR_HOST_PORT, hostPortPair);
        try {
            super.service(servletRequest, servletResponse);
        } catch (IOException ex) {
            servletResponse.sendError(Response.Status.NOT_FOUND.getStatusCode(), "This TensorBoard is not ready to serve requests right now, " + "try refreshing the page");
            return;
        }
    } else if (appMatcher.find()) {
        String appId = appMatcher.group(1);
        YarnApplicationstate appState = yarnApplicationstateFacade.findByAppId(appId);
        if (appState == null) {
            servletResponse.sendError(Response.Status.FORBIDDEN.getStatusCode(), "You don't have the access right for this application");
            return;
        }
        Users user = userFacade.findByEmail(email);
        String projectName = hdfsUsersBean.getProjectName(appState.getAppuser());
        Project project = projectFacade.findByName(projectName);
        if (project == null) {
            servletResponse.sendError(Response.Status.BAD_REQUEST.getStatusCode(), "Project does not exists");
            return;
        }
        if (!projectTeamFacade.isUserMemberOfProject(project, user)) {
            servletResponse.sendError(Response.Status.BAD_REQUEST.getStatusCode(), "You don't have the access right for this application");
            return;
        }
        if (appState.getAppsmstate() != null && (appState.getAppsmstate().equalsIgnoreCase(YarnApplicationState.FINISHED.toString()) || appState.getAppsmstate().equalsIgnoreCase(YarnApplicationState.KILLED.toString()))) {
            servletResponse.sendError(Response.Status.NOT_FOUND.getStatusCode(), "This TensorBoard has finished running.");
            return;
        }
        targetUri = uriToFinish;
        String theHost = "http://" + hostPortPair;
        URI targetUriHost;
        try {
            targetUriObj = new URI(targetUri);
            targetUriHost = new URI(theHost);
        } catch (Exception e) {
            servletResponse.sendError(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), "An error occurred serving the request.");
            LOGGER.log(Level.FINE, "An error occurred serving the request", e);
            return;
        }
        targetHost = URIUtils.extractHost(targetUriHost);
        servletRequest.setAttribute(ATTR_TARGET_URI, targetUri);
        servletRequest.setAttribute(ATTR_TARGET_HOST, targetHost);
        servletRequest.setAttribute(ATTR_URI_FINISH, uriToFinish);
        servletRequest.setAttribute(ATTR_HOST_PORT, hostPortPair);
        try {
            super.service(servletRequest, servletResponse);
        } catch (IOException ex) {
            servletResponse.sendError(Response.Status.NOT_FOUND.getStatusCode(), "This TensorBoard is not running right now.");
            return;
        }
    } else {
        servletResponse.sendError(Response.Status.FORBIDDEN.getStatusCode(), "You don't have the access right for this application");
        return;
    }
}

Example 3 with YarnApplicationstate

use of io.hops.hopsworks.persistence.entity.jobs.history.YarnApplicationstate in project hopsworks by logicalclocks.

the class LivyController method getLivySessions.

/**
 * Get Livy sessions for project, depending on service type.
 *
 * @param project
 * @return
 */
public List<LivyMsg.Session> getLivySessions(Project project) {
    List<LivyMsg.Session> sessions = new ArrayList<>();
    LivyMsg sessionList = getLivySessions();
    if (sessionList == null || sessionList.getSessions() == null || sessionList.getSessions().length == 0) {
        return sessions;
    }
    List<ProjectTeam> projectTeam = teambean.findMembersByProject(project);
    for (ProjectTeam member : projectTeam) {
        String hdfsUsername = hdfsUserBean.getHdfsUserName(project, member.getUser());
        for (LivyMsg.Session s : sessionList.getSessions()) {
            if (hdfsUsername.equals(s.getProxyUser())) {
                YarnApplicationstate appStates = appStateBean.findByAppId(s.getAppId());
                if (appStates == null) {
                    continue;
                }
                s.setOwner(member.getUser().getEmail());
                sessions.add(s);
            }
        }
    }
    return sessions;
}

Example 4 with YarnApplicationstate

use of io.hops.hopsworks.persistence.entity.jobs.history.YarnApplicationstate in project hopsworks by logicalclocks.

the class LivyController method getLivySessionsForProjectUser.

/**
 * Get all Jupyter livy sessions for project and user
 *
 * @param project
 * @param user
 * @return
 */
public List<LivyMsg.Session> getLivySessionsForProjectUser(Project project, Users user) {
    List<LivyMsg.Session> sessions = new ArrayList<>();
    LivyMsg sessionList = getLivySessions();
    if (sessionList == null || sessionList.getSessions() == null || sessionList.getSessions().length == 0) {
        return sessions;
    }
    String hdfsUsername = hdfsUserBean.getHdfsUserName(project, user);
    for (LivyMsg.Session s : sessionList.getSessions()) {
        if (hdfsUsername.equals(s.getProxyUser())) {
            YarnApplicationstate appStates = appStateBean.findByAppId(s.getAppId());
            if (appStates == null) {
                continue;
            }
            s.setOwner(user.getEmail());
            sessions.add(s);
        }
    }
    return sessions;
}

Example 5 with YarnApplicationstate

use of io.hops.hopsworks.persistence.entity.jobs.history.YarnApplicationstate in project hopsworks by logicalclocks.

the class FlinkProxyServlet method service.

@Override
protected void service(HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws ServletException, IOException {
    Users user;
    if (servletRequest.getUserPrincipal() == null) {
        // Check if API key is provided
        String authorizationHeader = servletRequest.getHeader("Authorization");
        if (Strings.isNullOrEmpty(authorizationHeader)) {
            servletResponse.sendError(401, "API key was not provided");
            return;
        } else {
            try {
                String key = authorizationHeader.substring(ApiKeyFilter.API_KEY.length()).trim();
                ApiKey apiKey = apiKeyController.getApiKey(key);
                user = apiKey.getUser();
            } catch (ApiKeyException e) {
                servletResponse.sendError(401, "Could not validate API key");
                return;
            }
        }
    } else {
        user = userFacade.findByEmail(servletRequest.getUserPrincipal().getName());
    }
    String uri = servletRequest.getRequestURI();
    Pattern appPattern = Pattern.compile("(application_.*?_\\d*)");
    Matcher appMatcher = appPattern.matcher(uri);
    String appId;
    String flinkMasterURL;
    if (appMatcher.find()) {
        appId = appMatcher.group(1);
        // Validate user is authorized to access to this yarn app
        YarnApplicationstate appState = yarnApplicationstateFacade.findByAppId(appId);
        // If job is not running, show relevant message
        if (!Strings.isNullOrEmpty(appState.getAppsmstate()) && (YarnApplicationState.valueOf(appState.getAppsmstate()) == YarnApplicationState.FAILED || YarnApplicationState.valueOf(appState.getAppsmstate()) == YarnApplicationState.FINISHED || YarnApplicationState.valueOf(appState.getAppsmstate()) == YarnApplicationState.KILLED)) {
            servletResponse.sendError(404, "This Flink cluster is not running. You can navigate to YARN and Logs for historical information on this " + "Flink cluster.");
            return;
        }
        HdfsUsers hdfsUser = hdfsUsersFacade.findByName(appState.getAppuser());
        if (!projectTeamFacade.isUserMemberOfProject(projectFacade.findByName(hdfsUser.getProject()), user)) {
            servletResponse.sendError(403, "You are not authorized to access this Flink cluster");
        }
        // Is this user member of the project?
        flinkMasterURL = flinkMasterAddrCache.get(appId);
        if (Strings.isNullOrEmpty(flinkMasterURL)) {
            servletResponse.sendError(404, "This Flink cluster is not running. You can navigate to YARN and Logs for historical information on this" + " Flink cluster.");
            return;
        }
        String theHost = "http://" + flinkMasterURL;
        URI targetUriHost;
        targetUri = theHost;
        try {
            targetUriObj = new URI(targetUri);
            targetUriHost = new URI(theHost);
        } catch (Exception e) {
            LOGGER.log(Level.INFO, "An error occurred serving the request", e);
            return;
        }
        targetHost = URIUtils.extractHost(targetUriHost);
        servletRequest.setAttribute(ATTR_TARGET_URI, targetUri);
        servletRequest.setAttribute(ATTR_TARGET_HOST, targetHost);
        servletRequest.setAttribute(ATTR_HOST_PORT, flinkMasterURL);
        super.service(servletRequest, servletResponse);
    } else {
        servletResponse.sendError(404, "This Flink cluster is not running. You can navigate to YARN and Logs for historical information on this " + "Flink cluster.");
    }
}