Examples with YarnException org.apache.hadoop.yarn.exceptions.YarnException used on opensource projects

Example 46 with YarnException

use of org.apache.hadoop.yarn.exceptions.YarnException in project hadoop by apache.

the class TestApplicationPriorityACLs method submitAppToRMWithInValidAcl.

private void submitAppToRMWithInValidAcl(String submitter, ApplicationSubmissionContext appSubmissionContext) throws YarnException, IOException, InterruptedException {
    ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
    SubmitApplicationRequest submitRequest = SubmitApplicationRequest.newInstance(appSubmissionContext);
    try {
        submitterClient.submitApplication(submitRequest);
    } catch (YarnException ex) {
        Assert.assertTrue(ex.getCause() instanceof RemoteException);
    }
}

Example 47 with YarnException

use of org.apache.hadoop.yarn.exceptions.YarnException in project hadoop by apache.

the class WebAppProxyServlet method methodAction.

/**
   * The action against the HTTP method.
   * @param req the HttpServletRequest
   * @param resp the HttpServletResponse
   * @param method the HTTP method
   * @throws ServletException
   * @throws IOException
   */
private void methodAction(final HttpServletRequest req, final HttpServletResponse resp, final HTTP method) throws ServletException, IOException {
    try {
        String userApprovedParamS = req.getParameter(ProxyUriUtils.PROXY_APPROVAL_PARAM);
        boolean userWasWarned = false;
        boolean userApproved = Boolean.parseBoolean(userApprovedParamS);
        boolean securityEnabled = isSecurityEnabled();
        boolean isRedirect = false;
        String pathInfo = req.getPathInfo();
        final String remoteUser = req.getRemoteUser();
        String[] parts = null;
        if (pathInfo != null) {
            // parsed
            if (pathInfo.startsWith(REDIRECT)) {
                pathInfo = pathInfo.substring(REDIRECT.length());
                isRedirect = true;
            }
            parts = pathInfo.split("/", 3);
        }
        if ((parts == null) || (parts.length < 2)) {
            LOG.warn("{} gave an invalid proxy path {}", remoteUser, pathInfo);
            notFound(resp, "Your path appears to be formatted incorrectly.");
            return;
        }
        //parts[0] is empty because path info always starts with a /
        String appId = parts[1];
        String rest = parts.length > 2 ? parts[2] : "";
        ApplicationId id = Apps.toAppID(appId);
        if (id == null) {
            LOG.warn("{} attempting to access {} that is invalid", remoteUser, appId);
            notFound(resp, appId + " appears to be formatted incorrectly.");
            return;
        }
        // already redirected the response, so we can just return.
        if (isRedirect && handleRedirect(appId, req, resp)) {
            return;
        }
        if (securityEnabled) {
            String cookieName = getCheckCookieName(id);
            Cookie[] cookies = req.getCookies();
            if (cookies != null) {
                for (Cookie c : cookies) {
                    if (cookieName.equals(c.getName())) {
                        userWasWarned = true;
                        userApproved = userApproved || Boolean.parseBoolean(c.getValue());
                        break;
                    }
                }
            }
        }
        boolean checkUser = securityEnabled && (!userWasWarned || !userApproved);
        FetchedAppReport fetchedAppReport;
        try {
            fetchedAppReport = getFetchedAppReport(id);
        } catch (ApplicationNotFoundException e) {
            fetchedAppReport = null;
        }
        ApplicationReport applicationReport = null;
        if (fetchedAppReport != null) {
            applicationReport = fetchedAppReport.getApplicationReport();
        }
        if (applicationReport == null) {
            LOG.warn("{} attempting to access {} that was not found", remoteUser, id);
            URI toFetch = ProxyUriUtils.getUriFromTrackingPlugins(id, this.trackingUriPlugins);
            if (toFetch != null) {
                ProxyUtils.sendRedirect(req, resp, toFetch.toString());
                return;
            }
            notFound(resp, "Application " + appId + " could not be found " + "in RM or history server");
            return;
        }
        URI trackingUri = getTrackingUri(req, resp, id, applicationReport.getOriginalTrackingUrl(), fetchedAppReport.getAppReportSource());
        // If the tracking URI is null, there was a redirect, so just return.
        if (trackingUri == null) {
            return;
        }
        String runningUser = applicationReport.getUser();
        if (checkUser && !runningUser.equals(remoteUser)) {
            LOG.info("Asking {} if they want to connect to the " + "app master GUI of {} owned by {}", remoteUser, appId, runningUser);
            warnUserPage(resp, ProxyUriUtils.getPathAndQuery(id, rest, req.getQueryString(), true), runningUser, id);
            return;
        }
        // Append the user-provided path and query parameter to the original
        // tracking url.
        URI toFetch = buildTrackingUrl(trackingUri, req, rest);
        LOG.info("{} is accessing unchecked {}" + " which is the app master GUI of {} owned by {}", remoteUser, toFetch, appId, runningUser);
        switch(applicationReport.getYarnApplicationState()) {
            case KILLED:
            case FINISHED:
            case FAILED:
                ProxyUtils.sendRedirect(req, resp, toFetch.toString());
                return;
            default:
        }
        Cookie c = null;
        if (userWasWarned && userApproved) {
            c = makeCheckCookie(id, true);
        }
        proxyLink(req, resp, toFetch, c, getProxyHost(), method);
    } catch (URISyntaxException | YarnException e) {
        throw new IOException(e);
    }
}

Example 48 with YarnException

use of org.apache.hadoop.yarn.exceptions.YarnException in project hadoop by apache.

the class TestClientServiceDelegate method testRMDownRestoreForJobStatusBeforeGetAMReport.

@Test
public void testRMDownRestoreForJobStatusBeforeGetAMReport() throws IOException {
    Configuration conf = new YarnConfiguration();
    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(any(GetJobReportRequest.class))).thenReturn(getJobReportResponse());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
        when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(new java.lang.reflect.UndeclaredThrowableException(new IOException("Connection refuced1"))).thenThrow(new java.lang.reflect.UndeclaredThrowableException(new IOException("Connection refuced2"))).thenReturn(getFinishedApplicationReport());
        ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
        JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
        verify(rmDelegate, times(3)).getApplicationReport(any(ApplicationId.class));
        Assert.assertNotNull(jobStatus);
    } catch (YarnException e) {
        throw new IOException(e);
    }
}

Example 49 with YarnException

use of org.apache.hadoop.yarn.exceptions.YarnException in project hadoop by apache.

the class TestClientServiceDelegate method testReconnectOnAMRestart.

@Test
public void testReconnectOnAMRestart() throws IOException {
    //as instantiateAMProxy is not called at all
    if (!isAMReachableFromClient) {
        return;
    }
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    // RM returns AM1 url, null, null and AM2 url on invocations.
    // Nulls simulate the time when AM2 is in the process of restarting.
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
        when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(getRunningApplicationReport("am1", 78)).thenReturn(getRunningApplicationReport(null, 0)).thenReturn(getRunningApplicationReport(null, 0)).thenReturn(getRunningApplicationReport("am2", 90));
    } catch (YarnException e) {
        throw new IOException(e);
    }
    GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
    when(jobReportResponse1.getJobReport()).thenReturn(MRBuilderUtils.newJobReport(jobId, "jobName-firstGen", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null, false, ""));
    // First AM returns a report with jobName firstGen and simulates AM shutdown
    // on second invocation.
    MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
    when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class))).thenReturn(jobReportResponse1).thenThrow(new RuntimeException("AM is down!"));
    GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
    when(jobReportResponse2.getJobReport()).thenReturn(MRBuilderUtils.newJobReport(jobId, "jobName-secondGen", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null, false, ""));
    // Second AM generation returns a report with jobName secondGen
    MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
    when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class))).thenReturn(jobReportResponse2);
    ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
    // First time, connection should be to AM1, then to AM2. Further requests
    // should use the same proxy to AM2 and so instantiateProxy shouldn't be
    // called.
    doReturn(firstGenAMProxy).doReturn(secondGenAMProxy).when(clientServiceDelegate).instantiateAMProxy(any(InetSocketAddress.class));
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-firstGen", jobStatus.getJobName());
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());
    verify(clientServiceDelegate, times(2)).instantiateAMProxy(any(InetSocketAddress.class));
}

Example 50 with YarnException

use of org.apache.hadoop.yarn.exceptions.YarnException in project hadoop by apache.

the class BaseAMRMProxyE2ETest method createAMRMProtocol.

protected ApplicationMasterProtocol createAMRMProtocol(YarnClient rmClient, ApplicationId appId, MiniYARNCluster cluster, final Configuration yarnConf) throws IOException, InterruptedException, YarnException {
    UserGroupInformation user = null;
    // Get the AMRMToken from AMRMProxy
    ApplicationReport report = rmClient.getApplicationReport(appId);
    user = UserGroupInformation.createProxyUser(report.getCurrentApplicationAttemptId().toString(), UserGroupInformation.getCurrentUser());
    ContainerManagerImpl containerManager = (ContainerManagerImpl) cluster.getNodeManager(0).getNMContext().getContainerManager();
    AMRMProxyTokenSecretManager amrmTokenSecretManager = containerManager.getAMRMProxyService().getSecretManager();
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = amrmTokenSecretManager.createAndGetAMRMToken(report.getCurrentApplicationAttemptId());
    SecurityUtil.setTokenService(token, containerManager.getAMRMProxyService().getBindAddress());
    user.addToken(token);
    return user.doAs(new PrivilegedExceptionAction<ApplicationMasterProtocol>() {

        @Override
        public ApplicationMasterProtocol run() throws Exception {
            return ClientRMProxy.createRMProxy(yarnConf, ApplicationMasterProtocol.class);
        }
    });
}