Examples with ClientToAMTokenIdentifier org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier used on opensource projects

Example 1 with ClientToAMTokenIdentifier

use of org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier in project hadoop by apache.

the class TestRMAppAttemptTransitions method testGetClientToken.

// this is to test user can get client tokens only after the client token
// master key is saved in the state store and also registered in
// ClientTokenSecretManager
@Test
public void testGetClientToken() throws Exception {
    assumeTrue(isSecurityEnabled);
    Container amContainer = allocateApplicationAttempt();
    // before attempt is launched, can not get ClientToken
    Token<ClientToAMTokenIdentifier> token = applicationAttempt.createClientToken(null);
    Assert.assertNull(token);
    token = applicationAttempt.createClientToken("clientuser");
    Assert.assertNull(token);
    launchApplicationAttempt(amContainer);
    // after attempt is launched , can get ClientToken
    token = applicationAttempt.createClientToken(null);
    Assert.assertNull(token);
    token = applicationAttempt.createClientToken("clientuser");
    Assert.assertNotNull(token);
    applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL));
    assertEquals(YarnApplicationAttemptState.LAUNCHED, applicationAttempt.createApplicationAttemptState());
    sendAttemptUpdateSavedEvent(applicationAttempt);
    // after attempt is killed, can not get Client Token
    token = applicationAttempt.createClientToken(null);
    Assert.assertNull(token);
    token = applicationAttempt.createClientToken("clientuser");
    Assert.assertNull(token);
}

Example 2 with ClientToAMTokenIdentifier

use of org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier in project hadoop by apache.

the class TestClientToAMTokens method testClientToAMTokens.

@Test
public void testClientToAMTokens() throws Exception {
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    // Set RPC engine to protobuf RPC engine
    RPC.setProtocolEngine(conf, CustomProtocol.class, ProtobufRpcEngine.class);
    UserGroupInformation.setConfiguration(conf);
    ContainerManagementProtocol containerManager = mock(ContainerManagementProtocol.class);
    StartContainersResponse mockResponse = mock(StartContainersResponse.class);
    when(containerManager.startContainers((StartContainersRequest) any())).thenReturn(mockResponse);
    final DrainDispatcher dispatcher = new DrainDispatcher();
    MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, getRMContext().getRMDelegationTokenSecretManager());
        }

        ;

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }

        @Override
        protected void doSecureLogin() throws IOException {
        }
    };
    rm.start();
    // Submit an app
    RMApp app = rm.submitApp(1024);
    // Set up a node.
    MockNM nm1 = rm.registerNode("localhost:1234", 3072);
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
    final MockAM mockAM = new MockAM(rm.getRMContext(), rm.getApplicationMasterService(), app.getCurrentAppAttempt().getAppAttemptId());
    UserGroupInformation appUgi = UserGroupInformation.createRemoteUser(appAttempt.toString());
    RegisterApplicationMasterResponse response = appUgi.doAs(new PrivilegedAction<RegisterApplicationMasterResponse>() {

        @Override
        public RegisterApplicationMasterResponse run() {
            RegisterApplicationMasterResponse response = null;
            try {
                response = mockAM.registerAppAttempt();
            } catch (Exception e) {
                Assert.fail("Exception was not expected");
            }
            return response;
        }
    });
    // Get the app-report.
    GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class);
    request.setApplicationId(app.getApplicationId());
    GetApplicationReportResponse reportResponse = rm.getClientRMService().getApplicationReport(request);
    ApplicationReport appReport = reportResponse.getApplicationReport();
    org.apache.hadoop.yarn.api.records.Token originalClientToAMToken = appReport.getClientToAMToken();
    // ClientToAMToken master key should have been received on register
    // application master response.
    Assert.assertNotNull(response.getClientToAMTokenMasterKey());
    Assert.assertTrue(response.getClientToAMTokenMasterKey().array().length > 0);
    // Start the AM with the correct shared-secret.
    ApplicationAttemptId appAttemptId = app.getAppAttempts().keySet().iterator().next();
    Assert.assertNotNull(appAttemptId);
    final CustomAM am = new CustomAM(appAttemptId, response.getClientToAMTokenMasterKey().array());
    am.init(conf);
    am.start();
    // Now the real test!
    // Set up clients to be able to pick up correct tokens.
    SecurityUtil.setSecurityInfoProviders(new CustomSecurityInfo());
    // Verify denial for unauthenticated user
    try {
        CustomProtocol client = RPC.getProxy(CustomProtocol.class, 1L, am.address, conf);
        client.ping(null, TestRpcBase.newEmptyRequest());
        fail("Access by unauthenticated user should fail!!");
    } catch (Exception e) {
        Assert.assertFalse(am.pinged);
    }
    Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(originalClientToAMToken, am.address);
    // Verify denial for a malicious user with tampered ID
    verifyTokenWithTamperedID(conf, am, token);
    // Verify denial for a malicious user with tampered user-name
    verifyTokenWithTamperedUserName(conf, am, token);
    // Now for an authenticated user
    verifyValidToken(conf, am, token);
    // Verify for a new version token
    verifyNewVersionToken(conf, am, token, rm);
    am.stop();
    rm.stop();
}

Example 3 with ClientToAMTokenIdentifier

use of org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier in project hadoop by apache.

the class TestClientToAMTokens method verifyNewVersionToken.

private void verifyNewVersionToken(final Configuration conf, final CustomAM am, Token<ClientToAMTokenIdentifier> token, MockRM rm) throws IOException, InterruptedException {
    UserGroupInformation ugi;
    ugi = UserGroupInformation.createRemoteUser("me");
    Token<ClientToAMTokenIdentifier> newToken = new Token<ClientToAMTokenIdentifier>(new ClientToAMTokenIdentifierForTest(token.decodeIdentifier(), "message"), am.getClientToAMTokenSecretManager());
    newToken.setService(token.getService());
    ugi.addToken(newToken);
    ugi.doAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            CustomProtocol client = RPC.getProxy(CustomProtocol.class, 1L, am.address, conf);
            client.ping(null, TestRpcBase.newEmptyRequest());
            Assert.assertTrue(am.pinged);
            return null;
        }
    });
}

Example 4 with ClientToAMTokenIdentifier

use of org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier in project hadoop by apache.

the class ClientServiceDelegate method getProxy.

private MRClientProtocol getProxy() throws IOException {
    if (realProxy != null) {
        return realProxy;
    }
    // Possibly allow nulls through the PB tunnel, otherwise deal with an exception
    // and redirect to the history server.
    ApplicationReport application = null;
    try {
        application = rm.getApplicationReport(appId);
    } catch (ApplicationNotFoundException e) {
        application = null;
    } catch (YarnException e2) {
        throw new IOException(e2);
    }
    if (application != null) {
        trackingUrl = application.getTrackingUrl();
    }
    InetSocketAddress serviceAddr = null;
    while (application == null || YarnApplicationState.RUNNING == application.getYarnApplicationState()) {
        if (application == null) {
            LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server.");
            return checkAndGetHSProxy(null, JobState.NEW);
        }
        try {
            if (application.getHost() == null || "".equals(application.getHost())) {
                LOG.debug("AM not assigned to Job. Waiting to get the AM ...");
                Thread.sleep(2000);
                LOG.debug("Application state is " + application.getYarnApplicationState());
                application = rm.getApplicationReport(appId);
                continue;
            } else if (UNAVAILABLE.equals(application.getHost())) {
                if (!amAclDisabledStatusLogged) {
                    LOG.info("Job " + jobId + " is running, but the host is unknown." + " Verify user has VIEW_JOB access.");
                    amAclDisabledStatusLogged = true;
                }
                return getNotRunningJob(application, JobState.RUNNING);
            }
            if (!conf.getBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, false)) {
                UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName());
                serviceAddr = NetUtils.createSocketAddrForHost(application.getHost(), application.getRpcPort());
                if (UserGroupInformation.isSecurityEnabled()) {
                    org.apache.hadoop.yarn.api.records.Token clientToAMToken = application.getClientToAMToken();
                    Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr);
                    newUgi.addToken(token);
                }
                LOG.debug("Connecting to " + serviceAddr);
                final InetSocketAddress finalServiceAddr = serviceAddr;
                realProxy = newUgi.doAs(new PrivilegedExceptionAction<MRClientProtocol>() {

                    @Override
                    public MRClientProtocol run() throws IOException {
                        return instantiateAMProxy(finalServiceAddr);
                    }
                });
            } else {
                if (!amAclDisabledStatusLogged) {
                    LOG.info("Network ACL closed to AM for job " + jobId + ". Not going to try to reach the AM.");
                    amAclDisabledStatusLogged = true;
                }
                return getNotRunningJob(null, JobState.RUNNING);
            }
            return realProxy;
        } catch (IOException e) {
            //possibly the AM has crashed
            //there may be some time before AM is restarted
            //keep retrying by getting the address from RM
            LOG.info("Could not connect to " + serviceAddr + ". Waiting for getting the latest AM address...");
            try {
                Thread.sleep(2000);
            } catch (InterruptedException e1) {
                LOG.warn("getProxy() call interruped", e1);
                throw new YarnRuntimeException(e1);
            }
            try {
                application = rm.getApplicationReport(appId);
            } catch (YarnException e1) {
                throw new IOException(e1);
            }
            if (application == null) {
                LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server.");
                return checkAndGetHSProxy(null, JobState.RUNNING);
            }
        } catch (InterruptedException e) {
            LOG.warn("getProxy() call interruped", e);
            throw new YarnRuntimeException(e);
        } catch (YarnException e) {
            throw new IOException(e);
        }
    }
    /** we just want to return if its allocating, so that we don't
     * block on it. This is to be able to return job status
     * on an allocating Application.
     */
    String user = application.getUser();
    if (user == null) {
        throw new IOException("User is not set in the application report");
    }
    if (application.getYarnApplicationState() == YarnApplicationState.NEW || application.getYarnApplicationState() == YarnApplicationState.NEW_SAVING || application.getYarnApplicationState() == YarnApplicationState.SUBMITTED || application.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.NEW);
    }
    if (application.getYarnApplicationState() == YarnApplicationState.FAILED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.FAILED);
    }
    if (application.getYarnApplicationState() == YarnApplicationState.KILLED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.KILLED);
    }
    //succeeded.
    if (application.getYarnApplicationState() == YarnApplicationState.FINISHED) {
        LOG.info("Application state is completed. FinalApplicationStatus=" + application.getFinalApplicationStatus().toString() + ". Redirecting to job history server");
        realProxy = checkAndGetHSProxy(application, JobState.SUCCEEDED);
    }
    return realProxy;
}

Example 5 with ClientToAMTokenIdentifier

use of org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier in project hadoop by apache.

the class RMAppImpl method createAndGetApplicationReport.

@Override
public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) {
    this.readLock.lock();
    try {
        ApplicationAttemptId currentApplicationAttemptId = null;
        org.apache.hadoop.yarn.api.records.Token clientToAMToken = null;
        String trackingUrl = UNAVAILABLE;
        String host = UNAVAILABLE;
        String origTrackingUrl = UNAVAILABLE;
        LogAggregationStatus logAggregationStatus = null;
        int rpcPort = -1;
        ApplicationResourceUsageReport appUsageReport = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
        FinalApplicationStatus finishState = getFinalApplicationStatus();
        String diags = UNAVAILABLE;
        float progress = 0.0f;
        org.apache.hadoop.yarn.api.records.Token amrmToken = null;
        if (allowAccess) {
            trackingUrl = getDefaultProxyTrackingUrl();
            if (this.currentAttempt != null) {
                currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
                trackingUrl = this.currentAttempt.getTrackingUrl();
                origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl();
                if (UserGroupInformation.isSecurityEnabled()) {
                    // get a token so the client can communicate with the app attempt
                    // NOTE: token may be unavailable if the attempt is not running
                    Token<ClientToAMTokenIdentifier> attemptClientToAMToken = this.currentAttempt.createClientToken(clientUserName);
                    if (attemptClientToAMToken != null) {
                        clientToAMToken = BuilderUtils.newClientToAMToken(attemptClientToAMToken.getIdentifier(), attemptClientToAMToken.getKind().toString(), attemptClientToAMToken.getPassword(), attemptClientToAMToken.getService().toString());
                    }
                }
                host = this.currentAttempt.getHost();
                rpcPort = this.currentAttempt.getRpcPort();
                appUsageReport = currentAttempt.getApplicationResourceUsageReport();
                progress = currentAttempt.getProgress();
                logAggregationStatus = this.getLogAggregationStatusForAppReport();
            }
            //if the diagnostics is not already set get it from attempt
            diags = getDiagnostics().toString();
            if (currentAttempt != null && currentAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
                if (getApplicationSubmissionContext().getUnmanagedAM() && clientUserName != null && getUser().equals(clientUserName)) {
                    Token<AMRMTokenIdentifier> token = currentAttempt.getAMRMToken();
                    if (token != null) {
                        amrmToken = BuilderUtils.newAMRMToken(token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString());
                    }
                }
            }
            RMAppMetrics rmAppMetrics = getRMAppMetrics();
            appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
            appUsageReport.setPreemptedMemorySeconds(rmAppMetrics.getPreemptedMemorySeconds());
            appUsageReport.setPreemptedVcoreSeconds(rmAppMetrics.getPreemptedVcoreSeconds());
        }
        if (currentApplicationAttemptId == null) {
            currentApplicationAttemptId = BuilderUtils.newApplicationAttemptId(this.applicationId, DUMMY_APPLICATION_ATTEMPT_NUMBER);
        }
        ApplicationReport report = BuilderUtils.newApplicationReport(this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, amrmToken, applicationTags, this.getApplicationPriority());
        report.setLogAggregationStatus(logAggregationStatus);
        report.setUnmanagedApp(submissionContext.getUnmanagedAM());
        report.setAppNodeLabelExpression(getAppNodeLabelExpression());
        report.setAmNodeLabelExpression(getAmNodeLabelExpression());
        ApplicationTimeout timeout = ApplicationTimeout.newInstance(ApplicationTimeoutType.LIFETIME, UNLIMITED, UNKNOWN);
        // are supported in YARN-5692, the below logic need to be changed.
        if (!this.applicationTimeouts.isEmpty()) {
            long timeoutInMillis = applicationTimeouts.get(ApplicationTimeoutType.LIFETIME).longValue();
            timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
            if (isAppInCompletedStates()) {
                // if application configured with timeout and finished before timeout
                // happens then remaining time should not be calculated.
                timeout.setRemainingTime(0);
            } else {
                timeout.setRemainingTime(Math.max((timeoutInMillis - systemClock.getTime()) / 1000, 0));
            }
        }
        report.setApplicationTimeouts(Collections.singletonMap(timeout.getTimeoutType(), timeout));
        return report;
    } finally {
        this.readLock.unlock();
    }
}