Examples with AMRMTokenIdentifier org.apache.hadoop.yarn.security.AMRMTokenIdentifier used on opensource projects

Example 21 with AMRMTokenIdentifier

use of org.apache.hadoop.yarn.security.AMRMTokenIdentifier in project hadoop by apache.

the class BaseAMRMProxyTest method getApplicationUserInfo.

protected ApplicationUserInfo getApplicationUserInfo(final int testAppId) {
    final ApplicationAttemptId attemptId = getApplicationAttemptId(testAppId);
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString());
    AMRMTokenIdentifier token = new AMRMTokenIdentifier(attemptId, 1);
    ugi.addTokenIdentifier(token);
    return new ApplicationUserInfo(ugi, attemptId);
}

Example 22 with AMRMTokenIdentifier

use of org.apache.hadoop.yarn.security.AMRMTokenIdentifier in project hadoop by apache.

the class RMAppImpl method createAndGetApplicationReport.

@Override
public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) {
    this.readLock.lock();
    try {
        ApplicationAttemptId currentApplicationAttemptId = null;
        org.apache.hadoop.yarn.api.records.Token clientToAMToken = null;
        String trackingUrl = UNAVAILABLE;
        String host = UNAVAILABLE;
        String origTrackingUrl = UNAVAILABLE;
        LogAggregationStatus logAggregationStatus = null;
        int rpcPort = -1;
        ApplicationResourceUsageReport appUsageReport = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
        FinalApplicationStatus finishState = getFinalApplicationStatus();
        String diags = UNAVAILABLE;
        float progress = 0.0f;
        org.apache.hadoop.yarn.api.records.Token amrmToken = null;
        if (allowAccess) {
            trackingUrl = getDefaultProxyTrackingUrl();
            if (this.currentAttempt != null) {
                currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
                trackingUrl = this.currentAttempt.getTrackingUrl();
                origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl();
                if (UserGroupInformation.isSecurityEnabled()) {
                    // get a token so the client can communicate with the app attempt
                    // NOTE: token may be unavailable if the attempt is not running
                    Token<ClientToAMTokenIdentifier> attemptClientToAMToken = this.currentAttempt.createClientToken(clientUserName);
                    if (attemptClientToAMToken != null) {
                        clientToAMToken = BuilderUtils.newClientToAMToken(attemptClientToAMToken.getIdentifier(), attemptClientToAMToken.getKind().toString(), attemptClientToAMToken.getPassword(), attemptClientToAMToken.getService().toString());
                    }
                }
                host = this.currentAttempt.getHost();
                rpcPort = this.currentAttempt.getRpcPort();
                appUsageReport = currentAttempt.getApplicationResourceUsageReport();
                progress = currentAttempt.getProgress();
                logAggregationStatus = this.getLogAggregationStatusForAppReport();
            }
            //if the diagnostics is not already set get it from attempt
            diags = getDiagnostics().toString();
            if (currentAttempt != null && currentAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
                if (getApplicationSubmissionContext().getUnmanagedAM() && clientUserName != null && getUser().equals(clientUserName)) {
                    Token<AMRMTokenIdentifier> token = currentAttempt.getAMRMToken();
                    if (token != null) {
                        amrmToken = BuilderUtils.newAMRMToken(token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString());
                    }
                }
            }
            RMAppMetrics rmAppMetrics = getRMAppMetrics();
            appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
            appUsageReport.setPreemptedMemorySeconds(rmAppMetrics.getPreemptedMemorySeconds());
            appUsageReport.setPreemptedVcoreSeconds(rmAppMetrics.getPreemptedVcoreSeconds());
        }
        if (currentApplicationAttemptId == null) {
            currentApplicationAttemptId = BuilderUtils.newApplicationAttemptId(this.applicationId, DUMMY_APPLICATION_ATTEMPT_NUMBER);
        }
        ApplicationReport report = BuilderUtils.newApplicationReport(this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, amrmToken, applicationTags, this.getApplicationPriority());
        report.setLogAggregationStatus(logAggregationStatus);
        report.setUnmanagedApp(submissionContext.getUnmanagedAM());
        report.setAppNodeLabelExpression(getAppNodeLabelExpression());
        report.setAmNodeLabelExpression(getAmNodeLabelExpression());
        ApplicationTimeout timeout = ApplicationTimeout.newInstance(ApplicationTimeoutType.LIFETIME, UNLIMITED, UNKNOWN);
        // are supported in YARN-5692, the below logic need to be changed.
        if (!this.applicationTimeouts.isEmpty()) {
            long timeoutInMillis = applicationTimeouts.get(ApplicationTimeoutType.LIFETIME).longValue();
            timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
            if (isAppInCompletedStates()) {
                // if application configured with timeout and finished before timeout
                // happens then remaining time should not be calculated.
                timeout.setRemainingTime(0);
            } else {
                timeout.setRemainingTime(Math.max((timeoutInMillis - systemClock.getTime()) / 1000, 0));
            }
        }
        report.setApplicationTimeouts(Collections.singletonMap(timeout.getTimeoutType(), timeout));
        return report;
    } finally {
        this.readLock.unlock();
    }
}

Example 23 with AMRMTokenIdentifier

use of org.apache.hadoop.yarn.security.AMRMTokenIdentifier in project hadoop by apache.

the class TestRMRestart method testAppAttemptTokensRestoredOnRMRestart.

@Test(timeout = 60000)
public void testAppAttemptTokensRestoredOnRMRestart() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<ApplicationId, ApplicationStateData> rmAppState = rmState.getApplicationState();
    MockRM rm1 = new TestSecurityMockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("0.0.0.0:4321", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // submit an app
    RMApp app1 = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), "default");
    // assert app info is saved
    ApplicationStateData appState = rmAppState.get(app1.getApplicationId());
    Assert.assertNotNull(appState);
    // Allocate the AM
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
    rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
    // assert attempt info is saved
    ApplicationAttemptStateData attemptState = appState.getAttempt(attemptId1);
    Assert.assertNotNull(attemptState);
    Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1), attemptState.getMasterContainer().getId());
    // the clientTokenMasterKey that are generated when
    // RMAppAttempt is created,
    byte[] clientTokenMasterKey = attempt1.getClientTokenMasterKey().getEncoded();
    // assert application credentials are saved
    Credentials savedCredentials = attemptState.getAppAttemptTokens();
    Assert.assertArrayEquals("client token master key not saved", clientTokenMasterKey, savedCredentials.getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
    // start new RM
    MockRM rm2 = new TestSecurityMockRM(conf, memStore);
    rm2.start();
    RMApp loadedApp1 = rm2.getRMContext().getRMApps().get(app1.getApplicationId());
    RMAppAttempt loadedAttempt1 = loadedApp1.getRMAppAttempt(attemptId1);
    // assert loaded attempt recovered
    Assert.assertNotNull(loadedAttempt1);
    // assert client token master key is recovered back to api-versioned
    // client token master key
    Assert.assertEquals("client token master key not restored", attempt1.getClientTokenMasterKey(), loadedAttempt1.getClientTokenMasterKey());
    // assert ClientTokenSecretManager also knows about the key
    Assert.assertArrayEquals(clientTokenMasterKey, rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1).getEncoded());
    // assert AMRMTokenSecretManager also knows about the AMRMToken password
    Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
    Assert.assertArrayEquals(amrmToken.getPassword(), rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(amrmToken.decodeIdentifier()));
}

Example 24 with AMRMTokenIdentifier

use of org.apache.hadoop.yarn.security.AMRMTokenIdentifier in project hadoop by apache.

the class TestAMRMRPCNodeUpdates method allocate.

private AllocateResponse allocate(final ApplicationAttemptId attemptId, final AllocateRequest req) throws Exception {
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString());
    Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(attemptId.getApplicationId()).getRMAppAttempt(attemptId).getAMRMToken();
    ugi.addTokenIdentifier(token.decodeIdentifier());
    return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() {

        @Override
        public AllocateResponse run() throws Exception {
            return amService.allocate(req);
        }
    });
}

Example 25 with AMRMTokenIdentifier

use of org.apache.hadoop.yarn.security.AMRMTokenIdentifier in project hadoop by apache.

the class TestZKRMStateStorePerf method run.

@SuppressWarnings("unchecked")
@Override
public int run(String[] args) {
    LOG.info("Starting ZKRMStateStorePerf ver." + version);
    int numApp = ZK_PERF_NUM_APP_DEFAULT;
    int numAppAttemptPerApp = ZK_PERF_NUM_APPATTEMPT_PER_APP;
    String hostPort = null;
    boolean launchLocalZK = true;
    if (args.length == 0) {
        System.err.println("Missing arguments.");
        return -1;
    }
    for (int i = 0; i < args.length; i++) {
        // parse command line
        if (args[i].equalsIgnoreCase("-appsize")) {
            numApp = Integer.parseInt(args[++i]);
        } else if (args[i].equalsIgnoreCase("-appattemptsize")) {
            numAppAttemptPerApp = Integer.parseInt(args[++i]);
        } else if (args[i].equalsIgnoreCase("-hostPort")) {
            hostPort = args[++i];
            launchLocalZK = false;
        } else if (args[i].equalsIgnoreCase("-workingZnode")) {
            workingZnode = args[++i];
        } else {
            System.err.println("Illegal argument: " + args[i]);
            return -1;
        }
    }
    if (launchLocalZK) {
        try {
            setUpZKServer();
        } catch (Exception e) {
            System.err.println("failed to setup. : " + e.getMessage());
            return -1;
        }
    }
    initStore(hostPort);
    long submitTime = System.currentTimeMillis();
    long startTime = System.currentTimeMillis() + 1234;
    ArrayList<ApplicationId> applicationIds = new ArrayList<>();
    ArrayList<RMApp> rmApps = new ArrayList<>();
    ArrayList<ApplicationAttemptId> attemptIds = new ArrayList<>();
    HashMap<ApplicationId, Set<ApplicationAttemptId>> appIdsToAttemptId = new HashMap<>();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    for (int i = 0; i < numApp; i++) {
        ApplicationId appId = ApplicationId.newInstance(clusterTimeStamp, i);
        applicationIds.add(appId);
        ArrayList<ApplicationAttemptId> attemptIdsForThisApp = new ArrayList<>();
        for (int j = 0; j < numAppAttemptPerApp; j++) {
            ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, j);
            attemptIdsForThisApp.add(attemptId);
        }
        appIdsToAttemptId.put(appId, new LinkedHashSet(attemptIdsForThisApp));
        attemptIds.addAll(attemptIdsForThisApp);
    }
    for (ApplicationId appId : applicationIds) {
        RMApp app = null;
        try {
            app = storeApp(store, appId, submitTime, startTime);
        } catch (Exception e) {
            System.err.println("failed to create Application Znode. : " + e.getMessage());
            return -1;
        }
        waitNotify(dispatcher);
        rmApps.add(app);
    }
    for (ApplicationAttemptId attemptId : attemptIds) {
        Token<AMRMTokenIdentifier> tokenId = generateAMRMToken(attemptId, appTokenMgr);
        SecretKey clientTokenKey = clientToAMTokenMgr.createMasterKey(attemptId);
        try {
            storeAttempt(store, attemptId, ContainerId.newContainerId(attemptId, 0L).toString(), tokenId, clientTokenKey, dispatcher);
        } catch (Exception e) {
            System.err.println("failed to create AppAttempt Znode. : " + e.getMessage());
            return -1;
        }
    }
    long storeStart = System.currentTimeMillis();
    try {
        store.loadState();
    } catch (Exception e) {
        System.err.println("failed to locaState from ZKRMStateStore. : " + e.getMessage());
        return -1;
    }
    long storeEnd = System.currentTimeMillis();
    long loadTime = storeEnd - storeStart;
    String resultMsg = "ZKRMStateStore takes " + loadTime + " msec to loadState.";
    LOG.info(resultMsg);
    System.out.println(resultMsg);
    // cleanup
    try {
        for (RMApp app : rmApps) {
            ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), app.getApplicationSubmissionContext(), app.getUser());
            ApplicationId appId = app.getApplicationId();
            Map m = mock(Map.class);
            when(m.keySet()).thenReturn(appIdsToAttemptId.get(appId));
            appState.attempts = m;
            store.removeApplicationStateInternal(appState);
        }
    } catch (Exception e) {
        System.err.println("failed to cleanup. : " + e.getMessage());
        return -1;
    }
    return 0;
}