Examples with MasterKeyData org.apache.hadoop.yarn.server.security.MasterKeyData used on opensource projects

Example 1 with MasterKeyData

use of org.apache.hadoop.yarn.server.security.MasterKeyData in project hadoop by apache.

the class NMContainerTokenSecretManager method retrievePassword.

/**
   * Override of this is to validate ContainerTokens generated by using
   * different {@link MasterKey}s.
   */
@Override
public synchronized byte[] retrievePassword(ContainerTokenIdentifier identifier) throws SecretManager.InvalidToken {
    int keyId = identifier.getMasterKeyId();
    MasterKeyData masterKeyToUse = null;
    if (this.previousMasterKey != null && keyId == this.previousMasterKey.getMasterKey().getKeyId()) {
        // A container-launch has come in with a token generated off the last
        // master-key
        masterKeyToUse = this.previousMasterKey;
    } else if (keyId == super.currentMasterKey.getMasterKey().getKeyId()) {
        // A container-launch has come in with a token generated off the current
        // master-key
        masterKeyToUse = super.currentMasterKey;
    }
    if (nodeHostAddr != null && !identifier.getNmHostAddress().equals(nodeHostAddr)) {
        // Valid container token used for incorrect node.
        throw new SecretManager.InvalidToken("Given Container " + identifier.getContainerID().toString() + " identifier is not valid for current Node manager. Expected : " + nodeHostAddr + " Found : " + identifier.getNmHostAddress());
    }
    if (masterKeyToUse != null) {
        return retrievePasswordInternal(identifier, masterKeyToUse);
    }
    // old-master-keys.
    throw new SecretManager.InvalidToken("Given Container " + identifier.getContainerID().toString() + " seems to have an illegally generated token.");
}

Example 2 with MasterKeyData

use of org.apache.hadoop.yarn.server.security.MasterKeyData in project hadoop by apache.

the class NMTokenSecretManagerInNM method recover.

public synchronized void recover() throws IOException {
    RecoveredNMTokensState state = stateStore.loadNMTokensState();
    MasterKey key = state.getCurrentMasterKey();
    if (key != null) {
        super.currentMasterKey = new MasterKeyData(key, createSecretKey(key.getBytes().array()));
    }
    key = state.getPreviousMasterKey();
    if (key != null) {
        previousMasterKey = new MasterKeyData(key, createSecretKey(key.getBytes().array()));
    }
    // restore the serial number from the current master key
    if (super.currentMasterKey != null) {
        super.serialNo = super.currentMasterKey.getMasterKey().getKeyId() + 1;
    }
    for (Map.Entry<ApplicationAttemptId, MasterKey> entry : state.getApplicationMasterKeys().entrySet()) {
        key = entry.getValue();
        oldMasterKeys.put(entry.getKey(), new MasterKeyData(key, createSecretKey(key.getBytes().array())));
    }
    // reconstruct app to app attempts map
    appToAppAttemptMap.clear();
    for (ApplicationAttemptId attempt : oldMasterKeys.keySet()) {
        ApplicationId app = attempt.getApplicationId();
        List<ApplicationAttemptId> attempts = appToAppAttemptMap.get(app);
        if (attempts == null) {
            attempts = new ArrayList<ApplicationAttemptId>();
            appToAppAttemptMap.put(app, attempts);
        }
        attempts.add(attempt);
    }
}

Example 3 with MasterKeyData

use of org.apache.hadoop.yarn.server.security.MasterKeyData in project hadoop by apache.

the class NMTokenSecretManagerInNM method retrievePassword.

/**
   * This method will be used to verify NMTokens generated by different master
   * keys.
   */
@Override
public synchronized byte[] retrievePassword(NMTokenIdentifier identifier) throws InvalidToken {
    int keyId = identifier.getKeyId();
    ApplicationAttemptId appAttemptId = identifier.getApplicationAttemptId();
    /*
     * MasterKey used for retrieving password will be as follows. 1) By default
     * older saved master key will be used. 2) If identifier's master key id
     * matches that of previous master key id then previous key will be used. 3)
     * If identifier's master key id matches that of current master key id then
     * current key will be used.
     */
    MasterKeyData oldMasterKey = oldMasterKeys.get(appAttemptId);
    MasterKeyData masterKeyToUse = oldMasterKey;
    if (previousMasterKey != null && keyId == previousMasterKey.getMasterKey().getKeyId()) {
        masterKeyToUse = previousMasterKey;
    } else if (keyId == currentMasterKey.getMasterKey().getKeyId()) {
        masterKeyToUse = currentMasterKey;
    }
    if (nodeId != null && !identifier.getNodeId().equals(nodeId)) {
        throw new InvalidToken("Given NMToken for application : " + appAttemptId.toString() + " is not valid for current node manager." + "expected : " + nodeId.toString() + " found : " + identifier.getNodeId().toString());
    }
    if (masterKeyToUse != null) {
        byte[] password = retrivePasswordInternal(identifier, masterKeyToUse);
        LOG.debug("NMToken password retrieved successfully!!");
        return password;
    }
    throw new InvalidToken("Given NMToken for application : " + appAttemptId.toString() + " seems to have been generated illegally.");
}

Example 4 with MasterKeyData

use of org.apache.hadoop.yarn.server.security.MasterKeyData in project hadoop by apache.

the class TestRMAppAttemptTransitions method setUp.

@SuppressWarnings("deprecation")
@Before
public void setUp() throws Exception {
    AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
    if (isSecurityEnabled) {
        authMethod = AuthenticationMethod.KERBEROS;
    }
    SecurityUtil.setAuthenticationMethod(authMethod, conf);
    UserGroupInformation.setConfiguration(conf);
    InlineDispatcher rmDispatcher = new InlineDispatcher();
    ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class);
    amLivelinessMonitor = mock(AMLivelinessMonitor.class);
    amFinishingMonitor = mock(AMLivelinessMonitor.class);
    writer = mock(RMApplicationHistoryWriter.class);
    MasterKeyData masterKeyData = amRMTokenManager.createNewMasterKey();
    when(amRMTokenManager.getMasterKey()).thenReturn(masterKeyData);
    rmContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, amRMTokenManager, new RMContainerTokenSecretManager(conf), nmTokenManager, clientToAMTokenManager);
    store = mock(RMStateStore.class);
    ((RMContextImpl) rmContext).setStateStore(store);
    publisher = mock(SystemMetricsPublisher.class);
    rmContext.setSystemMetricsPublisher(publisher);
    rmContext.setRMApplicationHistoryWriter(writer);
    scheduler = mock(YarnScheduler.class);
    masterService = mock(ApplicationMasterService.class);
    applicationMasterLauncher = mock(ApplicationMasterLauncher.class);
    rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher());
    rmDispatcher.register(RMAppEventType.class, new TestApplicationEventDispatcher());
    rmDispatcher.register(SchedulerEventType.class, new TestSchedulerEventDispatcher());
    rmDispatcher.register(AMLauncherEventType.class, new TestAMLauncherEventDispatcher());
    rmnodeEventHandler = mock(RMNodeImpl.class);
    rmDispatcher.register(RMNodeEventType.class, rmnodeEventHandler);
    rmDispatcher.init(conf);
    rmDispatcher.start();
    ApplicationId applicationId = MockApps.newAppID(appId++);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    resourceScheduler = mock(ResourceScheduler.class);
    ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class);
    when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
    when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
    when(resourceScheduler.getAppResourceUsageReport((ApplicationAttemptId) Matchers.any())).thenReturn(appResUsgRpt);
    spyRMContext = spy(rmContext);
    Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();
    final String user = MockApps.newUserName();
    final String queue = MockApps.newQueue();
    submissionContext = mock(ApplicationSubmissionContext.class);
    when(submissionContext.getQueue()).thenReturn(queue);
    Resource resource = BuilderUtils.newResource(1536, 1);
    ContainerLaunchContext amContainerSpec = BuilderUtils.newContainerLaunchContext(null, null, null, null, null, null);
    when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
    when(submissionContext.getResource()).thenReturn(resource);
    unmanagedAM = false;
    application = mock(RMAppImpl.class);
    applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1), application);
    when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
    when(application.getApplicationId()).thenReturn(applicationId);
    spyRMContext.getRMApps().put(application.getApplicationId(), application);
    testAppAttemptNewState();
}

Example 5 with MasterKeyData

use of org.apache.hadoop.yarn.server.security.MasterKeyData in project hadoop by apache.

the class TestAMRMTokens method testMasterKeyRollOver.

/**
   * Validate master-key-roll-over and that tokens are usable even after
   * master-key-roll-over.
   * 
   * @throws Exception
   */
@Test
public void testMasterKeyRollOver() throws Exception {
    conf.setLong(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, rolling_interval_sec);
    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:0");
    MyContainerManager containerManager = new MyContainerManager();
    final MockRMWithAMS rm = new MockRMWithAMS(conf, containerManager);
    rm.start();
    Long startTime = System.currentTimeMillis();
    final Configuration conf = rm.getConfig();
    final YarnRPC rpc = YarnRPC.create(conf);
    ApplicationMasterProtocol rmClient = null;
    AMRMTokenSecretManager appTokenSecretManager = rm.getRMContext().getAMRMTokenSecretManager();
    MasterKeyData oldKey = appTokenSecretManager.getMasterKey();
    Assert.assertNotNull(oldKey);
    try {
        MockNM nm1 = rm.registerNode("localhost:1234", 5120);
        RMApp app = rm.submitApp(1024);
        nm1.nodeHeartbeat(true);
        int waitCount = 0;
        while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts) {
            LOG.info("Waiting for AM Launch to happen..");
            Thread.sleep(1000);
        }
        Assert.assertNotNull(containerManager.containerTokens);
        RMAppAttempt attempt = app.getCurrentAppAttempt();
        ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
        // Create a client to the RM.
        UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
        Credentials credentials = containerManager.getContainerCredentials();
        final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
        Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
        currentUser.addToken(amRMToken);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class);
        rmClient.registerApplicationMaster(request);
        // One allocate call.
        AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
        Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
        // At mean time, the old AMRMToken should continue to work
        while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
            rmClient.allocate(allocateRequest);
            Thread.sleep(500);
        }
        MasterKeyData newKey = appTokenSecretManager.getMasterKey();
        Assert.assertNotNull(newKey);
        Assert.assertFalse("Master key should have changed!", oldKey.equals(newKey));
        // Another allocate call with old AMRMToken. Should continue to work.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
        waitCount = 0;
        while (waitCount++ <= maxWaitAttempts) {
            if (appTokenSecretManager.getCurrnetMasterKeyData() != oldKey) {
                break;
            }
            try {
                rmClient.allocate(allocateRequest);
            } catch (Exception ex) {
                break;
            }
            Thread.sleep(200);
        }
        // active the nextMasterKey, and replace the currentMasterKey
        Assert.assertTrue(appTokenSecretManager.getCurrnetMasterKeyData().equals(newKey));
        Assert.assertTrue(appTokenSecretManager.getMasterKey().equals(newKey));
        Assert.assertTrue(appTokenSecretManager.getNextMasterKeyData() == null);
        // Create a new Token
        Token<AMRMTokenIdentifier> newToken = appTokenSecretManager.createAndGetAMRMToken(applicationAttemptId);
        SecurityUtil.setTokenService(newToken, rmBindAddress);
        currentUser.addToken(newToken);
        // Another allocate call. Should continue to work.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        allocateRequest = Records.newRecord(AllocateRequest.class);
        Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
        // Should not work by using the old AMRMToken.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        try {
            currentUser.addToken(amRMToken);
            rmClient = createRMClient(rm, conf, rpc, currentUser);
            allocateRequest = Records.newRecord(AllocateRequest.class);
            Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
            Assert.fail("The old Token should not work");
        } catch (Exception ex) {
        // expect exception
        }
    } finally {
        rm.stop();
        if (rmClient != null) {
            // To avoid using cached client
            rpc.stopProxy(rmClient, conf);
        }
    }
}