Examples with MasterKeyData org.apache.hadoop.yarn.server.security.MasterKeyData used on opensource projects

Example 6 with MasterKeyData

use of org.apache.hadoop.yarn.server.security.MasterKeyData in project hadoop by apache.

the class AMRMTokenSecretManager method recover.

public void recover(RMState state) {
    if (state.getAMRMTokenSecretManagerState() != null) {
        // recover the current master key
        MasterKey currentKey = state.getAMRMTokenSecretManagerState().getCurrentMasterKey();
        this.currentMasterKey = new MasterKeyData(currentKey, createSecretKey(currentKey.getBytes().array()));
        // recover the next master key if not null
        MasterKey nextKey = state.getAMRMTokenSecretManagerState().getNextMasterKey();
        if (nextKey != null) {
            this.nextMasterKey = new MasterKeyData(nextKey, createSecretKey(nextKey.getBytes().array()));
            this.timer.schedule(new NextKeyActivator(), this.activationDelay);
        }
    }
}

Example 7 with MasterKeyData

use of org.apache.hadoop.yarn.server.security.MasterKeyData in project hadoop by apache.

the class RMStateStoreTestBase method testAMRMTokenSecretManagerStateStore.

public void testAMRMTokenSecretManagerStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
    System.out.println("Start testing");
    RMStateStore store = stateStoreHelper.getRMStateStore();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getStateStore()).thenReturn(store);
    Configuration conf = new YarnConfiguration();
    AMRMTokenSecretManager appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
    //create and save the first masterkey
    MasterKeyData firstMasterKeyData = appTokenMgr.createNewMasterKey();
    AMRMTokenSecretManagerState state1 = AMRMTokenSecretManagerState.newInstance(firstMasterKeyData.getMasterKey(), null);
    rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManager(state1, false);
    // load state
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Assert.assertNotNull(state.getAMRMTokenSecretManagerState());
    Assert.assertEquals(firstMasterKeyData.getMasterKey(), state.getAMRMTokenSecretManagerState().getCurrentMasterKey());
    Assert.assertNull(state.getAMRMTokenSecretManagerState().getNextMasterKey());
    //create and save the second masterkey
    MasterKeyData secondMasterKeyData = appTokenMgr.createNewMasterKey();
    AMRMTokenSecretManagerState state2 = AMRMTokenSecretManagerState.newInstance(firstMasterKeyData.getMasterKey(), secondMasterKeyData.getMasterKey());
    rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManager(state2, true);
    // load state
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    RMState state_2 = store.loadState();
    Assert.assertNotNull(state_2.getAMRMTokenSecretManagerState());
    Assert.assertEquals(firstMasterKeyData.getMasterKey(), state_2.getAMRMTokenSecretManagerState().getCurrentMasterKey());
    Assert.assertEquals(secondMasterKeyData.getMasterKey(), state_2.getAMRMTokenSecretManagerState().getNextMasterKey());
    // re-create the masterKeyData based on the recovered masterkey
    // should have the same secretKey
    appTokenMgr.recover(state_2);
    Assert.assertEquals(appTokenMgr.getCurrnetMasterKeyData().getSecretKey(), firstMasterKeyData.getSecretKey());
    Assert.assertEquals(appTokenMgr.getNextMasterKeyData().getSecretKey(), secondMasterKeyData.getSecretKey());
    store.close();
}

Example 8 with MasterKeyData

use of org.apache.hadoop.yarn.server.security.MasterKeyData in project hadoop by apache.

the class RMStateStoreTestBase method testRMAppStateStore.

void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, StoreStateVerifier verifier) throws Exception {
    long submitTime = System.currentTimeMillis();
    long startTime = System.currentTimeMillis() + 1234;
    Configuration conf = new YarnConfiguration();
    RMStateStore store = stateStoreHelper.getRMStateStore();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getStateStore()).thenReturn(store);
    AMRMTokenSecretManager appTokenMgr = spy(new AMRMTokenSecretManager(conf, rmContext));
    MasterKeyData masterKeyData = appTokenMgr.createNewMasterKey();
    when(appTokenMgr.getMasterKey()).thenReturn(masterKeyData);
    ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
    ApplicationAttemptId attemptId1 = ApplicationAttemptId.fromString("appattempt_1352994193343_0001_000001");
    ApplicationId appId1 = attemptId1.getApplicationId();
    storeApp(store, appId1, submitTime, startTime);
    verifier.afterStoreApp(store, appId1);
    // create application token and client token key for attempt1
    Token<AMRMTokenIdentifier> appAttemptToken1 = generateAMRMToken(attemptId1, appTokenMgr);
    SecretKey clientTokenKey1 = clientToAMTokenMgr.createMasterKey(attemptId1);
    ContainerId containerId1 = storeAttempt(store, attemptId1, "container_1352994193343_0001_01_000001", appAttemptToken1, clientTokenKey1, dispatcher).getMasterContainer().getId();
    String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
    ApplicationAttemptId attemptId2 = ApplicationAttemptId.fromString(appAttemptIdStr2);
    // create application token and client token key for attempt2
    Token<AMRMTokenIdentifier> appAttemptToken2 = generateAMRMToken(attemptId2, appTokenMgr);
    SecretKey clientTokenKey2 = clientToAMTokenMgr.createMasterKey(attemptId2);
    ContainerId containerId2 = storeAttempt(store, attemptId2, "container_1352994193343_0001_02_000001", appAttemptToken2, clientTokenKey2, dispatcher).getMasterContainer().getId();
    ApplicationAttemptId attemptIdRemoved = ApplicationAttemptId.fromString("appattempt_1352994193343_0002_000001");
    ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
    storeApp(store, appIdRemoved, submitTime, startTime);
    storeAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null, null, dispatcher);
    verifier.afterStoreAppAttempt(store, attemptIdRemoved);
    RMApp mockRemovedApp = mock(RMApp.class);
    RMAppAttemptMetrics mockRmAppAttemptMetrics = mock(RMAppAttemptMetrics.class);
    HashMap<ApplicationAttemptId, RMAppAttempt> attempts = new HashMap<ApplicationAttemptId, RMAppAttempt>();
    ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
    context.setApplicationId(appIdRemoved);
    when(mockRemovedApp.getSubmitTime()).thenReturn(submitTime);
    when(mockRemovedApp.getApplicationSubmissionContext()).thenReturn(context);
    when(mockRemovedApp.getAppAttempts()).thenReturn(attempts);
    when(mockRemovedApp.getUser()).thenReturn("user1");
    RMAppAttempt mockRemovedAttempt = mock(RMAppAttempt.class);
    when(mockRemovedAttempt.getAppAttemptId()).thenReturn(attemptIdRemoved);
    when(mockRemovedAttempt.getRMAppAttemptMetrics()).thenReturn(mockRmAppAttemptMetrics);
    when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()).thenReturn(new AggregateAppResourceUsage(0, 0));
    attempts.put(attemptIdRemoved, mockRemovedAttempt);
    store.removeApplication(mockRemovedApp);
    // remove application directory recursively.
    storeApp(store, appIdRemoved, submitTime, startTime);
    storeAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null, null, dispatcher);
    store.removeApplication(mockRemovedApp);
    // let things settle down
    Thread.sleep(1000);
    store.close();
    // give tester a chance to modify app state in the store
    modifyAppState();
    // load state
    store = stateStoreHelper.getRMStateStore();
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<ApplicationId, ApplicationStateData> rmAppState = state.getApplicationState();
    ApplicationStateData appState = rmAppState.get(appId1);
    // app is loaded
    assertNotNull(appState);
    // app is loaded correctly
    assertEquals(submitTime, appState.getSubmitTime());
    assertEquals(startTime, appState.getStartTime());
    // submission context is loaded correctly
    assertEquals(appId1, appState.getApplicationSubmissionContext().getApplicationId());
    ApplicationAttemptStateData attemptState = appState.getAttempt(attemptId1);
    // attempt1 is loaded correctly
    assertNotNull(attemptState);
    assertEquals(attemptId1, attemptState.getAttemptId());
    assertEquals(-1000, attemptState.getAMContainerExitStatus());
    // attempt1 container is loaded correctly
    assertEquals(containerId1, attemptState.getMasterContainer().getId());
    // attempt1 client token master key is loaded correctly
    assertArrayEquals(clientTokenKey1.getEncoded(), attemptState.getAppAttemptTokens().getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
    assertEquals("context", appState.getCallerContext().getContext());
    attemptState = appState.getAttempt(attemptId2);
    // attempt2 is loaded correctly
    assertNotNull(attemptState);
    assertEquals(attemptId2, attemptState.getAttemptId());
    // attempt2 container is loaded correctly
    assertEquals(containerId2, attemptState.getMasterContainer().getId());
    // attempt2 client token master key is loaded correctly
    assertArrayEquals(clientTokenKey2.getEncoded(), attemptState.getAppAttemptTokens().getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
    //******* update application/attempt state *******//
    ApplicationStateData appState2 = ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), appState.getApplicationSubmissionContext(), RMAppState.FINISHED, "appDiagnostics", 1234, appState.getCallerContext());
    appState2.attempts.putAll(appState.attempts);
    store.updateApplicationState(appState2);
    ApplicationAttemptStateData oldAttemptState = attemptState;
    ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.newInstance(oldAttemptState.getAttemptId(), oldAttemptState.getMasterContainer(), oldAttemptState.getAppAttemptTokens(), oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 100, oldAttemptState.getFinishTime(), 0, 0, 0, 0);
    store.updateApplicationAttemptState(newAttemptState);
    // test updating the state of an app/attempt whose initial state was not
    // saved.
    ApplicationId dummyAppId = ApplicationId.newInstance(1234, 10);
    ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl();
    dummyContext.setApplicationId(dummyAppId);
    ApplicationStateData dummyApp = ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), dummyContext, RMAppState.FINISHED, "appDiagnostics", 1234, null);
    store.updateApplicationState(dummyApp);
    ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.newInstance(dummyAppId, 6);
    ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.newInstance(dummyAttemptId, oldAttemptState.getMasterContainer(), oldAttemptState.getAppAttemptTokens(), oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 111, oldAttemptState.getFinishTime(), 0, 0, 0, 0);
    store.updateApplicationAttemptState(dummyAttempt);
    // let things settle down
    Thread.sleep(1000);
    store.close();
    // check updated application state.
    store = stateStoreHelper.getRMStateStore();
    store.setRMDispatcher(dispatcher);
    RMState newRMState = store.loadState();
    Map<ApplicationId, ApplicationStateData> newRMAppState = newRMState.getApplicationState();
    assertNotNull(newRMAppState.get(dummyApp.getApplicationSubmissionContext().getApplicationId()));
    ApplicationStateData updatedAppState = newRMAppState.get(appId1);
    assertEquals(appState.getApplicationSubmissionContext().getApplicationId(), updatedAppState.getApplicationSubmissionContext().getApplicationId());
    assertEquals(appState.getSubmitTime(), updatedAppState.getSubmitTime());
    assertEquals(appState.getStartTime(), updatedAppState.getStartTime());
    assertEquals(appState.getUser(), updatedAppState.getUser());
    // new app state fields
    assertEquals(RMAppState.FINISHED, updatedAppState.getState());
    assertEquals("appDiagnostics", updatedAppState.getDiagnostics());
    assertEquals(1234, updatedAppState.getFinishTime());
    // check updated attempt state
    assertNotNull(newRMAppState.get(dummyApp.getApplicationSubmissionContext().getApplicationId()).getAttempt(dummyAttemptId));
    ApplicationAttemptStateData updatedAttemptState = updatedAppState.getAttempt(newAttemptState.getAttemptId());
    assertEquals(oldAttemptState.getAttemptId(), updatedAttemptState.getAttemptId());
    assertEquals(containerId2, updatedAttemptState.getMasterContainer().getId());
    assertArrayEquals(clientTokenKey2.getEncoded(), attemptState.getAppAttemptTokens().getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
    // new attempt state fields
    assertEquals(RMAppAttemptState.FINISHED, updatedAttemptState.getState());
    assertEquals("myTrackingUrl", updatedAttemptState.getFinalTrackingUrl());
    assertEquals("attemptDiagnostics", updatedAttemptState.getDiagnostics());
    assertEquals(100, updatedAttemptState.getAMContainerExitStatus());
    assertEquals(FinalApplicationStatus.SUCCEEDED, updatedAttemptState.getFinalApplicationStatus());
    // assert store is in expected state after everything is cleaned
    assertTrue(stateStoreHelper.isFinalStateValid());
    store.close();
}

Example 9 with MasterKeyData

use of org.apache.hadoop.yarn.server.security.MasterKeyData in project hadoop by apache.

the class AMRMProxyService method updateAMRMTokens.

private void updateAMRMTokens(AMRMTokenIdentifier amrmTokenIdentifier, RequestInterceptorChainWrapper pipeline, AllocateResponse allocateResponse) {
    AMRMProxyApplicationContextImpl context = (AMRMProxyApplicationContextImpl) pipeline.getRootInterceptor().getApplicationContext();
    // the real ARMRMToken in the current context
    if (allocateResponse.getAMRMToken() != null) {
        LOG.info("RM rolled master-key for amrm-tokens");
        org.apache.hadoop.yarn.api.records.Token token = allocateResponse.getAMRMToken();
        // Do not propagate this info back to AM
        allocateResponse.setAMRMToken(null);
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newTokenId = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token.getIdentifier().array(), token.getPassword().array(), new Text(token.getKind()), new Text(token.getService()));
        context.setAMRMToken(newTokenId);
    }
    // Check if the local AMRMToken is rolled up and update the context and
    // response accordingly
    MasterKeyData nextMasterKey = this.secretManager.getNextMasterKeyData();
    if (nextMasterKey != null && nextMasterKey.getMasterKey().getKeyId() != amrmTokenIdentifier.getKeyId()) {
        Token<AMRMTokenIdentifier> localToken = context.getLocalAMRMToken();
        if (nextMasterKey.getMasterKey().getKeyId() != context.getLocalAMRMTokenKeyId()) {
            LOG.info("The local AMRMToken has been rolled-over." + " Send new local AMRMToken back to application: " + pipeline.getApplicationId());
            localToken = this.secretManager.createAndGetAMRMToken(pipeline.getApplicationAttemptId());
            context.setLocalAMRMToken(localToken);
        }
        allocateResponse.setAMRMToken(org.apache.hadoop.yarn.api.records.Token.newInstance(localToken.getIdentifier(), localToken.getKind().toString(), localToken.getPassword(), localToken.getService().toString()));
    }
}

Example 10 with MasterKeyData

use of org.apache.hadoop.yarn.server.security.MasterKeyData in project hadoop by apache.

the class NMTokenSecretManagerInNM method setMasterKey.

/**
   * Used by NodeManagers to create a token-secret-manager with the key
   * obtained from the RM. This can happen during registration or when the RM
   * rolls the master-key and signal the NM.
   */
@Private
public synchronized void setMasterKey(MasterKey masterKey) {
    // Update keys only if the key has changed.
    if (super.currentMasterKey == null || super.currentMasterKey.getMasterKey().getKeyId() != masterKey.getKeyId()) {
        LOG.info("Rolling master-key for container-tokens, got key with id " + masterKey.getKeyId());
        if (super.currentMasterKey != null) {
            updatePreviousMasterKey(super.currentMasterKey);
        }
        updateCurrentMasterKey(new MasterKeyData(masterKey, createSecretKey(masterKey.getBytes().array())));
    }
}