Examples with MasterKey org.apache.hadoop.yarn.server.api.records.MasterKey used on opensource projects

Example 1 with MasterKey

use of org.apache.hadoop.yarn.server.api.records.MasterKey in project hadoop by apache.

the class NMTokenSecretManagerInNM method recover.

public synchronized void recover() throws IOException {
    RecoveredNMTokensState state = stateStore.loadNMTokensState();
    MasterKey key = state.getCurrentMasterKey();
    if (key != null) {
        super.currentMasterKey = new MasterKeyData(key, createSecretKey(key.getBytes().array()));
    }
    key = state.getPreviousMasterKey();
    if (key != null) {
        previousMasterKey = new MasterKeyData(key, createSecretKey(key.getBytes().array()));
    }
    // restore the serial number from the current master key
    if (super.currentMasterKey != null) {
        super.serialNo = super.currentMasterKey.getMasterKey().getKeyId() + 1;
    }
    for (Map.Entry<ApplicationAttemptId, MasterKey> entry : state.getApplicationMasterKeys().entrySet()) {
        key = entry.getValue();
        oldMasterKeys.put(entry.getKey(), new MasterKeyData(key, createSecretKey(key.getBytes().array())));
    }
    // reconstruct app to app attempts map
    appToAppAttemptMap.clear();
    for (ApplicationAttemptId attempt : oldMasterKeys.keySet()) {
        ApplicationId app = attempt.getApplicationId();
        List<ApplicationAttemptId> attempts = appToAppAttemptMap.get(app);
        if (attempts == null) {
            attempts = new ArrayList<ApplicationAttemptId>();
            appToAppAttemptMap.put(app, attempts);
        }
        attempts.add(attempt);
    }
}

Example 2 with MasterKey

use of org.apache.hadoop.yarn.server.api.records.MasterKey in project hadoop by apache.

the class NMLeveldbStateStoreService method loadNMTokensState.

@Override
public RecoveredNMTokensState loadNMTokensState() throws IOException {
    RecoveredNMTokensState state = new RecoveredNMTokensState();
    state.applicationMasterKeys = new HashMap<ApplicationAttemptId, MasterKey>();
    LeveldbIterator iter = null;
    try {
        iter = new LeveldbIterator(db);
        iter.seek(bytes(NM_TOKENS_KEY_PREFIX));
        while (iter.hasNext()) {
            Entry<byte[], byte[]> entry = iter.next();
            String fullKey = asString(entry.getKey());
            if (!fullKey.startsWith(NM_TOKENS_KEY_PREFIX)) {
                break;
            }
            String key = fullKey.substring(NM_TOKENS_KEY_PREFIX.length());
            if (key.equals(CURRENT_MASTER_KEY_SUFFIX)) {
                state.currentMasterKey = parseMasterKey(entry.getValue());
            } else if (key.equals(PREV_MASTER_KEY_SUFFIX)) {
                state.previousMasterKey = parseMasterKey(entry.getValue());
            } else if (key.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
                ApplicationAttemptId attempt;
                try {
                    attempt = ApplicationAttemptId.fromString(key);
                } catch (IllegalArgumentException e) {
                    throw new IOException("Bad application master key state for " + fullKey, e);
                }
                state.applicationMasterKeys.put(attempt, parseMasterKey(entry.getValue()));
            }
        }
    } catch (DBException e) {
        throw new IOException(e);
    } finally {
        if (iter != null) {
            iter.close();
        }
    }
    return state;
}

Example 3 with MasterKey

use of org.apache.hadoop.yarn.server.api.records.MasterKey in project hadoop by apache.

the class TestRegisterNodeManagerResponse method testRoundTrip.

@Test
public void testRoundTrip() throws Exception {
    RegisterNodeManagerResponse resp = recordFactory.newRecordInstance(RegisterNodeManagerResponse.class);
    byte[] b = { 0, 1, 2, 3, 4, 5 };
    MasterKey containerTokenMK = recordFactory.newRecordInstance(MasterKey.class);
    containerTokenMK.setKeyId(54321);
    containerTokenMK.setBytes(ByteBuffer.wrap(b));
    resp.setContainerTokenMasterKey(containerTokenMK);
    MasterKey nmTokenMK = recordFactory.newRecordInstance(MasterKey.class);
    nmTokenMK.setKeyId(12345);
    nmTokenMK.setBytes(ByteBuffer.wrap(b));
    resp.setNMTokenMasterKey(nmTokenMK);
    resp.setNodeAction(NodeAction.NORMAL);
    assertEquals(NodeAction.NORMAL, resp.getNodeAction());
    // Verifying containerTokenMasterKey
    assertNotNull(resp.getContainerTokenMasterKey());
    assertEquals(54321, resp.getContainerTokenMasterKey().getKeyId());
    assertArrayEquals(b, resp.getContainerTokenMasterKey().getBytes().array());
    RegisterNodeManagerResponse respCopy = serDe(resp);
    assertEquals(NodeAction.NORMAL, respCopy.getNodeAction());
    assertNotNull(respCopy.getContainerTokenMasterKey());
    assertEquals(54321, respCopy.getContainerTokenMasterKey().getKeyId());
    assertArrayEquals(b, respCopy.getContainerTokenMasterKey().getBytes().array());
    // Verifying nmTokenMasterKey
    assertNotNull(resp.getNMTokenMasterKey());
    assertEquals(12345, resp.getNMTokenMasterKey().getKeyId());
    assertArrayEquals(b, resp.getNMTokenMasterKey().getBytes().array());
    respCopy = serDe(resp);
    assertEquals(NodeAction.NORMAL, respCopy.getNodeAction());
    assertNotNull(respCopy.getNMTokenMasterKey());
    assertEquals(12345, respCopy.getNMTokenMasterKey().getKeyId());
    assertArrayEquals(b, respCopy.getNMTokenMasterKey().getBytes().array());
}

Example 4 with MasterKey

use of org.apache.hadoop.yarn.server.api.records.MasterKey in project hadoop by apache.

the class TestNMLeveldbStateStoreService method testNMTokenStorage.

@Test
public void testNMTokenStorage() throws IOException {
    // test empty when no state
    RecoveredNMTokensState state = stateStore.loadNMTokensState();
    assertNull(state.getCurrentMasterKey());
    assertNull(state.getPreviousMasterKey());
    assertTrue(state.getApplicationMasterKeys().isEmpty());
    // store a master key and verify recovered
    NMTokenSecretManagerForTest secretMgr = new NMTokenSecretManagerForTest();
    MasterKey currentKey = secretMgr.generateKey();
    stateStore.storeNMTokenCurrentMasterKey(currentKey);
    restartStateStore();
    state = stateStore.loadNMTokensState();
    assertEquals(currentKey, state.getCurrentMasterKey());
    assertNull(state.getPreviousMasterKey());
    assertTrue(state.getApplicationMasterKeys().isEmpty());
    // store a previous key and verify recovered
    MasterKey prevKey = secretMgr.generateKey();
    stateStore.storeNMTokenPreviousMasterKey(prevKey);
    restartStateStore();
    state = stateStore.loadNMTokensState();
    assertEquals(currentKey, state.getCurrentMasterKey());
    assertEquals(prevKey, state.getPreviousMasterKey());
    assertTrue(state.getApplicationMasterKeys().isEmpty());
    // store a few application keys and verify recovered
    ApplicationAttemptId attempt1 = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1);
    MasterKey attemptKey1 = secretMgr.generateKey();
    stateStore.storeNMTokenApplicationMasterKey(attempt1, attemptKey1);
    ApplicationAttemptId attempt2 = ApplicationAttemptId.newInstance(ApplicationId.newInstance(2, 3), 4);
    MasterKey attemptKey2 = secretMgr.generateKey();
    stateStore.storeNMTokenApplicationMasterKey(attempt2, attemptKey2);
    restartStateStore();
    state = stateStore.loadNMTokensState();
    assertEquals(currentKey, state.getCurrentMasterKey());
    assertEquals(prevKey, state.getPreviousMasterKey());
    Map<ApplicationAttemptId, MasterKey> loadedAppKeys = state.getApplicationMasterKeys();
    assertEquals(2, loadedAppKeys.size());
    assertEquals(attemptKey1, loadedAppKeys.get(attempt1));
    assertEquals(attemptKey2, loadedAppKeys.get(attempt2));
    // add/update/remove keys and verify recovered
    ApplicationAttemptId attempt3 = ApplicationAttemptId.newInstance(ApplicationId.newInstance(5, 6), 7);
    MasterKey attemptKey3 = secretMgr.generateKey();
    stateStore.storeNMTokenApplicationMasterKey(attempt3, attemptKey3);
    stateStore.removeNMTokenApplicationMasterKey(attempt1);
    attemptKey2 = prevKey;
    stateStore.storeNMTokenApplicationMasterKey(attempt2, attemptKey2);
    prevKey = currentKey;
    stateStore.storeNMTokenPreviousMasterKey(prevKey);
    currentKey = secretMgr.generateKey();
    stateStore.storeNMTokenCurrentMasterKey(currentKey);
    restartStateStore();
    state = stateStore.loadNMTokensState();
    assertEquals(currentKey, state.getCurrentMasterKey());
    assertEquals(prevKey, state.getPreviousMasterKey());
    loadedAppKeys = state.getApplicationMasterKeys();
    assertEquals(2, loadedAppKeys.size());
    assertNull(loadedAppKeys.get(attempt1));
    assertEquals(attemptKey2, loadedAppKeys.get(attempt2));
    assertEquals(attemptKey3, loadedAppKeys.get(attempt3));
}

Example 5 with MasterKey

use of org.apache.hadoop.yarn.server.api.records.MasterKey in project hadoop by apache.

the class TestDistributedScheduler method setup.

private RequestInterceptor setup(Configuration conf, DistributedScheduler distributedScheduler) {
    NodeStatusUpdater nodeStatusUpdater = Mockito.mock(NodeStatusUpdater.class);
    Mockito.when(nodeStatusUpdater.getRMIdentifier()).thenReturn(12345l);
    NMContainerTokenSecretManager nmContainerTokenSecretManager = new NMContainerTokenSecretManager(conf);
    MasterKey mKey = new MasterKey() {

        @Override
        public int getKeyId() {
            return 1;
        }

        @Override
        public void setKeyId(int keyId) {
        }

        @Override
        public ByteBuffer getBytes() {
            return ByteBuffer.allocate(8);
        }

        @Override
        public void setBytes(ByteBuffer bytes) {
        }
    };
    nmContainerTokenSecretManager.setMasterKey(mKey);
    OpportunisticContainerAllocator containerAllocator = new OpportunisticContainerAllocator(nmContainerTokenSecretManager);
    NMTokenSecretManagerInNM nmTokenSecretManagerInNM = new NMTokenSecretManagerInNM();
    nmTokenSecretManagerInNM.setMasterKey(mKey);
    distributedScheduler.initLocal(1234, ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), containerAllocator, nmTokenSecretManagerInNM, "test");
    RequestInterceptor finalReqIntcptr = Mockito.mock(RequestInterceptor.class);
    distributedScheduler.setNextInterceptor(finalReqIntcptr);
    return finalReqIntcptr;
}