Examples with MasterKey org.apache.hadoop.yarn.server.api.records.MasterKey used on opensource projects

Example 11 with MasterKey

use of org.apache.hadoop.yarn.server.api.records.MasterKey in project hadoop by apache.

the class AMRMTokenSecretManager method recover.

public void recover(RMState state) {
    if (state.getAMRMTokenSecretManagerState() != null) {
        // recover the current master key
        MasterKey currentKey = state.getAMRMTokenSecretManagerState().getCurrentMasterKey();
        this.currentMasterKey = new MasterKeyData(currentKey, createSecretKey(currentKey.getBytes().array()));
        // recover the next master key if not null
        MasterKey nextKey = state.getAMRMTokenSecretManagerState().getNextMasterKey();
        if (nextKey != null) {
            this.nextMasterKey = new MasterKeyData(nextKey, createSecretKey(nextKey.getBytes().array()));
            this.timer.schedule(new NextKeyActivator(), this.activationDelay);
        }
    }
}

Example 12 with MasterKey

use of org.apache.hadoop.yarn.server.api.records.MasterKey in project hadoop by apache.

the class TestRMNMSecretKeys method validateRMNMKeyExchange.

private void validateRMNMKeyExchange(YarnConfiguration conf) throws Exception {
    // Default rolling and activation intervals are large enough, no need to
    // intervene
    final DrainDispatcher dispatcher = new DrainDispatcher();
    ResourceManager rm = new ResourceManager() {

        @Override
        protected void doSecureLogin() throws IOException {
        // Do nothing.
        }

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }

        @Override
        protected void startWepApp() {
        // Don't need it, skip.
        }
    };
    rm.init(conf);
    rm.start();
    // Testing ContainerToken and NMToken
    String containerToken = "Container Token : ";
    String nmToken = "NM Token : ";
    MockNM nm = new MockNM("host:1234", 3072, rm.getResourceTrackerService());
    RegisterNodeManagerResponse registrationResponse = nm.registerNode();
    MasterKey containerTokenMasterKey = registrationResponse.getContainerTokenMasterKey();
    Assert.assertNotNull(containerToken + "Registration should cause a key-update!", containerTokenMasterKey);
    MasterKey nmTokenMasterKey = registrationResponse.getNMTokenMasterKey();
    Assert.assertNotNull(nmToken + "Registration should cause a key-update!", nmTokenMasterKey);
    dispatcher.await();
    NodeHeartbeatResponse response = nm.nodeHeartbeat(true);
    Assert.assertNull(containerToken + "First heartbeat after registration shouldn't get any key updates!", response.getContainerTokenMasterKey());
    Assert.assertNull(nmToken + "First heartbeat after registration shouldn't get any key updates!", response.getNMTokenMasterKey());
    dispatcher.await();
    response = nm.nodeHeartbeat(true);
    Assert.assertNull(containerToken + "Even second heartbeat after registration shouldn't get any key updates!", response.getContainerTokenMasterKey());
    Assert.assertNull(nmToken + "Even second heartbeat after registration shouldn't get any key updates!", response.getContainerTokenMasterKey());
    dispatcher.await();
    // Let's force a roll-over
    rm.getRMContext().getContainerTokenSecretManager().rollMasterKey();
    rm.getRMContext().getNMTokenSecretManager().rollMasterKey();
    // Heartbeats after roll-over and before activation should be fine.
    response = nm.nodeHeartbeat(true);
    Assert.assertNotNull(containerToken + "Heartbeats after roll-over and before activation should not err out.", response.getContainerTokenMasterKey());
    Assert.assertNotNull(nmToken + "Heartbeats after roll-over and before activation should not err out.", response.getNMTokenMasterKey());
    Assert.assertEquals(containerToken + "Roll-over should have incremented the key-id only by one!", containerTokenMasterKey.getKeyId() + 1, response.getContainerTokenMasterKey().getKeyId());
    Assert.assertEquals(nmToken + "Roll-over should have incremented the key-id only by one!", nmTokenMasterKey.getKeyId() + 1, response.getNMTokenMasterKey().getKeyId());
    dispatcher.await();
    response = nm.nodeHeartbeat(true);
    Assert.assertNull(containerToken + "Second heartbeat after roll-over shouldn't get any key updates!", response.getContainerTokenMasterKey());
    Assert.assertNull(nmToken + "Second heartbeat after roll-over shouldn't get any key updates!", response.getNMTokenMasterKey());
    dispatcher.await();
    // Let's force activation
    rm.getRMContext().getContainerTokenSecretManager().activateNextMasterKey();
    rm.getRMContext().getNMTokenSecretManager().activateNextMasterKey();
    response = nm.nodeHeartbeat(true);
    Assert.assertNull(containerToken + "Activation shouldn't cause any key updates!", response.getContainerTokenMasterKey());
    Assert.assertNull(nmToken + "Activation shouldn't cause any key updates!", response.getNMTokenMasterKey());
    dispatcher.await();
    response = nm.nodeHeartbeat(true);
    Assert.assertNull(containerToken + "Even second heartbeat after activation shouldn't get any key updates!", response.getContainerTokenMasterKey());
    Assert.assertNull(nmToken + "Even second heartbeat after activation shouldn't get any key updates!", response.getNMTokenMasterKey());
    dispatcher.await();
    rm.stop();
}

Example 13 with MasterKey

use of org.apache.hadoop.yarn.server.api.records.MasterKey in project hadoop by apache.

the class TestYarnServerApiClasses method getMasterKey.

private MasterKey getMasterKey() {
    MasterKey key = recordFactory.newRecordInstance(MasterKey.class);
    key.setBytes(ByteBuffer.allocate(0));
    key.setKeyId(1);
    return key;
}

Example 14 with MasterKey

use of org.apache.hadoop.yarn.server.api.records.MasterKey in project hadoop by apache.

the class NodeStatusUpdaterImpl method registerWithRM.

@VisibleForTesting
protected void registerWithRM() throws YarnException, IOException {
    RegisterNodeManagerResponse regNMResponse;
    Set<NodeLabel> nodeLabels = nodeLabelsHandler.getNodeLabelsForRegistration();
    // during RM recovery
    synchronized (this.context) {
        List<NMContainerStatus> containerReports = getNMContainerStatuses();
        RegisterNodeManagerRequest request = RegisterNodeManagerRequest.newInstance(nodeId, httpPort, totalResource, nodeManagerVersionId, containerReports, getRunningApplications(), nodeLabels, physicalResource);
        if (containerReports != null) {
            LOG.info("Registering with RM using containers :" + containerReports);
        }
        regNMResponse = resourceTracker.registerNodeManager(request);
        // Make sure rmIdentifier is set before we release the lock
        this.rmIdentifier = regNMResponse.getRMIdentifier();
    }
    // if the Resource Manager instructs NM to shutdown.
    if (NodeAction.SHUTDOWN.equals(regNMResponse.getNodeAction())) {
        String message = "Message from ResourceManager: " + regNMResponse.getDiagnosticsMessage();
        throw new YarnRuntimeException("Recieved SHUTDOWN signal from Resourcemanager, Registration of NodeManager failed, " + message);
    }
    // if ResourceManager version is too old then shutdown
    if (!minimumResourceManagerVersion.equals("NONE")) {
        if (minimumResourceManagerVersion.equals("EqualToNM")) {
            minimumResourceManagerVersion = nodeManagerVersionId;
        }
        String rmVersion = regNMResponse.getRMVersion();
        if (rmVersion == null) {
            String message = "The Resource Manager's did not return a version. " + "Valid version cannot be checked.";
            throw new YarnRuntimeException("Shutting down the Node Manager. " + message);
        }
        if (VersionUtil.compareVersions(rmVersion, minimumResourceManagerVersion) < 0) {
            String message = "The Resource Manager's version (" + rmVersion + ") is less than the minimum " + "allowed version " + minimumResourceManagerVersion;
            throw new YarnRuntimeException("Shutting down the Node Manager on RM " + "version error, " + message);
        }
    }
    this.registeredWithRM = true;
    MasterKey masterKey = regNMResponse.getContainerTokenMasterKey();
    // StatusUpdater#start().
    if (masterKey != null) {
        this.context.getContainerTokenSecretManager().setMasterKey(masterKey);
    }
    masterKey = regNMResponse.getNMTokenMasterKey();
    if (masterKey != null) {
        this.context.getNMTokenSecretManager().setMasterKey(masterKey);
    }
    StringBuilder successfullRegistrationMsg = new StringBuilder();
    successfullRegistrationMsg.append("Registered with ResourceManager as ").append(this.nodeId);
    Resource newResource = regNMResponse.getResource();
    if (newResource != null) {
        updateNMResource(newResource);
        successfullRegistrationMsg.append(" with updated total resource of ").append(this.totalResource);
    } else {
        successfullRegistrationMsg.append(" with total resource of ").append(this.totalResource);
    }
    successfullRegistrationMsg.append(nodeLabelsHandler.verifyRMRegistrationResponseForNodeLabels(regNMResponse));
    LOG.info(successfullRegistrationMsg);
    LOG.info("Notifying ContainerManager to unblock new container-requests");
    this.context.getContainerManager().setBlockNewContainerRequests(false);
}

Example 15 with MasterKey

use of org.apache.hadoop.yarn.server.api.records.MasterKey in project hadoop by apache.

the class NMContainerTokenSecretManager method recover.

public synchronized void recover() throws IOException {
    RecoveredContainerTokensState state = stateStore.loadContainerTokensState();
    MasterKey key = state.getCurrentMasterKey();
    if (key != null) {
        super.currentMasterKey = new MasterKeyData(key, createSecretKey(key.getBytes().array()));
    }
    key = state.getPreviousMasterKey();
    if (key != null) {
        previousMasterKey = new MasterKeyData(key, createSecretKey(key.getBytes().array()));
    }
    // restore the serial number from the current master key
    if (super.currentMasterKey != null) {
        super.serialNo = super.currentMasterKey.getMasterKey().getKeyId() + 1;
    }
    for (Entry<ContainerId, Long> entry : state.getActiveTokens().entrySet()) {
        ContainerId containerId = entry.getKey();
        Long expTime = entry.getValue();
        List<ContainerId> containerList = recentlyStartedContainerTracker.get(expTime);
        if (containerList == null) {
            containerList = new ArrayList<ContainerId>();
            recentlyStartedContainerTracker.put(expTime, containerList);
        }
        if (!containerList.contains(containerId)) {
            containerList.add(containerId);
        }
    }
}