Examples with CheckpointCommand org.apache.hadoop.hdfs.server.protocol.CheckpointCommand used on opensource projects

Example 1 with CheckpointCommand

use of org.apache.hadoop.hdfs.server.protocol.CheckpointCommand in project hadoop by apache.

the class Checkpointer method doCheckpoint.

/**
   * Create a new checkpoint
   */
void doCheckpoint() throws IOException {
    BackupImage bnImage = getFSImage();
    NNStorage bnStorage = bnImage.getStorage();
    long startTime = monotonicNow();
    bnImage.freezeNamespaceAtNextRoll();
    NamenodeCommand cmd = getRemoteNamenodeProxy().startCheckpoint(backupNode.getRegistration());
    CheckpointCommand cpCmd = null;
    switch(cmd.getAction()) {
        case NamenodeProtocol.ACT_SHUTDOWN:
            shutdown();
            throw new IOException("Name-node " + backupNode.nnRpcAddress + " requested shutdown.");
        case NamenodeProtocol.ACT_CHECKPOINT:
            cpCmd = (CheckpointCommand) cmd;
            break;
        default:
            throw new IOException("Unsupported NamenodeCommand: " + cmd.getAction());
    }
    bnImage.waitUntilNamespaceFrozen();
    CheckpointSignature sig = cpCmd.getSignature();
    // Make sure we're talking to the same NN!
    sig.validateStorageInfo(bnImage);
    long lastApplied = bnImage.getLastAppliedTxId();
    LOG.debug("Doing checkpoint. Last applied: " + lastApplied);
    RemoteEditLogManifest manifest = getRemoteNamenodeProxy().getEditLogManifest(bnImage.getLastAppliedTxId() + 1);
    boolean needReloadImage = false;
    if (!manifest.getLogs().isEmpty()) {
        RemoteEditLog firstRemoteLog = manifest.getLogs().get(0);
        // to download and load the image.
        if (firstRemoteLog.getStartTxId() > lastApplied + 1) {
            LOG.info("Unable to roll forward using only logs. Downloading " + "image with txid " + sig.mostRecentCheckpointTxId);
            MD5Hash downloadedHash = TransferFsImage.downloadImageToStorage(backupNode.nnHttpAddress, sig.mostRecentCheckpointTxId, bnStorage, true, false);
            bnImage.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, sig.mostRecentCheckpointTxId, downloadedHash);
            lastApplied = sig.mostRecentCheckpointTxId;
            needReloadImage = true;
        }
        if (firstRemoteLog.getStartTxId() > lastApplied + 1) {
            throw new IOException("No logs to roll forward from " + lastApplied);
        }
        // get edits files
        for (RemoteEditLog log : manifest.getLogs()) {
            TransferFsImage.downloadEditsToStorage(backupNode.nnHttpAddress, log, bnStorage);
        }
        if (needReloadImage) {
            LOG.info("Loading image with txid " + sig.mostRecentCheckpointTxId);
            File file = bnStorage.findImageFile(NameNodeFile.IMAGE, sig.mostRecentCheckpointTxId);
            bnImage.reloadFromImageFile(file, backupNode.getNamesystem());
        }
        rollForwardByApplyingLogs(manifest, bnImage, backupNode.getNamesystem());
    }
    long txid = bnImage.getLastAppliedTxId();
    backupNode.namesystem.writeLock();
    try {
        backupNode.namesystem.setImageLoaded();
        if (backupNode.namesystem.getBlocksTotal() > 0) {
            long completeBlocksTotal = backupNode.namesystem.getCompleteBlocksTotal();
            backupNode.namesystem.getBlockManager().setBlockTotal(completeBlocksTotal);
        }
        bnImage.saveFSImageInAllDirs(backupNode.getNamesystem(), txid);
        if (!backupNode.namenode.isRollingUpgrade()) {
            bnImage.updateStorageVersion();
        }
    } finally {
        backupNode.namesystem.writeUnlock("doCheckpoint");
    }
    if (cpCmd.needToReturnImage()) {
        TransferFsImage.uploadImageFromStorage(backupNode.nnHttpAddress, conf, bnStorage, NameNodeFile.IMAGE, txid);
    }
    getRemoteNamenodeProxy().endCheckpoint(backupNode.getRegistration(), sig);
    if (backupNode.getRole() == NamenodeRole.BACKUP) {
        bnImage.convergeJournalSpool();
    }
    // keep registration up to date
    backupNode.setRegistration();
    long imageSize = bnImage.getStorage().getFsImageName(txid).length();
    LOG.info("Checkpoint completed in " + (monotonicNow() - startTime) / 1000 + " seconds." + " New Image Size: " + imageSize);
}