Examples with FSImage org.apache.hadoop.hdfs.server.namenode.FSImage used on opensource projects

Example 1 with FSImage

use of org.apache.hadoop.hdfs.server.namenode.FSImage in project hadoop by apache.

the class TestBootstrapStandbyWithQJM method testUpgrade.

private void testUpgrade(UpgradeState state) throws Exception {
    cluster.transitionToActive(0);
    final Configuration confNN1 = cluster.getConfiguration(1);
    final File current = cluster.getNameNode(1).getFSImage().getStorage().getStorageDir(0).getCurrentDir();
    final File tmp = cluster.getNameNode(1).getFSImage().getStorage().getStorageDir(0).getPreviousTmp();
    // shut down nn1
    cluster.shutdownNameNode(1);
    // make NN0 in upgrade state
    FSImage fsImage0 = cluster.getNameNode(0).getNamesystem().getFSImage();
    Whitebox.setInternalState(fsImage0, "isUpgradeFinalized", false);
    switch(state) {
        case RECOVER:
            // rename the current directory to previous.tmp in nn1
            NNStorage.rename(current, tmp);
            break;
        case FORMAT:
            // rename the current directory to a random name so it's not formatted
            final File wrongPath = new File(current.getParentFile(), "wrong");
            NNStorage.rename(current, wrongPath);
            break;
        default:
            break;
    }
    int rc = BootstrapStandby.run(new String[] { "-force" }, confNN1);
    assertEquals(0, rc);
    // Should have copied over the namespace from the standby
    FSImageTestUtil.assertNNHasCheckpoints(cluster, 1, ImmutableList.of(0));
    FSImageTestUtil.assertNNFilesMatch(cluster);
    // make sure the NN1 is in upgrade state, i.e., the previous directory has
    // been successfully created
    cluster.restartNameNode(1);
    assertFalse(cluster.getNameNode(1).getNamesystem().isUpgradeFinalized());
}

Example 2 with FSImage

use of org.apache.hadoop.hdfs.server.namenode.FSImage in project hadoop by apache.

the class BootstrapStandby method downloadImage.

private int downloadImage(NNStorage storage, NamenodeProtocol proxy, RemoteNameNodeInfo proxyInfo) throws IOException {
    // Load the newly formatted image, using all of the directories
    // (including shared edits)
    final long imageTxId = proxy.getMostRecentCheckpointTxId();
    final long curTxId = proxy.getTransactionID();
    FSImage image = new FSImage(conf);
    try {
        image.getStorage().setStorageInfo(storage);
        image.initEditLog(StartupOption.REGULAR);
        assert image.getEditLog().isOpenForRead() : "Expected edit log to be open for read";
        // start up from the last checkpoint on the active.
        if (!skipSharedEditsCheck && !checkLogsAvailableForRead(image, imageTxId, curTxId)) {
            return ERR_CODE_LOGS_UNAVAILABLE;
        }
        // Download that checkpoint into our storage directories.
        MD5Hash hash = TransferFsImage.downloadImageToStorage(proxyInfo.getHttpAddress(), imageTxId, storage, true, true);
        image.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, imageTxId, hash);
        // Write seen_txid to the formatted image directories.
        storage.writeTransactionIdFileToStorage(imageTxId, NameNodeDirType.IMAGE);
    } catch (IOException ioe) {
        throw ioe;
    } finally {
        image.close();
    }
    return 0;
}

Example 3 with FSImage

use of org.apache.hadoop.hdfs.server.namenode.FSImage in project hadoop by apache.

the class StandbyCheckpointer method doCheckpoint.

private void doCheckpoint(boolean sendCheckpoint) throws InterruptedException, IOException {
    assert canceler != null;
    final long txid;
    final NameNodeFile imageType;
    // Acquire cpLock to make sure no one is modifying the name system.
    // It does not need the full namesystem write lock, since the only thing
    // that modifies namesystem on standby node is edit log replaying.
    namesystem.cpLockInterruptibly();
    try {
        assert namesystem.getEditLog().isOpenForRead() : "Standby Checkpointer should only attempt a checkpoint when " + "NN is in standby mode, but the edit logs are in an unexpected state";
        FSImage img = namesystem.getFSImage();
        long prevCheckpointTxId = img.getStorage().getMostRecentCheckpointTxId();
        long thisCheckpointTxId = img.getCorrectLastAppliedOrWrittenTxId();
        assert thisCheckpointTxId >= prevCheckpointTxId;
        if (thisCheckpointTxId == prevCheckpointTxId) {
            LOG.info("A checkpoint was triggered but the Standby Node has not " + "received any transactions since the last checkpoint at txid " + thisCheckpointTxId + ". Skipping...");
            return;
        }
        if (namesystem.isRollingUpgrade() && !namesystem.getFSImage().hasRollbackFSImage()) {
            // if we will do rolling upgrade but have not created the rollback image
            // yet, name this checkpoint as fsimage_rollback
            imageType = NameNodeFile.IMAGE_ROLLBACK;
        } else {
            imageType = NameNodeFile.IMAGE;
        }
        img.saveNamespace(namesystem, imageType, canceler);
        txid = img.getStorage().getMostRecentCheckpointTxId();
        assert txid == thisCheckpointTxId : "expected to save checkpoint at txid=" + thisCheckpointTxId + " but instead saved at txid=" + txid;
        // Save the legacy OIV image, if the output dir is defined.
        String outputDir = checkpointConf.getLegacyOivImageDir();
        if (outputDir != null && !outputDir.isEmpty()) {
            img.saveLegacyOIVImage(namesystem, outputDir, canceler);
        }
    } finally {
        namesystem.cpUnlock();
    }
    //early exit if we shouldn't actually send the checkpoint to the ANN
    if (!sendCheckpoint) {
        return;
    }
    // Upload the saved checkpoint back to the active
    // Do this in a separate thread to avoid blocking transition to active, but don't allow more
    // than the expected number of tasks to run or queue up
    // See HDFS-4816
    ExecutorService executor = new ThreadPoolExecutor(0, activeNNAddresses.size(), 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(activeNNAddresses.size()), uploadThreadFactory);
    // for right now, just match the upload to the nn address by convention. There is no need to
    // directly tie them together by adding a pair class.
    List<Future<TransferFsImage.TransferResult>> uploads = new ArrayList<Future<TransferFsImage.TransferResult>>();
    for (final URL activeNNAddress : activeNNAddresses) {
        Future<TransferFsImage.TransferResult> upload = executor.submit(new Callable<TransferFsImage.TransferResult>() {

            @Override
            public TransferFsImage.TransferResult call() throws IOException {
                return TransferFsImage.uploadImageFromStorage(activeNNAddress, conf, namesystem.getFSImage().getStorage(), imageType, txid, canceler);
            }
        });
        uploads.add(upload);
    }
    InterruptedException ie = null;
    IOException ioe = null;
    int i = 0;
    boolean success = false;
    for (; i < uploads.size(); i++) {
        Future<TransferFsImage.TransferResult> upload = uploads.get(i);
        try {
            // TODO should there be some smarts here about retries nodes that are not the active NN?
            if (upload.get() == TransferFsImage.TransferResult.SUCCESS) {
                success = true;
                //avoid getting the rest of the results - we don't care since we had a successful upload
                break;
            }
        } catch (ExecutionException e) {
            ioe = new IOException("Exception during image upload: " + e.getMessage(), e.getCause());
            break;
        } catch (InterruptedException e) {
            ie = e;
            break;
        }
    }
    lastUploadTime = monotonicNow();
    // we are primary if we successfully updated the ANN
    this.isPrimaryCheckPointer = success;
    // exceptions, so we just handle the ones we expect.
    if (ie != null || ioe != null) {
        // cancel the rest of the tasks, and close the pool
        for (; i < uploads.size(); i++) {
            Future<TransferFsImage.TransferResult> upload = uploads.get(i);
            // The background thread may be blocked waiting in the throttler, so
            // interrupt it.
            upload.cancel(true);
        }
        // shutdown so we interrupt anything running and don't start anything new
        executor.shutdownNow();
        // this is a good bit longer than the thread timeout, just to make sure all the threads
        // that are not doing any work also stop
        executor.awaitTermination(500, TimeUnit.MILLISECONDS);
        // re-throw the exception we got, since one of these two must be non-null
        if (ie != null) {
            throw ie;
        } else if (ioe != null) {
            throw ioe;
        }
    }
}

Example 4 with FSImage

use of org.apache.hadoop.hdfs.server.namenode.FSImage in project hadoop by apache.

the class EditLogTailer method doTailEdits.

@VisibleForTesting
void doTailEdits() throws IOException, InterruptedException {
    // Write lock needs to be interruptible here because the 
    // transitionToActive RPC takes the write lock before calling
    // tailer.stop() -- so if we're not interruptible, it will
    // deadlock.
    namesystem.writeLockInterruptibly();
    try {
        FSImage image = namesystem.getFSImage();
        long lastTxnId = image.getLastAppliedTxId();
        if (LOG.isDebugEnabled()) {
            LOG.debug("lastTxnId: " + lastTxnId);
        }
        Collection<EditLogInputStream> streams;
        try {
            streams = editLog.selectInputStreams(lastTxnId + 1, 0, null, inProgressOk, true);
        } catch (IOException ioe) {
            // This is acceptable. If we try to tail edits in the middle of an edits
            // log roll, i.e. the last one has been finalized but the new inprogress
            // edits file hasn't been started yet.
            LOG.warn("Edits tailer failed to find any streams. Will try again " + "later.", ioe);
            return;
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("edit streams to load from: " + streams.size());
        }
        // Once we have streams to load, errors encountered are legitimate cause
        // for concern, so we don't catch them here. Simple errors reading from
        // disk are ignored.
        long editsLoaded = 0;
        try {
            editsLoaded = image.loadEdits(streams, namesystem);
        } catch (EditLogInputException elie) {
            editsLoaded = elie.getNumEditsLoaded();
            throw elie;
        } finally {
            if (editsLoaded > 0 || LOG.isDebugEnabled()) {
                LOG.debug(String.format("Loaded %d edits starting from txid %d ", editsLoaded, lastTxnId));
            }
        }
        if (editsLoaded > 0) {
            lastLoadTimeMs = monotonicNow();
        }
        lastLoadedTxnId = image.getLastAppliedTxId();
    } finally {
        namesystem.writeUnlock();
    }
}

Example 5 with FSImage

use of org.apache.hadoop.hdfs.server.namenode.FSImage in project hadoop by apache.

the class TestRollingUpgrade method testFinalize.

private void testFinalize(int nnCount) throws Exception {
    final Configuration conf = new HdfsConfiguration();
    MiniQJMHACluster cluster = null;
    final Path foo = new Path("/foo");
    final Path bar = new Path("/bar");
    try {
        cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
        MiniDFSCluster dfsCluster = cluster.getDfsCluster();
        dfsCluster.waitActive();
        // let other NN tail editlog every 1s
        for (int i = 1; i < nnCount; i++) {
            dfsCluster.getConfiguration(i).setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
        }
        dfsCluster.restartNameNodes();
        dfsCluster.transitionToActive(0);
        DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
        dfs.mkdirs(foo);
        FSImage fsimage = dfsCluster.getNamesystem(0).getFSImage();
        // start rolling upgrade
        RollingUpgradeInfo info = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
        Assert.assertTrue(info.isStarted());
        dfs.mkdirs(bar);
        queryForPreparation(dfs);
        // The NN should have a copy of the fsimage in case of rollbacks.
        Assert.assertTrue(fsimage.hasRollbackFSImage());
        info = dfs.rollingUpgrade(RollingUpgradeAction.FINALIZE);
        Assert.assertTrue(info.isFinalized());
        Assert.assertTrue(dfs.exists(foo));
        // Once finalized, there should be no more fsimage for rollbacks.
        Assert.assertFalse(fsimage.hasRollbackFSImage());
        // Should have no problem in restart and replaying edits that include
        // the FINALIZE op.
        dfsCluster.restartNameNode(0);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}