Examples with NameNodeMetrics org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics used on opensource projects

Example 1 with NameNodeMetrics

use of org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics in project hadoop by apache.

the class TestEditLog method testAutoSync.

/**
   * Regression test for HDFS-1112/HDFS-3020. Ensures that, even if
   * logSync isn't called periodically, the edit log will sync itself.
   */
@Test
public void testAutoSync() throws Exception {
    File logDir = new File(TEST_DIR, "testAutoSync");
    logDir.mkdirs();
    FSEditLog log = FSImageTestUtil.createStandaloneEditLog(logDir);
    String oneKB = StringUtils.byteToHexString(new byte[500]);
    try {
        log.openForWrite(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        NameNodeMetrics mockMetrics = Mockito.mock(NameNodeMetrics.class);
        log.setMetricsForTests(mockMetrics);
        for (int i = 0; i < 400; i++) {
            log.logDelete(oneKB, 1L, false);
        }
        // After ~400KB, we're still within the 512KB buffer size
        Mockito.verify(mockMetrics, Mockito.times(0)).addSync(Mockito.anyLong());
        // After ~400KB more, we should have done an automatic sync
        for (int i = 0; i < 400; i++) {
            log.logDelete(oneKB, 1L, false);
        }
        Mockito.verify(mockMetrics, Mockito.times(1)).addSync(Mockito.anyLong());
    } finally {
        log.close();
    }
}

Example 2 with NameNodeMetrics

use of org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics in project hadoop by apache.

the class ImageServlet method doPut.

@Override
protected void doPut(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
    try {
        ServletContext context = getServletContext();
        final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context);
        final Configuration conf = (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
        final PutImageParams parsedParams = new PutImageParams(request, response, conf);
        final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
        validateRequest(context, conf, request, response, nnImage, parsedParams.getStorageInfoString());
        UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction<Void>() {

            @Override
            public Void run() throws Exception {
                // if its not the active NN, then we need to notify the caller it was was the wrong
                // target (regardless of the fact that we got the image)
                HAServiceProtocol.HAServiceState state = NameNodeHttpServer.getNameNodeStateFromContext(getServletContext());
                if (state != HAServiceProtocol.HAServiceState.ACTIVE) {
                    // we need a different response type here so the client can differentiate this
                    // from the failure to upload due to (1) security, or (2) other checkpoints already
                    // present
                    response.sendError(HttpServletResponse.SC_EXPECTATION_FAILED, "Nameode " + request.getLocalAddr() + " is currently not in a state which can " + "accept uploads of new fsimages. State: " + state);
                    return null;
                }
                final long txid = parsedParams.getTxId();
                String remoteAddr = request.getRemoteAddr();
                ImageUploadRequest imageRequest = new ImageUploadRequest(txid, remoteAddr);
                final NameNodeFile nnf = parsedParams.getNameNodeFile();
                // if the node is attempting to upload an older transaction, we ignore it
                SortedSet<ImageUploadRequest> larger = currentlyDownloadingCheckpoints.tailSet(imageRequest);
                if (larger.size() > 0) {
                    response.sendError(HttpServletResponse.SC_CONFLICT, "Another checkpointer is already in the process of uploading a" + " checkpoint made up to transaction ID " + larger.last());
                    return null;
                }
                //make sure no one else has started uploading one
                if (!currentlyDownloadingCheckpoints.add(imageRequest)) {
                    response.sendError(HttpServletResponse.SC_CONFLICT, "Either current namenode is checkpointing or another" + " checkpointer is already in the process of " + "uploading a checkpoint made at transaction ID " + txid);
                    return null;
                }
                try {
                    if (nnImage.getStorage().findImageFile(nnf, txid) != null) {
                        response.sendError(HttpServletResponse.SC_CONFLICT, "Either current namenode has checkpointed or " + "another checkpointer already uploaded an " + "checkpoint for txid " + txid);
                        return null;
                    }
                    InputStream stream = request.getInputStream();
                    try {
                        long start = monotonicNow();
                        MD5Hash downloadImageDigest = TransferFsImage.handleUploadImageRequest(request, txid, nnImage.getStorage(), stream, parsedParams.getFileSize(), getThrottler(conf));
                        nnImage.saveDigestAndRenameCheckpointImage(nnf, txid, downloadImageDigest);
                        // Metrics non-null only when used inside name node
                        if (metrics != null) {
                            long elapsed = monotonicNow() - start;
                            metrics.addPutImage(elapsed);
                        }
                        // Now that we have a new checkpoint, we might be able to
                        // remove some old ones.
                        nnImage.purgeOldStorage(nnf);
                    } finally {
                        // remove the request once we've processed it, or it threw an error, so we
                        // aren't using it either
                        currentlyDownloadingCheckpoints.remove(imageRequest);
                        stream.close();
                    }
                } finally {
                    nnImage.removeFromCheckpointing(txid);
                }
                return null;
            }
        });
    } catch (Throwable t) {
        String errMsg = "PutImage failed. " + StringUtils.stringifyException(t);
        response.sendError(HttpServletResponse.SC_GONE, errMsg);
        throw new IOException(errMsg);
    }
}

Example 3 with NameNodeMetrics

use of org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics in project hadoop by apache.

the class BlockManager method processReport.

/**
   * The given storage is reporting all its blocks.
   * Update the (storage-->block list) and (block-->storage list) maps.
   *
   * @return true if all known storages of the given DN have finished reporting.
   * @throws IOException
   */
public boolean processReport(final DatanodeID nodeID, final DatanodeStorage storage, final BlockListAsLongs newReport, BlockReportContext context) throws IOException {
    namesystem.writeLock();
    //after acquiring write lock
    final long startTime = Time.monotonicNow();
    final long endTime;
    DatanodeDescriptor node;
    Collection<Block> invalidatedBlocks = Collections.emptyList();
    String strBlockReportId = context != null ? Long.toHexString(context.getReportId()) : "";
    try {
        node = datanodeManager.getDatanode(nodeID);
        if (node == null || !node.isRegistered()) {
            throw new IOException("ProcessReport from dead or unregistered node: " + nodeID);
        }
        // To minimize startup time, we discard any second (or later) block reports
        // that we receive while still in startup phase.
        DatanodeStorageInfo storageInfo = node.getStorageInfo(storage.getStorageID());
        if (storageInfo == null) {
            // We handle this for backwards compatibility.
            storageInfo = node.updateStorage(storage);
        }
        if (namesystem.isInStartupSafeMode() && storageInfo.getBlockReportCount() > 0) {
            blockLog.info("BLOCK* processReport 0x{}: " + "discarded non-initial block report from {}" + " because namenode still in startup phase", strBlockReportId, nodeID);
            blockReportLeaseManager.removeLease(node);
            return !node.hasStaleStorages();
        }
        if (context != null) {
            if (!blockReportLeaseManager.checkLease(node, startTime, context.getLeaseId())) {
                return false;
            }
        }
        if (storageInfo.getBlockReportCount() == 0) {
            // The first block report can be processed a lot more efficiently than
            // ordinary block reports.  This shortens restart times.
            blockLog.info("BLOCK* processReport 0x{}: Processing first " + "storage report for {} from datanode {}", strBlockReportId, storageInfo.getStorageID(), nodeID.getDatanodeUuid());
            processFirstBlockReport(storageInfo, newReport);
        } else {
            invalidatedBlocks = processReport(storageInfo, newReport, context);
        }
        storageInfo.receivedBlockReport();
    } finally {
        endTime = Time.monotonicNow();
        namesystem.writeUnlock();
    }
    for (Block b : invalidatedBlocks) {
        blockLog.debug("BLOCK* processReport 0x{}: {} on node {} size {} does not" + " belong to any file", strBlockReportId, b, node, b.getNumBytes());
    }
    // Log the block report processing stats from Namenode perspective
    final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
    if (metrics != null) {
        metrics.addBlockReport((int) (endTime - startTime));
    }
    blockLog.info("BLOCK* processReport 0x{}: from storage {} node {}, " + "blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " + "invalidatedBlocks: {}", strBlockReportId, storage.getStorageID(), nodeID, newReport.getNumberOfBlocks(), node.hasStaleStorages(), (endTime - startTime), invalidatedBlocks.size());
    return !node.hasStaleStorages();
}

Example 4 with NameNodeMetrics

use of org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics in project hadoop by apache.

the class CacheManager method processCacheReport.

public final void processCacheReport(final DatanodeID datanodeID, final List<Long> blockIds) throws IOException {
    namesystem.writeLock();
    final long startTime = Time.monotonicNow();
    final long endTime;
    try {
        final DatanodeDescriptor datanode = blockManager.getDatanodeManager().getDatanode(datanodeID);
        if (datanode == null || !datanode.isRegistered()) {
            throw new IOException("processCacheReport from dead or unregistered datanode: " + datanode);
        }
        processCacheReportImpl(datanode, blockIds);
    } finally {
        endTime = Time.monotonicNow();
        namesystem.writeUnlock("processCacheReport");
    }
    // Log the block report processing stats from Namenode perspective
    final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
    if (metrics != null) {
        metrics.addCacheBlockReport((int) (endTime - startTime));
    }
    LOG.debug("Processed cache report from {}, blocks: {}, " + "processing time: {} msecs", datanodeID, blockIds.size(), (endTime - startTime));
}

Example 5 with NameNodeMetrics

use of org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics in project hadoop by apache.

the class FSNamesystem method loadFromDisk.

/**
   * Instantiates an FSNamesystem loaded from the image and edits
   * directories specified in the passed Configuration.
   *
   * @param conf the Configuration which specifies the storage directories
   *             from which to load
   * @return an FSNamesystem which contains the loaded namespace
   * @throws IOException if loading fails
   */
static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
    checkConfiguration(conf);
    FSImage fsImage = new FSImage(conf, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf));
    FSNamesystem namesystem = new FSNamesystem(conf, fsImage, false);
    StartupOption startOpt = NameNode.getStartupOption(conf);
    if (startOpt == StartupOption.RECOVER) {
        namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    }
    long loadStart = monotonicNow();
    try {
        namesystem.loadFSImage(startOpt);
    } catch (IOException ioe) {
        LOG.warn("Encountered exception loading fsimage", ioe);
        fsImage.close();
        throw ioe;
    }
    long timeTakenToLoadFSImage = monotonicNow() - loadStart;
    LOG.info("Finished loading FSImage in " + timeTakenToLoadFSImage + " msecs");
    NameNodeMetrics nnMetrics = NameNode.getNameNodeMetrics();
    if (nnMetrics != null) {
        nnMetrics.setFsImageLoadTime((int) timeTakenToLoadFSImage);
    }
    namesystem.getFSDirectory().createReservedStatuses(namesystem.getCTime());
    return namesystem;
}