Examples with StorageReport org.apache.hadoop.hdfs.server.protocol.StorageReport used on opensource projects

Example 6 with StorageReport

use of org.apache.hadoop.hdfs.server.protocol.StorageReport in project hadoop by apache.

the class TestDeadDatanode method testDeadDatanode.

/**
   * Test to ensure namenode rejects request from dead datanode
   * - Start a cluster
   * - Shutdown the datanode and wait for it to be marked dead at the namenode
   * - Send datanode requests to Namenode and make sure it is rejected 
   *   appropriately.
   */
@Test
public void testDeadDatanode() throws Exception {
    Configuration conf = new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
    cluster = new MiniDFSCluster.Builder(conf).build();
    cluster.waitActive();
    String poolId = cluster.getNamesystem().getBlockPoolId();
    // wait for datanode to be marked live
    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeRegistration reg = InternalDataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId);
    DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), true, 20000);
    // Shutdown and wait for datanode to be marked dead
    dn.shutdown();
    DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), false, 20000);
    DatanodeProtocol dnp = cluster.getNameNodeRpc();
    ReceivedDeletedBlockInfo[] blocks = { new ReceivedDeletedBlockInfo(new Block(0), ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) };
    StorageReceivedDeletedBlocks[] storageBlocks = { new StorageReceivedDeletedBlocks(new DatanodeStorage(reg.getDatanodeUuid()), blocks) };
    // Ensure blockReceived call from dead datanode is not rejected with
    // IOException, since it's async, but the node remains unregistered.
    dnp.blockReceivedAndDeleted(reg, poolId, storageBlocks);
    BlockManager bm = cluster.getNamesystem().getBlockManager();
    // IBRs are async, make sure the NN processes all of them.
    bm.flushBlockOps();
    assertFalse(bm.getDatanodeManager().getDatanode(reg).isRegistered());
    // Ensure blockReport from dead datanode is rejected with IOException
    StorageBlockReport[] report = { new StorageBlockReport(new DatanodeStorage(reg.getDatanodeUuid()), BlockListAsLongs.EMPTY) };
    try {
        dnp.blockReport(reg, poolId, report, new BlockReportContext(1, 0, System.nanoTime(), 0L, true));
        fail("Expected IOException is not thrown");
    } catch (IOException ex) {
    // Expected
    }
    // Ensure heartbeat from dead datanode is rejected with a command
    // that asks datanode to register again
    StorageReport[] rep = { new StorageReport(new DatanodeStorage(reg.getDatanodeUuid()), false, 0, 0, 0, 0, 0) };
    DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null, true, SlowPeerReports.EMPTY_REPORT).getCommands();
    assertEquals(1, cmd.length);
    assertEquals(cmd[0].getAction(), RegisterCommand.REGISTER.getAction());
}

Example 7 with StorageReport

use of org.apache.hadoop.hdfs.server.protocol.StorageReport in project hadoop by apache.

the class DatanodeProtocolServerSideTranslatorPB method sendHeartbeat.

@Override
public HeartbeatResponseProto sendHeartbeat(RpcController controller, HeartbeatRequestProto request) throws ServiceException {
    HeartbeatResponse response;
    try {
        final StorageReport[] report = PBHelperClient.convertStorageReports(request.getReportsList());
        VolumeFailureSummary volumeFailureSummary = request.hasVolumeFailureSummary() ? PBHelper.convertVolumeFailureSummary(request.getVolumeFailureSummary()) : null;
        response = impl.sendHeartbeat(PBHelper.convert(request.getRegistration()), report, request.getCacheCapacity(), request.getCacheUsed(), request.getXmitsInProgress(), request.getXceiverCount(), request.getFailedVolumes(), volumeFailureSummary, request.getRequestFullBlockReportLease(), PBHelper.convertSlowPeerInfo(request.getSlowPeersList()));
    } catch (IOException e) {
        throw new ServiceException(e);
    }
    HeartbeatResponseProto.Builder builder = HeartbeatResponseProto.newBuilder();
    DatanodeCommand[] cmds = response.getCommands();
    if (cmds != null) {
        for (int i = 0; i < cmds.length; i++) {
            if (cmds[i] != null) {
                builder.addCmds(PBHelper.convert(cmds[i]));
            }
        }
    }
    builder.setHaStatus(PBHelper.convert(response.getNameNodeHaState()));
    RollingUpgradeStatus rollingUpdateStatus = response.getRollingUpdateStatus();
    if (rollingUpdateStatus != null) {
        // V2 is always set for newer datanodes.
        // To be compatible with older datanodes, V1 is set to null
        //  if the RU was finalized.
        RollingUpgradeStatusProto rus = PBHelperClient.convertRollingUpgradeStatus(rollingUpdateStatus);
        builder.setRollingUpgradeStatusV2(rus);
        if (!rollingUpdateStatus.isFinalized()) {
            builder.setRollingUpgradeStatus(rus);
        }
    }
    builder.setFullBlockReportLeaseId(response.getFullBlockReportLeaseId());
    return builder.build();
}

Example 8 with StorageReport

use of org.apache.hadoop.hdfs.server.protocol.StorageReport in project hadoop by apache.

the class TestDatanodeReport method assertReports.

static void assertReports(int numDatanodes, DatanodeReportType type, DFSClient client, List<DataNode> datanodes, String bpid) throws IOException {
    final DatanodeInfo[] infos = client.datanodeReport(type);
    assertEquals(numDatanodes, infos.length);
    final DatanodeStorageReport[] reports = client.getDatanodeStorageReport(type);
    assertEquals(numDatanodes, reports.length);
    for (int i = 0; i < infos.length; i++) {
        assertEquals(infos[i], reports[i].getDatanodeInfo());
        final DataNode d = findDatanode(infos[i].getDatanodeUuid(), datanodes);
        if (bpid != null) {
            //check storage
            final StorageReport[] computed = reports[i].getStorageReports();
            Arrays.sort(computed, CMP);
            final StorageReport[] expected = d.getFSDataset().getStorageReports(bpid);
            Arrays.sort(expected, CMP);
            assertEquals(expected.length, computed.length);
            for (int j = 0; j < expected.length; j++) {
                assertEquals(expected[j].getStorage().getStorageID(), computed[j].getStorage().getStorageID());
            }
        }
    }
}

Example 9 with StorageReport

use of org.apache.hadoop.hdfs.server.protocol.StorageReport in project hadoop by apache.

the class DatanodeDescriptor method pruneStorageMap.

/**
   * Remove stale storages from storageMap. We must not remove any storages
   * as long as they have associated block replicas.
   */
private void pruneStorageMap(final StorageReport[] reports) {
    synchronized (storageMap) {
        LOG.debug("Number of storages reported in heartbeat={};" + " Number of storages in storageMap={}", reports.length, storageMap.size());
        HashMap<String, DatanodeStorageInfo> excessStorages;
        // Init excessStorages with all known storages.
        excessStorages = new HashMap<>(storageMap);
        // Remove storages that the DN reported in the heartbeat.
        for (final StorageReport report : reports) {
            excessStorages.remove(report.getStorage().getStorageID());
        }
        // blocks.
        for (final DatanodeStorageInfo storageInfo : excessStorages.values()) {
            if (storageInfo.numBlocks() == 0) {
                storageMap.remove(storageInfo.getStorageID());
                LOG.info("Removed storage {} from DataNode {}", storageInfo, this);
            } else {
                // This can occur until all block reports are received.
                LOG.debug("Deferring removal of stale storage {} with {} blocks", storageInfo, storageInfo.numBlocks());
            }
        }
    }
}

Example 10 with StorageReport

use of org.apache.hadoop.hdfs.server.protocol.StorageReport in project hadoop by apache.

the class BPServiceActor method sendHeartBeat.

HeartbeatResponse sendHeartBeat(boolean requestBlockReportLease) throws IOException {
    scheduler.scheduleNextHeartbeat();
    StorageReport[] reports = dn.getFSDataset().getStorageReports(bpos.getBlockPoolId());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Sending heartbeat with " + reports.length + " storage reports from service actor: " + this);
    }
    final long now = monotonicNow();
    scheduler.updateLastHeartbeatTime(now);
    VolumeFailureSummary volumeFailureSummary = dn.getFSDataset().getVolumeFailureSummary();
    int numFailedVolumes = volumeFailureSummary != null ? volumeFailureSummary.getFailedStorageLocations().length : 0;
    final boolean slowPeersReportDue = scheduler.isSlowPeersReportDue(now);
    final SlowPeerReports slowPeers = slowPeersReportDue && dn.getPeerMetrics() != null ? SlowPeerReports.create(dn.getPeerMetrics().getOutliers()) : SlowPeerReports.EMPTY_REPORT;
    HeartbeatResponse response = bpNamenode.sendHeartbeat(bpRegistration, reports, dn.getFSDataset().getCacheCapacity(), dn.getFSDataset().getCacheUsed(), dn.getXmitsInProgress(), dn.getXceiverCount(), numFailedVolumes, volumeFailureSummary, requestBlockReportLease, slowPeers);
    if (slowPeersReportDue) {
        // If the report was due and successfully sent, schedule the next one.
        scheduler.scheduleNextSlowPeerReport();
    }
    return response;
}