Examples with DatanodeStorage org.apache.hadoop.hdfs.server.protocol.DatanodeStorage used on opensource projects

Example 1 with DatanodeStorage

use of org.apache.hadoop.hdfs.server.protocol.DatanodeStorage in project hadoop by apache.

the class TestReplicationPolicy method testConvertLastBlockToUnderConstructionDoesNotCauseSkippedReplication.

@Test(timeout = 60000)
public void testConvertLastBlockToUnderConstructionDoesNotCauseSkippedReplication() throws IOException {
    Namesystem mockNS = mock(Namesystem.class);
    when(mockNS.hasWriteLock()).thenReturn(true);
    BlockManager bm = new BlockManager(mockNS, false, new HdfsConfiguration());
    LowRedundancyBlocks lowRedundancyBlocks = bm.neededReconstruction;
    long blkID1 = ThreadLocalRandom.current().nextLong();
    if (blkID1 < 0) {
        blkID1 *= -1;
    }
    long blkID2 = ThreadLocalRandom.current().nextLong();
    if (blkID2 < 0) {
        blkID2 *= -1;
    }
    BlockInfo block1 = genBlockInfo(blkID1);
    BlockInfo block2 = genBlockInfo(blkID2);
    // Adding QUEUE_LOW_REDUNDANCY block
    lowRedundancyBlocks.add(block1, 0, 0, 1, 1);
    // Adding QUEUE_LOW_REDUNDANCY block
    lowRedundancyBlocks.add(block2, 0, 0, 1, 1);
    List<List<BlockInfo>> chosenBlocks;
    // Choose 1 block from lowRedundancyBlocks. Then it should pick 1 block
    // from QUEUE_VERY_LOW_REDUNDANCY.
    chosenBlocks = lowRedundancyBlocks.chooseLowRedundancyBlocks(1);
    assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
    final BlockInfoContiguous info = new BlockInfoContiguous(block1, (short) 1);
    final BlockCollection mbc = mock(BlockCollection.class);
    when(mbc.getId()).thenReturn(1000L);
    when(mbc.getLastBlock()).thenReturn(info);
    when(mbc.getPreferredBlockSize()).thenReturn(block1.getNumBytes() + 1);
    when(mbc.isUnderConstruction()).thenReturn(true);
    ContentSummary cs = mock(ContentSummary.class);
    when(cs.getLength()).thenReturn((long) 1);
    when(mbc.computeContentSummary(bm.getStoragePolicySuite())).thenReturn(cs);
    info.setBlockCollectionId(1000);
    bm.addBlockCollection(info, mbc);
    DatanodeStorageInfo[] storageAry = { new DatanodeStorageInfo(dataNodes[0], new DatanodeStorage("s1")) };
    info.convertToBlockUnderConstruction(BlockUCState.UNDER_CONSTRUCTION, storageAry);
    DatanodeStorageInfo storage = mock(DatanodeStorageInfo.class);
    DatanodeDescriptor dn = mock(DatanodeDescriptor.class);
    when(dn.isDecommissioned()).thenReturn(true);
    when(storage.getState()).thenReturn(DatanodeStorage.State.NORMAL);
    when(storage.getDatanodeDescriptor()).thenReturn(dn);
    when(storage.removeBlock(any(BlockInfo.class))).thenReturn(true);
    when(storage.addBlock(any(BlockInfo.class))).thenReturn(DatanodeStorageInfo.AddBlockResult.ADDED);
    info.addStorage(storage, info);
    BlockInfo lastBlk = mbc.getLastBlock();
    when(mbc.getLastBlock()).thenReturn(lastBlk, info);
    bm.convertLastBlockToUnderConstruction(mbc, 0L);
    // Choose 1 block from lowRedundancyBlocks. Then it should pick 1 block
    // from QUEUE_VERY_LOW_REDUNDANCY.
    // This block remains and should not be skipped over.
    chosenBlocks = lowRedundancyBlocks.chooseLowRedundancyBlocks(1);
    assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
}

Example 2 with DatanodeStorage

use of org.apache.hadoop.hdfs.server.protocol.DatanodeStorage in project hadoop by apache.

the class BlockReportTestBase method getBlockReports.

// Generate a block report, optionally corrupting the generation
// stamp and/or length of one block.
private static StorageBlockReport[] getBlockReports(DataNode dn, String bpid, boolean corruptOneBlockGs, boolean corruptOneBlockLen) {
    Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = dn.getFSDataset().getBlockReports(bpid);
    // Send block report
    StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.size()];
    boolean corruptedGs = false;
    boolean corruptedLen = false;
    int reportIndex = 0;
    for (Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
        DatanodeStorage dnStorage = kvPair.getKey();
        BlockListAsLongs blockList = kvPair.getValue();
        // Walk the list of blocks until we find one each to corrupt the
        // generation stamp and length, if so requested.
        BlockListAsLongs.Builder builder = BlockListAsLongs.builder();
        for (BlockReportReplica block : blockList) {
            if (corruptOneBlockGs && !corruptedGs) {
                long gsOld = block.getGenerationStamp();
                long gsNew;
                do {
                    gsNew = rand.nextInt();
                } while (gsNew == gsOld);
                block.setGenerationStamp(gsNew);
                LOG.info("Corrupted the GS for block ID " + block);
                corruptedGs = true;
            } else if (corruptOneBlockLen && !corruptedLen) {
                long lenOld = block.getNumBytes();
                long lenNew;
                do {
                    lenNew = rand.nextInt((int) lenOld - 1);
                } while (lenNew == lenOld);
                block.setNumBytes(lenNew);
                LOG.info("Corrupted the length for block ID " + block);
                corruptedLen = true;
            }
            builder.add(new BlockReportReplica(block));
        }
        reports[reportIndex++] = new StorageBlockReport(dnStorage, builder.build());
    }
    return reports;
}

Example 3 with DatanodeStorage

use of org.apache.hadoop.hdfs.server.protocol.DatanodeStorage in project hadoop by apache.

the class TestPendingReconstruction method testBlockReceived.

/**
   * Test if DatanodeProtocol#blockReceivedAndDeleted can correctly update the
   * pending reconstruction. Also make sure the blockReceivedAndDeleted call is
   * idempotent to the pending reconstruction.
   */
@Test
public void testBlockReceived() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
    MiniDFSCluster cluster = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_COUNT).build();
        cluster.waitActive();
        DistributedFileSystem hdfs = cluster.getFileSystem();
        FSNamesystem fsn = cluster.getNamesystem();
        BlockManager blkManager = fsn.getBlockManager();
        final String file = "/tmp.txt";
        final Path filePath = new Path(file);
        short replFactor = 1;
        DFSTestUtil.createFile(hdfs, filePath, 1024L, replFactor, 0);
        // temporarily stop the heartbeat
        ArrayList<DataNode> datanodes = cluster.getDataNodes();
        for (int i = 0; i < DATANODE_COUNT; i++) {
            DataNodeTestUtils.setHeartbeatsDisabledForTests(datanodes.get(i), true);
        }
        hdfs.setReplication(filePath, (short) DATANODE_COUNT);
        BlockManagerTestUtil.computeAllPendingWork(blkManager);
        assertEquals(1, blkManager.pendingReconstruction.size());
        INodeFile fileNode = fsn.getFSDirectory().getINode4Write(file).asFile();
        BlockInfo[] blocks = fileNode.getBlocks();
        assertEquals(DATANODE_COUNT - 1, blkManager.pendingReconstruction.getNumReplicas(blocks[0]));
        LocatedBlock locatedBlock = hdfs.getClient().getLocatedBlocks(file, 0).get(0);
        DatanodeInfo existingDn = (locatedBlock.getLocations())[0];
        int reportDnNum = 0;
        String poolId = cluster.getNamesystem().getBlockPoolId();
        // report to NN
        for (int i = 0; i < DATANODE_COUNT && reportDnNum < 2; i++) {
            if (!datanodes.get(i).getDatanodeId().equals(existingDn)) {
                DatanodeRegistration dnR = datanodes.get(i).getDNRegistrationForBP(poolId);
                StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks(new DatanodeStorage("Fake-storage-ID-Ignored"), new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo(blocks[0], BlockStatus.RECEIVED_BLOCK, "") }) };
                cluster.getNameNodeRpc().blockReceivedAndDeleted(dnR, poolId, report);
                reportDnNum++;
            }
        }
        // IBRs are async, make sure the NN processes all of them.
        cluster.getNamesystem().getBlockManager().flushBlockOps();
        assertEquals(DATANODE_COUNT - 3, blkManager.pendingReconstruction.getNumReplicas(blocks[0]));
        // let the same datanodes report again
        for (int i = 0; i < DATANODE_COUNT && reportDnNum < 2; i++) {
            if (!datanodes.get(i).getDatanodeId().equals(existingDn)) {
                DatanodeRegistration dnR = datanodes.get(i).getDNRegistrationForBP(poolId);
                StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks(new DatanodeStorage("Fake-storage-ID-Ignored"), new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo(blocks[0], BlockStatus.RECEIVED_BLOCK, "") }) };
                cluster.getNameNodeRpc().blockReceivedAndDeleted(dnR, poolId, report);
                reportDnNum++;
            }
        }
        cluster.getNamesystem().getBlockManager().flushBlockOps();
        assertEquals(DATANODE_COUNT - 3, blkManager.pendingReconstruction.getNumReplicas(blocks[0]));
        // re-enable heartbeat for the datanode that has data
        for (int i = 0; i < DATANODE_COUNT; i++) {
            DataNodeTestUtils.setHeartbeatsDisabledForTests(datanodes.get(i), false);
            DataNodeTestUtils.triggerHeartbeat(datanodes.get(i));
        }
        Thread.sleep(5000);
        assertEquals(0, blkManager.pendingReconstruction.size());
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 4 with DatanodeStorage

use of org.apache.hadoop.hdfs.server.protocol.DatanodeStorage in project hadoop by apache.

the class BPServiceActor method blockReport.

/**
   * Report the list blocks to the Namenode
   * @return DatanodeCommands returned by the NN. May be null.
   * @throws IOException
   */
List<DatanodeCommand> blockReport(long fullBrLeaseId) throws IOException {
    final ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
    // Flush any block information that precedes the block report. Otherwise
    // we have a chance that we will miss the delHint information
    // or we will report an RBW replica after the BlockReport already reports
    // a FINALIZED one.
    ibrManager.sendIBRs(bpNamenode, bpRegistration, bpos.getBlockPoolId(), dn.getMetrics());
    long brCreateStartTime = monotonicNow();
    Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = dn.getFSDataset().getBlockReports(bpos.getBlockPoolId());
    // Convert the reports to the format expected by the NN.
    int i = 0;
    int totalBlockCount = 0;
    StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.size()];
    for (Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
        BlockListAsLongs blockList = kvPair.getValue();
        reports[i++] = new StorageBlockReport(kvPair.getKey(), blockList);
        totalBlockCount += blockList.getNumberOfBlocks();
    }
    // Send the reports to the NN.
    int numReportsSent = 0;
    int numRPCs = 0;
    boolean success = false;
    long brSendStartTime = monotonicNow();
    long reportId = generateUniqueBlockReportId();
    boolean useBlocksBuffer = bpRegistration.getNamespaceInfo().isCapabilitySupported(NamespaceInfo.Capability.STORAGE_BLOCK_REPORT_BUFFERS);
    blockReportSizes.clear();
    try {
        if (totalBlockCount < dnConf.blockReportSplitThreshold) {
            // Below split threshold, send all reports in a single message.
            DatanodeCommand cmd = bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), reports, new BlockReportContext(1, 0, reportId, fullBrLeaseId, true));
            blockReportSizes.add(calculateBlockReportPBSize(useBlocksBuffer, reports));
            numRPCs = 1;
            numReportsSent = reports.length;
            if (cmd != null) {
                cmds.add(cmd);
            }
        } else {
            // Send one block report per message.
            for (int r = 0; r < reports.length; r++) {
                StorageBlockReport[] singleReport = { reports[r] };
                DatanodeCommand cmd = bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), singleReport, new BlockReportContext(reports.length, r, reportId, fullBrLeaseId, true));
                blockReportSizes.add(calculateBlockReportPBSize(useBlocksBuffer, singleReport));
                numReportsSent++;
                numRPCs++;
                if (cmd != null) {
                    cmds.add(cmd);
                }
            }
        }
        success = true;
    } finally {
        // Log the block report processing stats from Datanode perspective
        long brSendCost = monotonicNow() - brSendStartTime;
        long brCreateCost = brSendStartTime - brCreateStartTime;
        dn.getMetrics().addBlockReport(brSendCost);
        final int nCmds = cmds.size();
        LOG.info((success ? "S" : "Uns") + "uccessfully sent block report 0x" + Long.toHexString(reportId) + ",  containing " + reports.length + " storage report(s), of which we sent " + numReportsSent + "." + " The reports had " + totalBlockCount + " total blocks and used " + numRPCs + " RPC(s). This took " + brCreateCost + " msec to generate and " + brSendCost + " msecs for RPC and NN processing." + " Got back " + ((nCmds == 0) ? "no commands" : ((nCmds == 1) ? "one command: " + cmds.get(0) : (nCmds + " commands: " + Joiner.on("; ").join(cmds)))) + ".");
    }
    scheduler.updateLastBlockReportTime(monotonicNow());
    scheduler.scheduleNextBlockReport();
    return cmds.size() == 0 ? null : cmds;
}

Example 5 with DatanodeStorage

use of org.apache.hadoop.hdfs.server.protocol.DatanodeStorage in project hadoop by apache.

the class TestDFSShell method getMaterializedReplicas.

private static List<MaterializedReplica> getMaterializedReplicas(MiniDFSCluster cluster) throws IOException {
    List<MaterializedReplica> replicas = new ArrayList<>();
    String poolId = cluster.getNamesystem().getBlockPoolId();
    List<Map<DatanodeStorage, BlockListAsLongs>> blocks = cluster.getAllBlockReports(poolId);
    for (int i = 0; i < blocks.size(); i++) {
        Map<DatanodeStorage, BlockListAsLongs> map = blocks.get(i);
        for (Map.Entry<DatanodeStorage, BlockListAsLongs> e : map.entrySet()) {
            for (Block b : e.getValue()) {
                replicas.add(cluster.getMaterializedReplica(i, new ExtendedBlock(poolId, b)));
            }
        }
    }
    return replicas;
}